diff --git a/.clang-format b/.clang-format index 5846e1951..407695b06 100644 --- a/.clang-format +++ b/.clang-format @@ -22,7 +22,7 @@ Cpp11BracedListStyle: 'true' KeepEmptyLinesAtTheStartOfBlocks: 'true' NamespaceIndentation: Inner CompactNamespaces: 'true' -PenaltyBreakString: '3' +PenaltyBreakString: '1000' SpaceBeforeParens: ControlStatements SpacesInAngles: 'false' SpacesInContainerLiterals: 'false' diff --git a/.drone.jsonnet b/.drone.jsonnet index 4016808cc..0e878bbec 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -201,6 +201,16 @@ local windows_cross_pipeline(name, }] else []) ); +local live_test_step(image, mode) = { + name: 'live tests (' + mode + ')', + image: image, + pull: 'always', + commands: apt_setup(image, default_test_deps) + [ + 'cd build', + './tests/testLive --' + mode + ' --log-level warning --colour-mode ansi -d yes "[file]"', + ], +}; + // Live Pro-backend integration test: build testAll with the dev-server hook, stand up an ephemeral // backend (throwaway postgres + flask, provider_dry_run) via tests/pro_backend/run-dev-backend.sh, // and run the [pro_live] suite against it. The backend is a separate Python service, checked out at @@ -409,6 +419,20 @@ local static_build(name, // Various debian builds debian_build('Debian sid', docker_base + 'debian-sid'), + // Debian sid with session-router + live file transfer tests + local live_image = docker_base + 'debian-sid'; + debian_build( + 'Debian sid (live tests)', + live_image, + cmake_extra='-DENABLE_NETWORKING=ON -DENABLE_NETWORKING_SROUTER=ON -DBUILD_LIVE_TESTS=ON', + ) + { + steps: super.steps + [ + live_test_step(live_image, 'onionreq'), + live_test_step(live_image, 'srouter'), + live_test_step(live_image, 'direct'), + ], + }, + // Live Pro-backend integration tests (ephemeral backend + [pro_live]). pro_backend_live_pipeline('Debian sid (Pro backend live)', docker_base + 'debian-sid'), @@ -433,7 +457,7 @@ local static_build(name, ]), // Macos builds: - mac_builder('macOS Intel (Release)', allow_test_fail=true/*the current intel mac has issues*/), + //mac_builder('macOS Intel (Release)', allow_test_fail=true/*the current intel mac has issues*/), mac_builder('macOS Arm64 (Release)', arch='arm64'), mac_builder('macOS Arm64 (Debug)', arch='arm64', build_type='Debug'), diff --git a/.gitignore b/.gitignore index 1fe18bd5a..5fade4edc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /build*/ /compile_commands.json +# Downloaded and generated by utils/update-ip-country-db.py, required by -DWITH_IP_GEOLOCATION=ON +/src/network/ip_country/data.cpp /.cache/ +/.claude/ /.vscode/ .DS_STORE diff --git a/.gitmodules b/.gitmodules index a029c2087..5a2ee4c52 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,24 +1,21 @@ -[submodule "external/libsodium-internal"] - path = external/libsodium-internal - url = https://github.com/session-foundation/libsodium-internal.git [submodule "tests/Catch2"] path = tests/Catch2 url = https://github.com/catchorg/Catch2 [submodule "external/ios-cmake"] path = external/ios-cmake url = https://github.com/leetal/ios-cmake -[submodule "external/zstd"] - path = external/zstd - url = https://github.com/facebook/zstd.git [submodule "external/protobuf"] path = external/protobuf url = https://github.com/protocolbuffers/protobuf.git [submodule "external/session-router"] path = external/session-router url = https://github.com/session-foundation/session-router.git -[submodule "external/simdutf"] - path = external/simdutf - url = https://github.com/simdutf/simdutf.git +[submodule "external/session-sqlite"] + path = external/session-sqlite + url = https://github.com/session-foundation/session-sqlite.git [submodule "external/date"] path = external/date url = https://github.com/HowardHinnant/date.git +[submodule "cmake/session-deps"] + path = cmake/session-deps + url = https://github.com/session-foundation/session-deps.git diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..d06dac03d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,83 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build Commands + +```bash +# Configure (out-of-source build required) +cmake -G Ninja -S . -B build-claude + +# Build +cmake --build build-claude --parallel --verbose + +# Run tests +./build-claude/tests/testAll [test-tag-or-name] + +# Regenerate protobuf files +cmake --build build-claude --target regen-protobuf --parallel +``` + +### Notable CMake Options + +- `-DBUILD_STATIC_DEPS=ON` — force all deps to build statically (no system libs) +- `-DENABLE_ONIONREQ=ON/OFF` — include onion request / network functionality (default ON) +- `-DWARNINGS_AS_ERRORS=ON` — treat warnings as errors +- `-DSUBMODULE_CHECK=OFF` — skip submodule freshness checks (useful during dev) +- `-DWITH_IP_GEOLOCATION=ON` — bundle the DB-IP IP-to-country database, +1.79MB (default OFF, in + which case `session::ip_country` lookups all report unknown). Requires running + `utils/update-ip-country-db.py` first: the generated table is not committed, and cmake fails with + instructions if it is missing. + +## Architecture Overview + +This is **libsession-util**, the C++20 utility library for Session clients. It provides: + +1. **Cryptographic primitives** (`libsession::crypto`) — Ed25519/X25519 keys, blinding, hashing, encryption (session protocol, multi-encrypt, attachments), XEd25519 signatures. + +2. **Config sync system** (`libsession::config`) — CRDT-style distributed config that syncs across Session devices via swarm storage. Each config type has a namespace: + - `UserProfile`, `Contacts`, `ConvoInfoVolatile`, `UserGroups` — per-user configs + - `GroupKeys`, `GroupInfo`, `GroupMembers` — shared group configs (closed groups) + - `Local` — device-local config (never pushed to swarm) + - Config messages use bt-encoding (bencode), seqno-based CRDT merge with deterministic tie-breaking. See `docs/api/docs/config_merge_logic.md` for protocol details. + +3. **Core** (`libsession::core`) — Persistent client state backed by SQLite. The `Core` class owns `CoreComponent`-derived members (`Globals`, `Devices`, `Pro`) that share a connection pool. Migrations live in `src/core/schema/` as `NNN_name.sql` or `NNN_name.cpp` files. + +4. **Onion requests** (`libsession::onionreq`, optional) — Builder/parser for onion-routed requests to the Session network. + +### Library Targets and Dependencies + +``` +util ← file, logging, util (uses zstd, simdutf) +crypto ← util + libsodium (blinding, ed25519, session_encrypt, etc.) +config ← crypto + libsodium + protos (all config types) +core ← crypto + SQLite + mlkem768 (PQC key encapsulation) +onionreq ← crypto + quic + nettle (optional) +``` + +All targets are aliased as `libsession::util`, `libsession::crypto`, etc. + +### Header Layout + +Public headers are in `include/session/`: +- `include/session/config/` — config type headers (`.h` = C API, `.hpp` = C++ API) +- `include/session/config/groups/` — closed group configs (keys, info, members) +- `include/session/core/` — Core persistent state components +- `include/session/onionreq/` — onion request types + +### Dependency System + +Dependencies are managed via `cmake/session-deps/` which provides `session_dep()` and `session_dep_or_submodule()` macros. These first try system libraries; if not found they fall back to static builds. External submodules live in `external/` (oxen-logging, nlohmann-json, ios-cmake, protobuf, oxen-libquic). + +### Tests + +Tests use Catch2. Most tests are compiled into `testAll`; logging tests are isolated in `testLogging` because they modify global sink/level state. Filter tests with Catch2 tag syntax, e.g. `./Build/tests/testAll "[config]"`. + +### Dual C/C++ API + +Many headers come in pairs: `foo.h` (C API for FFI use) and `foo.hpp` (C++ API). The C API generally is a wrapper around the primary C++ API. When adding new public functionality, consider whether a C API is needed. + +## Code Style + +- **Prefer DRY code**: when logic is duplicated across two or more call sites, extract a shared helper. Do this proactively when writing new code, not only when asked. +- **Specify the shape upfront**: when asked to implement something that overlaps with existing code, identify and extract the shared piece before writing the new code, so duplication never appears in the first place. diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e4365205..4245461fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ if(CCACHE_PROGRAM) endif() project(libsession-util - VERSION 1.9.1 + VERSION 2.0.0 DESCRIPTION "Session client utility library" LANGUAGES ${LANGS}) @@ -28,6 +28,7 @@ set(LIBSESSION_LIBVERSION ${PROJECT_VERSION}) include(GNUInstallDirs) list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +include(SessionSchema) # No in-source building include(MacroEnsureOutOfSourceBuild) @@ -58,11 +59,16 @@ else() set(static_default ON) endif() -option(BUILD_STATIC_DEPS "Build all dependencies statically rather than trying to link to them on the system" ${static_default}) +# Override the default OFF value set in cmake/session-deps/Deps.cmake so that BUILD_STATIC_DEPS +# defaults to the same value as BUILD_SHARED_LIBS (i.e. static by default). +set(BUILD_STATIC_DEPS ${static_default} CACHE BOOL "Build all dependencies statically rather than trying to link to them on the system") + +include(cmake/session-deps/Deps.cmake) + option(STATIC_BUNDLE "Build a single static .a containing everything (both code and dependencies)" ${BUILD_STATIC_DEPS}) if(STATIC_BUNDLE AND NOT BUILD_STATIC_DEPS) - message(FATAL_ERROR "STATIC_BUNDLE requires BUILD_STATIC_DEPS=ON (cannot build a static bundle when using shared/system libs)") + message(FATAL_ERROR "STATIC_BUNDLE requires BUILD_STATIC_DEPS to be enabled") endif() if(BUILD_SHARED_LIBS OR libsession_IS_TOPLEVEL_PROJECT) @@ -79,16 +85,20 @@ else() set(use_lto_default ON) endif() -option(WARNINGS_AS_ERRORS "Treat all compiler warnings as errors" OFF) option(WARN_UNUSED_PARAMETERS "Enabled unused parameter warnings" ON) +option(WARNINGS_AS_ERRORS "Treat all compiler warnings as errors" OFF) +option(FATAL_MISSING_DECLARATIONS "Developer/CI option: fatal error on non-static definitions without prior declarations (-Werror=missing-declarations)" OFF) option(STATIC_LIBSTD "Statically link libstdc++/libgcc" ${default_static_libstd}) option(USE_LTO "Use Link-Time Optimization" ${use_lto_default}) -# Provide this as an option for now because GMP and Desktop are sometimes unhappy with each other. -option(ENABLE_NETWORKING "Build with networking functionality" ON) -option(ENABLE_NETWORKING_SROUTER "Build with session-router networking support (requires ENABLE_NETWORKING)" ON) +option(ENABLE_NETWORKING_SROUTER "Build with session-router networking support" ON) + +# Off by default: it adds ~1.8MB of database to the binary, and a client that already ships its own +# geo data wants nothing to do with it. With it off the lookup API still exists and reports every +# address as unknown, so nothing needs an #ifdef. +option(WITH_IP_GEOLOCATION "Build with the bundled DB-IP IP-to-country database" OFF) if(USE_LTO) include(CheckIPOSupported) @@ -107,6 +117,17 @@ if(IPO_ENABLED AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) endif() +# USE_LTO is the single LTO knob. Two submodule-provided options default ON independently of it: +# SESSIONDEPS_LTO (session-deps, several copies sharing one cache variable) controls LTO for static +# dependency builds, and WITH_LTO (session-router's enable_lto.cmake) controls LTO for session-router's +# own targets. Leaving either ON in a non-LTO build breaks the link: LTO'd static archives can't be +# consumed by a non-LTO link -- clang in particular emits pure bitcode archives that a plain bfd link +# reports as "file format not recognized", or silently drops the members (undefined references). Force +# both to follow USE_LTO, seeding these shared cache variables before any submodule evaluates its own +# option() so there is exactly one LTO switch. +set(SESSIONDEPS_LTO ${USE_LTO} CACHE BOOL "Use LTO for static dependency builds, where supported" FORCE) +set(WITH_LTO ${USE_LTO} CACHE BOOL "enable lto on compile time" FORCE) + if(STATIC_LIBSTD) add_link_options(-static-libstdc++) if(NOT CMAKE_CXX_COMPILER_ID MATCHES Clang) @@ -122,30 +143,24 @@ include(AddStaticBundleLib) # Always build PIC set(CMAKE_POSITION_INDEPENDENT_CODE ON) +# For a static-deps build we build everything from source; don't let oxen-logging pick up system +# fmt/spdlog via find_package. Beyond the usual reasons to prefer our own versions, a system +# fmt/spdlog is built against libstdc++ and cannot be linked into a libc++ build (undefined +# std::__1 symbols), so force the bundled submodules which build with our toolchain. +if(BUILD_STATIC_DEPS) + set(OXEN_LOGGING_FORCE_SUBMODULES ON CACHE INTERNAL "") +endif() + add_subdirectory(external) -if(ENABLE_NETWORKING) - if(NOT TARGET nettle::nettle) - if(BUILD_STATIC_DEPS) - message(FATAL_ERROR "Internal error: nettle::nettle target (expected via libquic BUILD_STATIC_DEPS) not found") - else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(NETTLE REQUIRED IMPORTED_TARGET nettle) - add_library(nettle INTERFACE) - target_link_libraries(nettle INTERFACE PkgConfig::NETTLE) - add_library(nettle::nettle ALIAS nettle) - endif() - endif() -endif() +session_dep(nettle 3) +# Unicode normalisation and case folding for mnemonic input; see src/mnemonics/mnemonics.cpp. +session_dep(libutf8proc 2.5) add_subdirectory(src) add_subdirectory(proto) -if (BUILD_STATIC_DEPS) - include(StaticBuild) -endif() - if(STATIC_BUNDLE) include(combine_archives) diff --git a/cmake/SessionSchema.cmake b/cmake/SessionSchema.cmake new file mode 100644 index 000000000..1f7285b30 --- /dev/null +++ b/cmake/SessionSchema.cmake @@ -0,0 +1,107 @@ +# Generates a database migration registry from a directory of migration files. +# +# Any file in the calling directory named NNN_*.sql or NNN_*.cpp is a one-time migration; see +# src/core/schema/README for the rules they follow. This turns them into a +# `std::span` named MIGRATIONS in the requested namespace, +# for passing to Core (as its own registry, or via a schema_extension option). +# +# session_schema_dir( +# TARGET # target the generated sources are compiled into +# NAMESPACE # namespace to define the registry in, e.g. session::core::schema +# DECLARE_HEADER
# header declaring `extern const std::span +# # MIGRATIONS;` in that namespace, included by the generated +# # definition so the two cannot drift apart +# ) +# +# Migration functions always take (session::sqlite::Connection&, session::core::Core&) regardless of +# which namespace they live in: the Migration type is Core's, and a layer above Core has no +# instance of itself to be handed during Core construction anyway. + +set(SESSION_SCHEMA_TEMPLATE_DIR "${CMAKE_CURRENT_LIST_DIR}/schema") + +function(session_schema_dir) + cmake_parse_arguments(PARSE_ARGV 0 SCHEMA "" "TARGET;NAMESPACE;DECLARE_HEADER" "") + + foreach(required TARGET NAMESPACE DECLARE_HEADER) + if(NOT SCHEMA_${required}) + message(FATAL_ERROR "session_schema_dir: ${required} is required") + endif() + endforeach() + + set(SCHEMA_NAMESPACE "${SCHEMA_NAMESPACE}") + set(SCHEMA_DECLARE_HEADER "${SCHEMA_DECLARE_HEADER}") + + # Watch the directory so that adding or removing a migration re-runs CMake: + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ".") + + file(GLOB SCHEMA_FILES "[0-9]*.sql" "[0-9]*.cpp") + + # Order migrations by the name recorded in migrations_applied, not by filename: the extension + # is not part of a migration's identity, and including it flips the order whenever one name is + # a prefix of another, since "001_foo+002.sql" sorts before "001_foo.sql" ('+' is 0x2B, '.' is + # 0x2E). + # + # Decorate, sort, undecorate. The separator has to sort below every character a name can + # contain or the prefix case breaks again one level down, so it is a control character rather + # than any punctuation. + string(ASCII 1 SCHEMA_SEP) + list(TRANSFORM SCHEMA_FILES REPLACE "^(.*/)([^/]*)\\.(sql|cpp)$" "\\2${SCHEMA_SEP}\\1\\2.\\3" + OUTPUT_VARIABLE SCHEMA_DECORATED) + list(SORT SCHEMA_DECORATED) + list(TRANSFORM SCHEMA_DECORATED REPLACE "^[^${SCHEMA_SEP}]*${SCHEMA_SEP}" "" + OUTPUT_VARIABLE SCHEMA_FILES) + + list(LENGTH SCHEMA_FILES SCHEMA_COUNT) + + set(DECLARATIONS "") + set(SCHEMA_ENTRIES "") + set(SCHEMA_SOURCES "") + + foreach(f IN LISTS SCHEMA_FILES) + get_filename_component(filename "${f}" NAME) + string(REGEX REPLACE "\\.(sql|cpp)$" "" basename "${filename}") + if(CMAKE_MATCH_1 STREQUAL "sql") + set(is_sql TRUE) + else() + set(is_sql FALSE) + endif() + + # Watch individual files so edits trigger a re-configure: + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${f}") + + string(MAKE_C_IDENTIFIER "apply_${basename}" FUNC_NAME) + if(is_sql) + file(RELATIVE_PATH SCHEMA_FULL_FILENAME "${PROJECT_SOURCE_DIR}" "${f}") + file(READ "${f}" SCHEMA_SQL) + set(wrapper_cpp "${CMAKE_CURRENT_BINARY_DIR}/apply_schema__${basename}__sql.cpp") + configure_file("${SESSION_SCHEMA_TEMPLATE_DIR}/apply_schema.cpp.in" "${wrapper_cpp}" @ONLY) + list(APPEND SCHEMA_SOURCES "${wrapper_cpp}") + else() + list(APPEND SCHEMA_SOURCES "${f}") + endif() + + string(APPEND DECLARATIONS "extern void ${FUNC_NAME}(session::sqlite::Connection&, session::core::Core&);\n") + string(APPEND SCHEMA_ENTRIES " session::core::schema::Migration{\"${basename}\", &${FUNC_NAME}},\n") + endforeach() + + # An optional full_schema.sql holds the schema as it stands after every migration above. A + # database with none of this owner's migrations applied is built from it directly and has them + # all recorded without running, so the file -- not the accumulated migration chain -- is what + # anyone reads to see the current schema. It has no numeric prefix, so the glob above skips it. + set(SCHEMA_FULL "") + set(full_schema "${CMAKE_CURRENT_SOURCE_DIR}/full_schema.sql") + if(EXISTS "${full_schema}") + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${full_schema}") + file(READ "${full_schema}" SCHEMA_FULL) + endif() + + configure_file("${SESSION_SCHEMA_TEMPLATE_DIR}/schema_migrations.hpp.in" + "${CMAKE_CURRENT_BINARY_DIR}/schema_migrations.hpp" @ONLY) + configure_file("${SESSION_SCHEMA_TEMPLATE_DIR}/schema_registry.cpp.in" + "${CMAKE_CURRENT_BINARY_DIR}/schema_registry.cpp" @ONLY) + list(APPEND SCHEMA_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/schema_registry.cpp") + + target_sources(${SCHEMA_TARGET} PRIVATE ${SCHEMA_SOURCES}) + # The generated wrappers include schema_migrations.hpp from alongside themselves. + target_include_directories(${SCHEMA_TARGET} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") +endfunction() diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake deleted file mode 100644 index 49863f8c8..000000000 --- a/cmake/StaticBuild.cmake +++ /dev/null @@ -1,224 +0,0 @@ -# cmake bits to do a full static build, downloading and building all dependencies. - -# Most of these are CACHE STRINGs so that you can override them using -DWHATEVER during cmake -# invocation to override. - -set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") - -include(ExternalProject) - -set(DEPS_DESTDIR ${CMAKE_BINARY_DIR}/static-deps) -set(DEPS_SOURCEDIR ${CMAKE_BINARY_DIR}/static-deps-sources) - -file(MAKE_DIRECTORY ${DEPS_DESTDIR}/include) - -add_library(libsession-external-libs INTERFACE IMPORTED GLOBAL) -target_include_directories(libsession-external-libs SYSTEM BEFORE INTERFACE ${DEPS_DESTDIR}/include) - -set(deps_cc "${CMAKE_C_COMPILER}") -set(deps_cxx "${CMAKE_CXX_COMPILER}") - - -function(expand_urls output source_file) - set(expanded) - foreach(mirror ${ARGN}) - list(APPEND expanded "${mirror}/${source_file}") - endforeach() - set(${output} "${expanded}" PARENT_SCOPE) -endfunction() - -function(add_static_target target ext_target libname) - add_library(${target} STATIC IMPORTED GLOBAL) - add_dependencies(${target} ${ext_target}) - target_link_libraries(${target} INTERFACE libsession-external-libs) - set_target_properties(${target} PROPERTIES - IMPORTED_LOCATION ${DEPS_DESTDIR}/lib/${libname} - ) - if(ARGN) - target_link_libraries(${target} INTERFACE ${ARGN}) - endif() - libsession_static_bundle(${target}) -endfunction() - - - -set(cross_host "") -set(cross_rc "") -if(CMAKE_CROSSCOMPILING) - if(APPLE AND NOT ARCH_TRIPLET AND APPLE_TARGET_TRIPLE) - set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") - endif() - set(cross_host "--host=${ARCH_TRIPLET}") - if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) - set(cross_rc "WINDRES=${CMAKE_RC_COMPILER}") - endif() -endif() -if(ANDROID) - set(android_toolchain_suffix linux-android) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) - set(cross_host "--host=x86_64-linux-android") - set(android_compiler_prefix x86_64) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix x86_64) - set(android_toolchain_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(cross_host "--host=i686-linux-android") - set(android_compiler_prefix i686) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix i686) - set(android_toolchain_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(cross_host "--host=armv7a-linux-androideabi") - set(android_compiler_prefix armv7a) - set(android_compiler_suffix linux-androideabi${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix arm) - set(android_toolchain_suffix linux-androideabi) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) - set(cross_host "--host=aarch64-linux-android") - set(android_compiler_prefix aarch64) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix aarch64) - set(android_toolchain_suffix linux-android) - else() - message(FATAL_ERROR "unknown android arch: ${CMAKE_ANDROID_ARCH_ABI}") - endif() - set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}-clang") - set(deps_cxx "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}-clang++") - set(deps_ld "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_toolchain_suffix}-ld") - set(deps_ranlib "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_toolchain_prefix}-${android_toolchain_suffix}-ranlib") - set(deps_ar "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_toolchain_prefix}-${android_toolchain_suffix}-ar") -endif() - -set(deps_CFLAGS "-O2") -set(deps_CXXFLAGS "-O2") - -if(CMAKE_C_COMPILER_LAUNCHER) - set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}") -endif() -if(CMAKE_CXX_COMPILER_LAUNCHER) - set(deps_cxx "${CMAKE_CXX_COMPILER_LAUNCHER} ${deps_cxx}") -endif() - -if(WITH_LTO) - set(deps_CFLAGS "${deps_CFLAGS} -flto") -endif() - -if(APPLE AND CMAKE_OSX_DEPLOYMENT_TARGET) - if(SDK_NAME) - set(deps_CFLAGS "${deps_CFLAGS} -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - else() - set(deps_CFLAGS "${deps_CFLAGS} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif() -endif() - -if(_winver) - set(deps_CFLAGS "${deps_CFLAGS} -D_WIN32_WINNT=${_winver}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -D_WIN32_WINNT=${_winver}") -endif() - - -if("${CMAKE_GENERATOR}" STREQUAL "Unix Makefiles") - set(_make $(MAKE)) -else() - set(_make make) -endif() - - -# Builds a target; takes the target name (e.g. "readline") and builds it in an external project with -# target name suffixed with `_external`. Its upper-case value is used to get the download details -# (from the variables set above). The following options are supported and passed through to -# ExternalProject_Add if specified. If omitted, these defaults are used: -set(build_def_DEPENDS "") -set(build_def_PATCH_COMMAND "") -set(build_def_CONFIGURE_COMMAND ./configure ${cross_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic - "CC=${deps_cc}" "CXX=${deps_cxx}" "CFLAGS=${deps_CFLAGS}" "CXXFLAGS=${deps_CXXFLAGS}" ${cross_rc}) -set(build_def_CONFIGURE_EXTRA "") -set(build_def_BUILD_COMMAND ${_make}) -set(build_def_INSTALL_COMMAND ${_make} install) -set(build_def_BUILD_BYPRODUCTS ${DEPS_DESTDIR}/lib/lib___TARGET___.a ${DEPS_DESTDIR}/include/___TARGET___.h) - -function(build_external target) - set(options DEPENDS PATCH_COMMAND CONFIGURE_COMMAND CONFIGURE_EXTRA BUILD_COMMAND INSTALL_COMMAND BUILD_BYPRODUCTS) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "${options}") - foreach(o ${options}) - if(NOT DEFINED arg_${o}) - set(arg_${o} ${build_def_${o}}) - endif() - endforeach() - string(REPLACE ___TARGET___ ${target} arg_BUILD_BYPRODUCTS "${arg_BUILD_BYPRODUCTS}") - - string(TOUPPER "${target}" prefix) - expand_urls(urls ${${prefix}_SOURCE} ${${prefix}_MIRROR}) - set(extract_ts) - if(NOT CMAKE_VERSION VERSION_LESS 3.24) - set(extract_ts DOWNLOAD_EXTRACT_TIMESTAMP ON) - endif() - ExternalProject_Add("${target}_external" - DEPENDS ${arg_DEPENDS} - BUILD_IN_SOURCE ON - PREFIX ${DEPS_SOURCEDIR} - URL ${urls} - URL_HASH ${${prefix}_HASH} - DOWNLOAD_NO_PROGRESS ON - PATCH_COMMAND ${arg_PATCH_COMMAND} - CONFIGURE_COMMAND ${arg_CONFIGURE_COMMAND} ${arg_CONFIGURE_EXTRA} - BUILD_COMMAND ${arg_BUILD_COMMAND} - INSTALL_COMMAND ${arg_INSTALL_COMMAND} - BUILD_BYPRODUCTS ${arg_BUILD_BYPRODUCTS} - EXCLUDE_FROM_ALL ON - ${extract_ts} - ) -endfunction() - - -set(apple_cflags_arch) -set(apple_cxxflags_arch) -set(apple_ldflags_arch) -set(gmp_build_host "${cross_host}") -if(APPLE AND CMAKE_CROSSCOMPILING) - if(gmp_build_host MATCHES "^(.*-.*-)ios([0-9.]+)(-.*)?$") - set(gmp_build_host "${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") - endif() - if(gmp_build_host MATCHES "^(.*-.*-.*)-simulator$") - set(gmp_build_host "${CMAKE_MATCH_1}") - endif() - - set(apple_arch) - if(ARCH_TRIPLET MATCHES "^(arm|aarch)64.*") - set(apple_arch "arm64") - elseif(ARCH_TRIPLET MATCHES "^x86_64.*") - set(apple_arch "x86_64") - else() - message(FATAL_ERROR "Don't know how to specify -arch for GMP for ${ARCH_TRIPLET} (${APPLE_TARGET_TRIPLE})") - endif() - - set(apple_cflags_arch " -arch ${apple_arch}") - set(apple_cxxflags_arch " -arch ${apple_arch}") - if(CMAKE_OSX_DEPLOYMENT_TARGET) - if (SDK_NAME) - set(apple_ldflags_arch " -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - elseif(CMAKE_OSX_DEPLOYMENT_TARGET) - set(apple_ldflags_arch " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif() - endif() - set(apple_ldflags_arch "${apple_ldflags_arch} -arch ${apple_arch}") - - if(CMAKE_OSX_SYSROOT) - foreach(f c cxx ld) - set(apple_${f}flags_arch "${apple_${f}flags_arch} -isysroot ${CMAKE_OSX_SYSROOT}") - endforeach() - endif() -elseif(gmp_build_host STREQUAL "") - set(gmp_build_host "--build=${CMAKE_LIBRARY_ARCHITECTURE}") -endif() - -link_libraries(-static-libstdc++) -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - link_libraries(-static-libgcc) -endif() -if(MINGW) - link_libraries(-Wl,-Bstatic -lpthread) -endif() diff --git a/cmake/schema/apply_schema.cpp.in b/cmake/schema/apply_schema.cpp.in new file mode 100644 index 000000000..7780f91a9 --- /dev/null +++ b/cmake/schema/apply_schema.cpp.in @@ -0,0 +1,12 @@ +// Auto-generated by CMake from @SCHEMA_FULL_FILENAME@. Do not edit. +#include "schema_migrations.hpp" + +namespace @SCHEMA_NAMESPACE@ { + +void @FUNC_NAME@(session::sqlite::Connection& conn, session::core::Core&) { + conn.sql.exec(R"_SQL_DELIM_( +@SCHEMA_SQL@ +)_SQL_DELIM_"); +} + +} // namespace @SCHEMA_NAMESPACE@ diff --git a/cmake/schema/schema_migrations.hpp.in b/cmake/schema/schema_migrations.hpp.in new file mode 100644 index 000000000..a0ee5141a --- /dev/null +++ b/cmake/schema/schema_migrations.hpp.in @@ -0,0 +1,14 @@ +// Auto-generated by CMake from cmake/schema/schema_migrations.hpp.in. Do not edit. +#pragma once + +#include + +namespace session::core { +class Core; +} + +namespace @SCHEMA_NAMESPACE@ { + +@DECLARATIONS@ + +} // namespace @SCHEMA_NAMESPACE@ diff --git a/cmake/schema/schema_registry.cpp.in b/cmake/schema/schema_registry.cpp.in new file mode 100644 index 000000000..65ad34abb --- /dev/null +++ b/cmake/schema/schema_registry.cpp.in @@ -0,0 +1,19 @@ +// Auto-generated by CMake from cmake/schema/schema_registry.cpp.in. Do not edit. + +#include <@SCHEMA_DECLARE_HEADER@> + +#include "schema_migrations.hpp" + +namespace @SCHEMA_NAMESPACE@ { + +// Explicitly sized rather than deduced: a directory with a full_schema.sql and no deltas yet has +// no migrations at all, and class template argument deduction cannot cope with an empty list. +static const std::array migrations = { +@SCHEMA_ENTRIES@ +}; + +const std::span MIGRATIONS{migrations}; + +const std::string_view FULL_SCHEMA = R"_SQL_DELIM_(@SCHEMA_FULL@)_SQL_DELIM_"; + +} // namespace @SCHEMA_NAMESPACE@ diff --git a/cmake/session-deps b/cmake/session-deps new file mode 160000 index 000000000..30b200920 --- /dev/null +++ b/cmake/session-deps @@ -0,0 +1 @@ +Subproject commit 30b200920fdeffedc898a8e3fa21259104098069 diff --git a/docs/messages-v1.md b/docs/messages-v1.md new file mode 100644 index 000000000..c88a5387b --- /dev/null +++ b/docs/messages-v1.md @@ -0,0 +1,53 @@ +# Session message format v1 + +Session currently uses an overcomplicated Protobuf message encoding; since PFS+PQ encryption +requires a backwards-incompatible message encryption change already, this is the right time to also +moderately clean up that message format. + +The new message format is described in ./protocol-v2.md, in the section "One-to-one Message +Encryption". + +The existing format is as follows, from outermost (fully encoded and encrypted) to innermost (fully +decoded): + +- Protobuf `WebSocketMessage` - this is a pointless wrapper. It is always constructed with + type=Type::REQUEST, and `request` set to a WebSocketRequestMessage. + +- Protobuf `WebSocketRequestMessage` - this is another pointless wrapper. It is always constructed + with everything empty except `body`, and body contains a *serialized* Envelope. + +- The bytes then decode to a protobuf `Envelope` value; this contains: + - type=Type::SESSION_MESSAGE (also pointless: CLOSED_GROUP_MESSAGE is no longer used). + - timestamp=...(value is sometimes used with v1 message, but will not be used in v2 messages)... + - content=encrypted body (bytes) + - proSig = 64-bytes + +`proSig` here is a signature over the encrypted body, but cannot yet be verified until later in +message handling (once the pubkey is known, which is inside the decrypted plaintext payload), and so +is simply retained for later use. This *may* be an actual valid Pro signature, or may be a dummy +value included to obscure whether the message actually includes a valid Pro signature or not. + +The encrypted body is a libsodium "sealed box", which encrypts the value: + + plaintext = Msg || Padding || SenderEd || Sig + +where Padding consists of an initial 0x80 byte followed by any number of 0x00 bytes (to obfuscate +message size from someone who observes the encrypted content; this is typically selected to make the +combined Msg || Padding value a multiple of 160 bytes). SenderEd is the 32-byte Ed25519 (not +X25519) pubkey of the sender, which can be converted to X25519 to obtain the session_id (without the +leading 0x05 prefix byte). + +Sig here is an Ed25519 signature of the value: + + Msg || Padding || SenderEd[32B] || RecipientX[32B] + +where RecipientX is the target recipient X25519 pubkey (that is: 33-byte raw session ID, with the +leading 0x05 byte stripped off). Note that RecipientX is implied and not actually included in the +message. + +The Sig value is checked against the implied message, and if this signature failed, the message is +discarded as invalid. + +If accepted, the Msg value (i.e. with padding removed) is then parsed as a protobuf Content. + +Further details of message handling is not dealt with here. diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md new file mode 100644 index 000000000..ebac72b1a --- /dev/null +++ b/docs/protocol-v2.md @@ -0,0 +1,825 @@ +# Session Protocol v2 - design details + +This update to the Session protocol aims to add multiple capabilities to Session, with the headline +features being perfect forward secrecy (PFS) and post-quantum (PQ) encryption for 1-1 direct +messages (DMs). + +This document breaks down the design into multiple components. + +The underlying goal here is protection of DM message content: it should be impossible for someone +with (only) the long-term Session secret key to decrypt past, present, or future messages. +Potential adversaries here are assumed to be able to log all stored swarm messages (e.g. by access +to a swarm storage server) for future decryption if that Session long-term private key is eventually +compromised. + +## Linked device configuration + +Currently Session config data stored in an account's swarm is encrypted with the long-term Session +encryption key (hereafter referred to as the root key). While acceptable for some data, this is not +usable for the PFS keys used in DMs as any later compromise of the root key. The future adversary +with a compromised root key could use that key to decrypt past messages containing temporary +decryption keys, and then use those to decrypt the message contents. + +Thus the first part of the protocol is to construct shared keys among linked devices and use these +keys for all PFS-related key storage. + +There are two types of messages involved here: +- Shared encrypted data +- Linked device group join handshakes + +Shared encrypted data (stored in namespace 21) is where all unique shared encrypted data will be +stored, including keys usable for PFS and PQ. Join handshakes (also stored in this namespace) will +be used by a new device asking to join the encrypted device group. + +Both message types share the one namespace rather than being split across two: each is a bt-encoded +dict whose `""` key carries a single-character type tag — `"G"` for a device group message, `"L"` +for a link request — so a client distinguishes them from the message itself rather than from where +it was stored. + +### Inner device data + +The innermost (plaintext) layer is a bt-encoded dict of per-device records. It is not a +libsession-util config object: config messages derive their encryption key from the account root +key, which would make the data readable by any holder of that key, and a device outside the group +must be able to see and verify a message that it cannot read. + +The merge rules follow those of config data: each device writes only its own record, a record is +accepted only if its `seq` exceeds the value already held, and a record absent from a message is +left unchanged rather than removed (see "Removed devices" below). + +Device data might consist of the following (but bencoded, not json; note also that the verbose key +names used here are not the actual key names that will be in the encoded data): + + { + "devices": { + "unique-client-identifier1": { + "type": "i", + "seq": 1234, + "timestamp": 1777777777, + "description": "iPhone OS 28 - Alice's phone", + "version": 2025000, + "unknown_extra_field": { "x": 123, "y": [1,3,997,"x"] }, + "device_pubkey_x25519": "abcdef123...(32 bytes)", + "device_pubkey_mlkem768": "...(1184 bytes)..." + }, + "unique-client-identifier2": { + "type": "d", + "seq": 10, + "timestamp": 1777777778, + "description": "Linux - betwixt", + "version": 2000000, + "device_pubkey_x25519": "987654321...", + "device_pubkey_mlkem768": "...(1184 bytes)..." + }, + "unique-client-identifier3": { + "type": "a", + "seq": 515, + "timestamp": 1777777779, + "description": "Alice's tablet - Android 17", + "version": 3000007, + "device_pubkey_x25519": "999666333...", + "device_pubkey_mlkem768": "...(1184 bytes)..." + } + }, + "account_keys": { /* discussed layer */ } + } + + Where: + +- `unique-client-identifier` is a unique, 32-byte value generated by the device during initial + setup. A 32-byte pure random (binary) value is suggested, but devices can choose something else + as long as it is unique per device and should not be reused if a user wipes the application data + from the device. This unique identifier should be used for uniqueness, not description: there is + a separate description field for device descriptive details. (If clients have a good value that is + not 32-byte to start with it is suggested to pad it or use a blake2b hash to shrink it the + required 32-byte value). + +- `type` indicates the session platform: "i" for iOS, "a" for Android, "d" for Desktop. Other + single-letter values are reserved, but longer (free-form) values are permitted for custom client + implementations. + +- `seq` is a monotonic numeric identifier that a device updates each time it changes its device + data. Other devices will ignore any update that is not larger than the most recent value they + know of. + +- `timestamp` is the unix timestamp when the device last updated its data. Its primary purpose is + to allow other devices a (very rough) indicator of when a device was last active. Note that + clients typically only update this when rotating keys, and so devices should only treat this as a + device being active sometime recently, and not a precise "last active" timestamp. + +- `version` indicates the client version encoded as an integer where application version M.m.p is + encoded as 1000000×M + 1000×m + p, e.g. the above represent application versions 2.25.0, 2.0.0, + and 3.0.7, respectively. "Extra" tags like "-alpha6" and so on are not supported or included. + +- `device_pubkey_x25519` and `_mlkem768` are short-term x25519 and MLKEM768 pubkeys, respectively, + used for encrypting device config messages. These keys must be randomly generated: they must + *not* be derivable from any existing account or device keys, and must not be disclosed outside the + device: someone with both keys will be able to encrypt messages for the device in question (until + keys rotate). + + Although clients can generate these in any way (i.e. the generation mechanism is opaque to anyone + outside the device itself) we use a single, random 32-byte value that is expanded into 32+64 byte + X25519 and ML-KEM seeds using SHAKE256("SessionDeviceKeys" || seed_32B), to align with other key + expansion (such as for accounts) used elsewhere in this protocol. + +- `description` is free-form text describing the device, capped at 64 bytes. This is a byte limit + rather than a character limit; truncation must fall on a UTF-8 character boundary. It is the only + field of a record not generated by libsession, and so the only one that would otherwise leave a + record unbounded in size. + +- Any other fields (such as `"unknown_extra_field"` in the above example) are preserved rather than + dropped, so that a device running an older version does not discard fields added by a newer one. + These are for future versions of libsession rather than for client data: a field added here is + republished by every device on the account, and must fit within the per-record budget below. + +A typical record size is: + 32 + 3 # device ID + encoding overhead + + 3 + 3 # type encoding + + 3 + [3-6] # seqno encoding. 6 would be for a 4-digit number, which would be sufficient for years of rotations + + 3 + 12 # timestamp encoding + + 3 + 3 + [~28] # Variable: example is for 28-byte "iPhone OS 28 - Alice's phone" + + 3 + 9 # typical 7-digit version encoding + + 3 + 3 + 32 # x25519 pk + + 3 + 5 + 1184 # mlkem768 + ============== + 1341 bytes (approximate; some field lengths are variable) + +A payload also carries the account key list, which is present regardless of the number of devices. +Its size is bounded by the retention and rotation periods: keys rotate every 12h and are retained +for 16 days, giving at most 33 entries of about 70 bytes each — an allowance of **2300 bytes**. + +Padding is therefore to a size of: + + 2300 + 6400×N + +where the 2300 covers the account key list and each 6400-byte bucket covers four devices at a +budget of 1500 bytes per device record, the remainder of the bucket being available for removal +tombstones (see "Removed devices"). A payload carries as many tombstones as fit without crossing +into the next bucket, the most recently removed being kept; devices arrive at the same set from the +same data. + +The budget is not enforced. Records or account keys exceeding their allowance push the payload into +the next bucket rather than being rejected. + +Note that the padded size is not what tells a reader how many devices an account has: the `keys` +list in the outer structure is unencrypted and already gives that count, and is what a storage +server reads to enforce a device limit (see "Extension - Pro subscriptions" below). + +### Removed devices + +A removed device is written into the devices dict as a tombstone: its ID maps to the unix timestamp +of the removal, where a live device maps to a sub-dict. The type of the value distinguishes the +two. + +A device holding an existing record for the removed device marks that record removed and retains the +rest of its fields; a device with no record for it ignores the entry. Since a record absent from a +message is left unchanged rather than removed, a removal must be stated in this way to propagate at +all. + +A tombstone and a live record for one device ID cannot coexist, so a device that is removed and +later rejoins the group must generate a new device ID. + +The removed device is not among the message recipients and cannot decrypt the payload; see +"Announcing removals" below. + +### Device data encryption + +When a client creates or makes any update to the above (which generally should only consist of +changes to its own "unique-client-id" sub-object) it then encodes and encrypts the config update +(using libsession's config diff + update mechanism) using a random `key_base` value for the +encryption. + +Note that this random `key_base` value is different from regular config messages: regular config +uses the account root key as the key_base so that other clients with the root key can derive the +encryption key. That does not work here, however, because this data is encrypted using relatively +short-lived, quantum resistant device keys deliberately not linked to the main key. Instead we +generate a random key and then encrypt that random key for each linked device. + +Before the encryption is actually performed, null byte padding is appended to the encoded value to +bring the plaintext to 2300 + 6400×N bytes (see "Inner device data" above), so that the message size +reveals only which bucket the payload falls in rather than its contents. A payload consisting of 1 +through 4 devices therefore encrypts to an identical size. On decryption the trailing null bytes +are stripped; the payload is a bt-encoded dict, which always ends in `e`, so trailing nulls are +unambiguously padding. + +The 32-byte symmetric encryption key, `key_base`, is itself separately encrypted for each device as +described below. This process involves generating a single ephemeral X25519 keypair and per-device +ML-KEM ciphertexts and including those on the outside of the message. + +This encrypted config message gets stuffed into another bt-encoded message at this point, encoded +as: + +```json + { + "A": "pubkey...", // ephemeral X25519 pubkey + "ciphertexts": "[ct123...][ct456...][ct789...][ctabc...]"], + "keys": "[abc123...][def456...][789aaa...][888bbb...]], + "kicked": "[k123...][k456...][k789...][kabc...]", + "payload": "...encrypted payload...", + "signature": "...above data signed with long-term account key...", + } +``` + +where: +- `payload` contains the encrypted payload the device needs, using xchacha20-poly1305 encryption + with the random base key, `key_base`. +- `A` is a single ephemeral X25519 key used for symmetric encryption keys +- `kicked` announces removals to devices that can no longer read the payload; see "Announcing + removals" below. +- `ciphertexts` is a packed binary value of N×4×1088 bytes where each 1088 byte segment contains an + ML-KEM768 ciphertext for one of the accounts devices. When the number of devices is not a + multiple of 4, the unused slots are filled with random data. +- `keys` are the encrypted values of `key_base`, encrypted for each device using xchacha20 (NB: this + stage of encryption does *not* include poly1305 authentication) with a unique device key + (described below). Each entry is 34 bytes: a 2-byte device key index followed by the 32-byte + encrypted key. Like `ciphertext`, this is always a multiple of 4 entries (i.e. 4×34 bytes), with + random noise used for unused slots. The keys here must be in the same order as `ciphertexts`: that is, the 3rd + 1088-byte slice of ciphertexts is the ciphertext for the device whose encrypted value is the 3rd + 32-byte slice of `keys`. + +Actual encryption is performed as follows: + +1. For each device 𝑖 ∈ {1, ..., 𝑁}, use its pubkey to generate an encapsulated MLKEM-768 secret for that device. + + sᵢ, cᵢ = one encapsulated secret + ciphertext for device i's MLKEM pubkey + +2. Pad the ciphertexts as needed with random values of the same length to bring it up to the next + multiple of 4 ciphertexts: + + cᵢ = random(1088), 𝑖 ∈ {𝑁+1, ..., 𝑁⌈𝑁/4⌉} + +3. Generated a shuffled ordering over the 𝑁⌈𝑁/4⌉ elements (and store it; it will be used again in a + later step), and build the final packed ciphertext value by concatenating all of the cᵢ values + together in shuffled order: + + ciphertexts = cⱼ || cₖ || cₗ | ... + +4. Generate an ephemeral X25519 keypair, a/A. + +5. Generate the deterministic nonce for the encrypted `payload` value: + + payload_nonce = BLAKE2b_24(ciphertexts, key=A, pers="SessionDevDNonce") + +6. Generate a secure random 32-byte device key for encrypting the padded payload data. + + key_base = random(32) + +7. Encrypt the payload data: + + enc_payload = XChaCha20Poly1305(plaintext_payload, key=key_base, nonce=payload_nonce) + +8. For each real device (i.e. not the dummy padding entries), calculate the base key encryption + nonce as: + + knonceᵢ = BLAKE2b_24(ciphertextᵢ || enc_payload, key=A, pers="SessionDevKNonce") + +9. Calculate the per-device symmetric encryption key for `key_base` for each device i as follows. + Given: + + a, A = X25519 keypair from step 4. + sᵢ, cᵢ = one encapsulated secret + ciphertext from step 2. + Bᵢ = device's current X25519 pubkey + Mᵢ = device's current MLKEM pubkey + + the key is encrypted using: + + kᵢ = BLAKE2b_32(aBᵢ || A || Bᵢ || sᵢ || Mᵢ, pers="SessionDevKeyKey") + keyᵢ = XChaCha20(message=key_base, key=kᵢ, nonce=knonceᵢ) + + This key is prefixed with a two-byte device key index, computed as: + + dkᵢ = BLAKE2b_2(A || Bᵢ || Mᵢ || cᵢ || keyᵢ, pers="SessionDevKeyIdx") + + This indicator is effectively a cheap checksum designed to short-circuit shared secret + calculation attempts without revealing anything identifying to outside observers: by computing a + much cheaper hash before attempting the full MLKEM decapsulation + X25519 + XChaCha20 + poly1305 + full payload decryption, it can instead short-circuit almost all encrypted values not meant for + it, while still revealing nothing to outside observers (because the device MLKEM and X pubkeys + are not known outside the device group itself). + + Note that this key value is *not* authenticated aside from this checksum (i.e. it is just + XChaCha20, not XChaCha20+poly1305): authentication happens as part of the final `devices` + ciphertext decryption, and so does not need to be included here: if the final decryption failed, + that indicates that the key decryption was not correct (or that the message was tampered with). + +10. Pad the key list as needed with random values of the same length to bring it up to the next + multiple of 4 ciphertexts: + + dkᵢ || keyᵢ = random(2+32), 𝑖 ∈ {𝑁+1, ..., 𝑁⌈𝑁/4⌉} + +11. All 4⌈𝑁/4⌉ keyᵢ values (i.e. real + dummy) are then packed in the same shuffled ordering used in + step 3 into a packed `keys` value: + + keys = dkⱼ || keyⱼ || dkₖ || keyₖ || dkₗ || keyₗ | ... + +The final message is then constructed as described above for upload to the swarm. Note that there +is no additional encryption: the outer structure here is intentionally visible. + +This message is then uploaded to the swarm. If the message contains more than 4 ciphertexts/keys +then the upload request must prove Pro status to the receiving swarm member (otherwise storage +server will reject it for being too large). + +(Note that the Pro restriction will likely not be imposed in the initial PFS release) + +### Announcing removals + +A removed device is not among the message recipients and so cannot decrypt the payload containing +its tombstone. It is not given a key for that message either: the payload contains the account key +list, including any key generated as part of the removal. + +Removals are instead announced in the `kicked` field of the outer, unencrypted structure. Each +entry is: + + kickᵢ = BLAKE2b_16(seed || deviceidᵢ, key=A, pers="SessionDevKicked") + +where `seed` is the account root seed and `A` is the message's ephemeral X25519 pubkey. + +A removed device computes this value for its own device ID to detect its removal. Devices still in +the group take removals from the tombstones in the payload and have no use for this field. Without +the root seed the entries are indistinguishable from random. + +The list is built from the tombstones the payload carries, recomputed for each message: the entries +cannot be recovered from a previous message, since a device ID cannot be derived from an entry and +the value depends on that message's `A`. + +`A` is included so that the entries differ from message to message. Computed without it an entry +would be constant for a given device ID, and the real entries could be identified as those appearing +in two consecutive messages. + +The list is padded to a multiple of 4 entries with random values and shuffled, as `ciphertexts` and +`keys` are, so that its length indicates only which bucket the removal count falls in. + +An adversary who obtains the root seed can compute these values, and so recover an account's removal +history from stored messages. + +## New device setup + +When setting up a new Session instance on a device (either after wiping and restoring, or on a new +device) the device must check for an existing linked device config in namespace 21. + +If no linked device config messages exist then this is a brand new account (or an account that +has not been used in some time), and so the device can simply construct a linked device config +with only itself as a member. (The actual stored data in this single device case is not +particularly useful, but is needed to allow other linked devices to properly link). + +If a linked device config exists, and its device it is able to successfully decrypt one of the +device keys, then it is *already* part of the linked device group and there is nothing extra needed +beyond uploading the new linked device config. + +Otherwise, the new device must request to join the linked device group, and that is what the rest of +this section details. + +### Initiating a device link + +A new "device link" message type is introduced, and will be stored in the account's (private) +message namespace 21. This message is constructed as follows: + + { + "id": "unique-client-identifier1", + "info": { + "type": "i", + "timestamp": 1777777777, + "seq": 1, + "description": "iPhone OS 28 - Alice's phone", + "version": 2025000, + "unknown_extra_field": { "x": 123, "y": [1,3,997,"x"] }, + "device_pubkey_x25519": "abcdef123...", + "device_pubkey_mlkem768": "...(1184 bytes)..." + } + } + +That is, it is simply the information to add to the linked device list plus some metadata. This +request is encrypted using the session account root key, and uploaded to namespace 21 with a TTL of +10 minutes. (Since device linking requires a user to have access to both devices at the same time, +a longer TTL accomplishes nothing). Note that the above is not signed explicitly: the recipient +already needs the account long-term root key to decrypt the content, and so an additional signature +by that same key would add nothing. + +#### Handshake short authentication string + +This initial key also implies a human-readable short authentication string that allows a user to +verify that the request being accepted on a linked device matches the one that originated on the new +device. This authentication string is an emoji sequence selected from a set of 64 distinct emoji +values as specified in the Matrix specification's short authentication string emoji list (see +https://spec.matrix.org/v1.17/client-server-api/#sas-method-emoji). Clients shall show the first 7 +characters of this emoji, with an optional "extended" view (either a toggle, or visual indicator) to +show an extended version of the SAS with 14 additional characters (21 total) that users can use for +extra assurance. + +The exact sequence is calculated from a list of 6-bit (0-63) integer values that determine the index +of the emoji value, generated as follows: + +- seed = Argon2id(M, salt=blake2b(M, size=16, pers="SessionLinkEmoji"), size=16, cost=16MiB, ops=2) + where M is the decrypted device link message data. +- emoji indices are then selected by interpreting the resulting 16 bytes as a 128-bit, little-endian + encoded integer where index 0 is the value of the least significant 6 bits, index 1 is bits 6-11, + and so on. +- the final secret displayed on both devices is then 7 characters joined with spaces between each + emoji. When displaying an extended version, devices are recommended to format as 3 lines of 7 + characters each. + +This short string simply serves as a quick visual representation of the device key. This prevents a +rogue device with the root Session key from being able to wait for and quickly replace a device +linking message with its own version with a replaced key to masquerade as a different device joining +a device group: such an attacker would immediately alert the user because of the different SAS key. + +We use a memory-hard Argon2id hash here to make collisions costly: if a device with compromised root +keys (but not device group keys) wanted to gain entry into the device group, it would have to notice +the linking request in the swarm and quickly replace it with its own linking request to fool the +user into accepting *it* rather than the intended device. Given that the 7 emoji sequence offers +only 42 bits of entropy, a cheap hash here (such as BLAKE2b) could conceivably be collided within +the short window before the user accepts the request; a memory-hard hash makes that infeasible. + +### Device link request handling + +Upon receiving a (valid) device link request in namespace 21, an existing (linked) device must +display to the user a screen with the new device details, asking for confirmation of the new linked +device. This information should generally consist of the device type, description, and version, +time the request was made, and the short authentication string. + +In some circumstances, additional information might also need to be confirmed or requested: + +- if the new device has the same device identifier as an existing device then the user should be + told that accepting this will replace the existing device in the device group. (This path is + relatively rare, but would apply, for instance, if someone restores their system from a backup + with expired keys that needs to re-join the device group with new keys.) + +- if there are no available additional linked device slots (i.e. because the user is not a Pro user, + and has used all available non-Pro device slots) then the user must be informed and given a list + of existing devices to kick out of the device group. The *current* device, if shown at all, + should not be selectable in this list. + +The user is then given a choice to accept or deny the linking request. + +#### Device link request denial + +If the user chooses to deny the request then the device should delete the linking request from +namespace, and take no other action. + +#### Device link request acceptance + +Upon accepting a device linking request, the existing linked device accepting the new device must: + +- update the linked device configuration with the new device details +- regenerate the new linked device encryption with the new details, and newly encrypted for the new + device. +- push the updated linked device config to the account's swarm. + +The device that requested linking, meanwhile, continues to monitor namespace 21 for an updated +device message that it is successfully able to decrypt. + +# Account keys + +Account public keys consist of a pair of public keys: an ML-KEM-768 pubkey, and a X25519 pubkey. We +effectively follow the fundamentals of the draft X-Wing construction, but with some small +modifications as needed for the specifics of Session message construction. + +Account public keys are stored in an account's namespace -21, which is a public "outbox" that only +the account owner can upload to, and anyone can fetch from. This mechanism is used by other clients +to retrieve the public keys necessary to contact a user. + +The message in this outbox consists of a bt-encoded dict containing: + +- `"M"` -- ML-KEM-768 pubkey (1184 bytes). +- `"X"` -- X25519 pubkey (32 bytes). +- `"~"` -- "positive alternative" Ed25519 signature over the previous values in the dict (64 bytes). + +Additional keys are ignored (they could be used by future versions to provide additional account key +data). + +Clients should upload the message with a maximum (currently 30-day) lifetime so that even if all of +a client's devices are offline for an extended period, the keys remain available for other clients +attempting to contact them. (Account keys would not rotate with such an extended outage of all +account devices, but this is preferred to falling back to the long-term key). + +Clients are expected to re-fetch this key outbox before sending messages if their cached copy is no +longer fresh. A cached entry passes through three states: + +- **fresh** -- less than 24h old. Usable directly; no fetch is made. +- **stale** -- between 24h and 48h old. Still usable, but a background re-fetch is started at the + same time, so a send is never blocked waiting on the network for a key we already have. +- **expired** -- more than 48h old. Not usable; a fetch must complete before a PFS message can be + sent. Until it does, the sender falls back as described in "Non-PFS fallback" below. + +A fetch that completes successfully but finds no valid keys published for the account records a +negative result ("NAK") rather than nothing at all, which suppresses further fetch attempts for that +account for 1h. Without this, every message to an account that has not published keys would incur a +fresh (and futile) network round trip. + +This signed key payload should be rotated periodically by any of the clients; how rotation works +specifically is discussed later. + +When validating the key payload signature, the client converts the recipient's long-term account key +(which is an X25519 pubkey) into the positive alternative Ed25519 of the two possible associated +Ed25519 pubkeys, and verifies the previous values in the message using that pubkey. + +(To elaborate on this positive alternative signature, which used in some existing Session code for +signature verification: because the recipient only knows the session ID -- which is an X25519 +pubkey, converted from the true Ed25519 pubkey of the client -- the recipient can recover the +underlying Ed25519 pubkey *except for its sign*. The tweaked signature mechanism accounts for this +if the actual pubkey is negative by using the negative value of the Ed25519 private scalar to +compose the signature, which results in an Ed25519 signature verifiable with the positive +alternative of the two possible pubkeys that result from converting the session ID X25519 pubkey +back to an Ed25519 pubkey. This "assume positive" approach is the same as that used in Signal's +XEd25519, but unlike XEd25519's random nonce use, retains Ed25519's use of the seed as keying +material, thus maintaining Ed25519's fully deterministic signature generation.) + +## One-to-one Message Encryption + +Encryption keys used to encrypt DMs, given a set of published ML-KEM-768 and X25519 pubkeys, is +constructed as follows: + +0. If the cached keys for the account are no longer fresh (see "Account keys" above), keys for the + contact are refreshed. Clients should generally initiate this preemptively, such as when opening a + conversation with stale keys, to avoid the extra fetch latency when actually sending a message. + + This provides the recipient's current account pubkeys, M (ML-KEM-768) and X (X25519). + +1. Generate an ephemeral X25519 keypair: e/E. + +2. Using the recipients *long term pubkey* S (i.e. Session ID without the 05 prefix) compute a "key + indicator shared secret": + + kiss = BLAKE2b(E || S, key=eS, personalization="Session-Msg-KISS", length=2) + + (Do not be alarmed at the long term key (S) usage here: this is only used for a tiny bit of + metadata obfuscation in the outer encoding, but *not* for the actual message encryption). + +3. Compute an encrypted key indicator by taking the first two bytes of the ML-KEM-768 pubkey and + XORing these with the two bytes of the `kiss`: + + ki = M[0:2] ⊕ kiss + + (This encrypted value is an important optimization that allows the recipient to identify *which* + of its current and recent keys was used -- without it, it would have to trial decrypt using many + different keys, which would be annoyingly expensive, particularly when processing many incoming + messages when coming online after an extended offline period. While this key indicator could be + provided plaintext as plaintext without compromising message security, doing so would leak some + metadata about the message sender: because not all senders refresh the remote's pubkeys at the + same time, repeated messages from "early" refreshes and from "late" refreshes would be + correlatable through their selected index. By using basic encryption with the long term key for + this index, to anyone without the recipient's long term key, no correlation can be drawn among + senders based on key refresh times). + + Note that we do *not* use a MAC or AEAD here, and thus do not detect tampering with this key + index. This is deliberate: any tampering with the key *will* still result in failure because it + will either indicate a key that does not exist (failing decryption), or it will point to the + wrong key, which will also fail decryption (because of the encrypted message's AEAD). + +4. Generate an ML-KEM-768 encapsulated shared secret and ciphertext: + + ssₘ, ciphertext = Encapsulate(M) + +5. Generate an X25519 shared secret from the ephemeral and recipient account keys (*not* the + long-term key): + + ssₓ = eX + +6. The message encryption key is constructed by using "X-Wing" key derivation mechanism (which is + basically just a hashed combination of X25519 and MLKEM shared secrets) to generate the X-Wing + shared secret: + + ss = SHA3-256(ssₘ || ssₓ || E || X || '\.//^\') + + (Note that the cryptography here is exactly that of X-Wing, however it must calculated separately + rather than use a single library X-Wing calculation because we also use the ephemeral X25519 + private key `e` back in step 2 for the key indicator; in normal X-Wing implementations this key + is not exposed). + + This is then fed into SHAKE256("SessionV2MessageSS" || ss) and "squeezed" to produce a 32-byte + key, `k`, and 24-byte nonce, `n`, which are used below to encrypt the message body. + +7. The overall inner plaintext message is constructed using a bt-encoded dictionary containing: + + - `"S"` -- the sender's Ed25519 pubkey (32 bytes). *Not* the session ID, but the session ID can + be easily derived from it. + - `"c"` -- the message content. This is the encoded `Content` protobuf data + - `"~"` -- Ed25519 signature, verifiable with S; this signs the value: + BLAKE2b(..., size=64, key=recipient_sessionid_33B, pers="SessionV2Message") + where "..." is the entire bt-encoded content (which may contain other, unknown keys) up to but + not including the `"~"` key. (This is oxenc's standard append_signature/consume_signature API). + - `"~P"` -- optional Session Pro Ed25519 signature. If this message uses Session Pro features + then this is the signature verifying that the sender is a valid Session Pro account. (The + public key itself is contained within the protobuf, and so when parsing, this value is merely + extracted but verification is deferred). Like ~, this signature is over everything before it + in the encoded dict (include the ~ signature). + + This value is tail-padded with null (0x00) bytes such that: + - the post-encryption size (see step 9) is a multiple of 256 bytes. + - the padded message size is always at least 256 bytes. (When combined with the previous rule, + this means the minimum padded message size is 396 bytes, with the current encryption data + adding 1140 bytes). + +8. The value is encrypted using XChaCha20+poly1305 encryption using key `k` and nonce `n` from step + 6. + +9. The encrypted value then written as a concatenation of: + + - `0x00 0x02` -- two bytes that identify the message as a Session v2 message. The 0x00 in + particular is needed to unambiguously identify it as *not* a protobuf message (Session v1 + messages are protobuf-encoded on top of the encryption layer). Values other than 0x02 and 0x01 + (used for non-PFS fallback, discussed below) are reserved for future use. + - `ki` -- two byte encrypted key indicator from step 3 + - E -- 32-byte ephemeral X25519 pubkey + - ciphertext -- 1088-byte ML-KEM-768 ciphertext + - encrypted value from step 8. + + and so, with padding applied, all messages should have sizes of the sum of: + - 2 -- message prefix + - 2 -- key indicator + - 32 -- ephemeral X25519 pubkey + - 1088 -- MLKEM ciphertext + - 16-byte -- poly1305 MAC, inside the encrypted payload. + - 396 + 256×N, N >= 0 -- padded message size + + that is, 1536 + 256×N (e.g. 1536 minimum, but expandable in increments of 256 bytes). + +## Account Key Rotation + +Clients rotate the account keys, shared by all devices, after 12h+ε. ε here is a random delay +between -1h and 1h that is calculated differently for each device, to reduce the possibility of +collisions and to obscure the exact device timings of key publishing. Details of ε selection are +discussed below. + +An account key itself is a single 32-byte seed value that is expanded into 96 bytes using +SHAKE256("SessionAccountKeys" || seed) to produce: + + - 32B X25519 private key + - 64B MLKEM-768 seed + +To perform a rotation, the device generates a new secure random 32-byte seed and updates the existing +seed (if any) to be marked as rotated away at the current timestamp. Any old keys that have been +rotated away for more than 16 days are discarded. + +The list of all account seeds is then added to the "account_keys" section of the device group +payload, including the creation and rotation timestamps (to synchronize with other devices). + +This updated device group payload is then re-encrypted for current devices and pushed to the swarm. +Simultaneously, the rotating client also pushes a device account public key to namespace -21, as +described above. + +### Account Key Rotation timer + +In order to determine when a device should initiate key rotation on behalf of the device group, a +random delay is deterministically calculated by each device based on the current active account +seed and when the current seed became active: + + t = t_prev + 12h + ε + ε = -1h + 2h × (1 - u^N) + +where t_prev is the unix timestamp when the current seed was created, and t is the target timestamp +for the current device. `u` here is a device-specific quasi-random value distributed as Unif[0, 1], +and N is the number of account devices. + +`u` itself can be computed in any way so long as it changes with the current seed and device id, but +we suggest the following (and this is used in libsession-util): + +- the seed is computed as BLAKE2b(deviceid, key=current_seed, pers="SessionAccKeyRot", size=8) +- that seed is interpreted as a little-endian, 64-bit unsigned integer +- that integer is then converted to double and divided by 0x1p64 (that is: the double value 2^64) +- the resulting value is the `u` value for the calculation of ε + +This then provides a consistent `t` value for any given device for any current seed value. That +value is a unix timestamp: the device should rotate when the timestamp is reached (or exceeded). + +This construction is specifically designed to statistically mask the number of devices in an +account: with N active devices, the distribution of the *smallest* ε value (which is when the +effective account key rotation occurs, assuming all devices are online) over the N devices works out +to `Unif[11h, 13h]`, which is exactly the same as the uniform distribution of a single active +device. + +This distribution is also designed to break up the rotation times to reduce the occurences of +conflicting updates (which then have to be resolved) versus having all devices try rotating at the +same time. + +Note that Storage Server public outbox namespaces can only hold one message at a time: uploading a +new message replaces an existing one (if present), and so clients will only see one at a time. It +*is*, however, possible for two devices in a group to race to upload a new key, and so any such +rotation attempt must preserve *both* keys to account for the brief window where some client may +have fetched the quickly replaced key. + +Account keys in general must be preserved (in the device group config) for at least 15 days (the +1-to-1 message max TTL of 14 days plus an allowance of up to 24h since the sender last re-fetched +account pubkeys) *after* they have been rotated away so that an offline device is able to encrypt +any incoming messages that may still be in the swarm. We use 16 days to include an extra 24h safety +buffer. + +### Unimplemented accelerated rotation + +One idea that is possible but not currently implemented is to allow earlier rotation when lots of +messages are being received. One possible downside would be the metadata leakage: an outside +observing watching the public account keys for an account could infer that the account is receiving +lots of messages because of a higher rotation frequency. + + +# Non-PFS fallback (post transition) + +When a client seeks to contact another Session account that does *not* have PFS keys available (for +example, because the client has been offline for too long, or because the client has not published +its PFS+PQ keys, or because the current client failed to parse the published PFS+PQ pubkey data, the +client sends a message using the receiver's long term key. + +During the transition period, these messages are backwards compatible "v1" direct messages (see +the Transition Period section below) as the client may be communicating with a client that has not +yet upgraded to support the Session v2 PFS+PQ key encryption. + +Once the transition period is over, clients start sending using a new v2-like message format, +so as to shed the legacy encoding of the current v1 message format. + +Such a message is encoded identically to be externally indistinguishable from a PFS+PQ v2 message, +by using random values for `ki` and `ciphertext`. + +The actual encryption keys used are entirely different (since there is no actual PFS or PQ keys to +be used): for a recipient Session ID `S`, where `R` is the X25519 pubkey of that session ID (i.e. +the key with the 05 stripped off), the encryption keys are generated using a random ephemeral X25519 +keypair (e, E) as: + + ssᵣ = eR // = rE, when computed by the recipient + ss = SHA3-256(ssᵣ || R || E || 'SessionV2NonPFS') + x = SHAKE256("SessionV2NonPFSSS" || ss) + k, n = 32 and 24-byte "squeezes" of x + +(The key construction here deliberately mirrors the encryption used for PFS+PQ messages, but adapted +for use with X25519 keys using only the long-term keypair). + +The `k` and `n` here are the key and nonce used for the actual encrypted messages, and uses the same +xchacha20+poly1305 encryption as is used for PFS+PQ messages. + +The plaintext inner payload is identical to a PFS+PQ message bt-encoded with sender Ed25519 pubkey, +content, signature, and optional Pro signature. The plaintext payload is also null byte padded with +the same padding rules as PFS+PQ messages (but without needing to include PQ ciphertext, the minimum +size of such a message is 256 bytes instead of 1536 bytes). + +As with PFS+PQ messages, the content (excluding added padding) is the encoded Protobuf message. + +When clients attempt decryption, they first attempt the normal PFS+PQ decyption using the +(supposedly) encrypted `ki` value. (This is quite unlikely to actually match any current PFS+PQ +keys, but even if it does match, PFS+PQ decryption will fail.) + +Upon failure to decrypt using normal PFS+PQ, the client attempts a fallback, non-PFS decryption +using the key/nonce described above. + +If this decryption *succeeds* the client should process the message normally, but clients should +provide some indication to the user reading the message that the message decryption fell back to +weakened security. + +## Pre-PFS compatibility grace period + +Within at least the first 6 months of PFS+PQ being available on all Session clients, Non-PFS +messages do *not* follow the above non-PFS encryption mechanism, but rather use the current "v1" +Session message protocol (which involves multiple layers of independent protobuf encoding around the +Content). + +# Migration + +Clients that support PFS+PQ and successfully find a PFS+PQ pubkey record for a session account being +messaged should automatically opt-in to using PFS+PQ encryption. We use the fact that a published +pubkey message exists as an indication that the client's devices supports the new encryption and +message format. + +As a result, clients *must not* publish PFS+PQ records until all of their devices are upgraded to +support PFS. This, unfortunately, is currently made complicated by the fact that multi-device +Session accounts are not fully aware of how many devices are using the account (= one of the goals +of this redesign), and so this may require manual user intervention. + +Details TBD. + + +# Push notifications + +Android and iOS require obtaining the new keys for push notifications to work: new pushed messages +might use a new key that the device has not yet learned about. However the encrypted account +devices message is much too large to fit into push notifications (and on top of that, neither +Firebase nor APNS guarantee delivery of push notifications to the device). + +To work around this, we will require updating Session-ios and -android with code that attempts +decryption using current known keys and, upon failure, initiates an onion routed request to fetch +new account device data messages from the account's swarm. + +Both platforms provide approximately 30s to process the request, and both also appear to require +that such a high-priority notification actually produce a notification within that 30s, and so the +design here will: + +- identify whether a v1 or a v2 message. If v1, carry out existing decryption. +- for v2: + - attempt decryption; if it succeeds with existing recent keys, display the notification (or on + iOS, mutate the notification to show the decrypted content). + - if it fails, make a connection to the account swarm to fetch new device account messages, + process those messages (to learn new keys) and then attempt decryption again. + - if it succeeds, show the notification content. + - if it still fails, show an error message ("Could not decrypt incoming message") as the + notification. + - (optional) if a recent decryption failed with the same indicated decryption key, cool down for a + while before going back to the account device key. + +# Extension - Pro subscriptions for >4 linked devices on an account + +The spec above allows for limits on the number of linked devices for a single account. This is +intended to be used to allow more accounts for Pro users by having storage server deny linked device +config storage containing more than 4 keys. + +This will require modifying storage server to: +- understand and handling Pro proofs for validating the pro status of a requestor +- parsing the outermost linked device configs to determine the number of keys in the message, and + denying storage for linked device configs with more than 4 keys unless accompanied with a Pro + proof. diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 305c3749b..305b091d2 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,131 +1,67 @@ -option(SUBMODULE_CHECK "Enables checking that vendored library submodules are up to date" ON) - -if(SUBMODULE_CHECK) - find_package(Git) - if(GIT_FOUND) - function(check_submodule relative_path) - execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE localHead) - execute_process(COMMAND git rev-parse "HEAD:external/${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) - string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate) - if (upToDate) - message(STATUS "Submodule 'external/${relative_path}' is up-to-date") - else() - message(FATAL_ERROR "Submodule 'external/${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --recursive\nor run cmake with -DSUBMODULE_CHECK=OFF") - endif() - - # Extra arguments check nested submodules - foreach(submod ${ARGN}) - execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path}/${submod} OUTPUT_VARIABLE localHead) - execute_process(COMMAND git rev-parse "HEAD:${submod}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE checkedHead) - string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate) - if (NOT upToDate) - message(FATAL_ERROR "Nested submodule '${relative_path}/${submod}' is not up-to-date. Please update with\ngit submodule update --init --recursive\nor run cmake with -DSUBMODULE_CHECK=OFF") - endif() - endforeach() - endfunction () - - message(STATUS "Checking submodules") - check_submodule(ios-cmake) - check_submodule(libsodium-internal) - check_submodule(zstd) - check_submodule(protobuf) - check_submodule(session-router) - check_submodule(date) - endif() -endif() +include(../cmake/session-deps/Deps.cmake) + +message(STATUS "Checking submodules") +check_submodule(ios-cmake) +check_submodule(session-router + external/oxen-libquic + external/oxen-libquic/external/oxen-logging + external/oxen-libquic/external/oxen-logging/fmt + external/oxen-libquic/external/oxen-logging/spdlog + external/nlohmann) +check_submodule(protobuf) +check_submodule(date) +check_submodule(session-sqlite SQLiteCpp cmake/session-deps) + +# 1.2.2+ required for logging sink removal support +set(OXEN_LOGGING_MIN_VERSION 1.2.2 CACHE STRING "") if(NOT BUILD_STATIC_DEPS AND NOT FORCE_ALL_SUBMODULES) find_package(PkgConfig REQUIRED) endif() -macro(libsession_system_or_submodule BIGNAME smallname target pkgconf subdir) - if(NOT TARGET ${target}) - option(FORCE_${BIGNAME}_SUBMODULE "force using ${smallname} submodule" OFF) - if(NOT BUILD_STATIC_DEPS AND NOT FORCE_${BIGNAME}_SUBMODULE AND NOT FORCE_ALL_SUBMODULES) - pkg_check_modules(${BIGNAME} ${pkgconf} IMPORTED_TARGET GLOBAL) - endif() - if(${BIGNAME}_FOUND) - add_library(${smallname} INTERFACE IMPORTED GLOBAL) - if(NOT TARGET PkgConfig::${BIGNAME} AND CMAKE_VERSION VERSION_LESS "3.21") - # Work around cmake bug 22180 (PkgConfig::THING not set if no flags needed) - else() - target_link_libraries(${smallname} INTERFACE PkgConfig::${BIGNAME}) - endif() - message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}") - else() - message(STATUS "using ${smallname} submodule ${subdir}") - add_subdirectory(${subdir}) - endif() - if(NOT TARGET ${target}) - add_library(${target} ALIAS ${smallname}) - endif() - if(BUILD_STATIC_DEPS AND STATIC_BUNDLE) - libsession_static_bundle(${smallname}::${smallname}) - endif() - endif() -endmacro() - - -set(deps_cc "${CMAKE_C_COMPILER}") -set(cross_host "") -set(cross_rc "") -if(CMAKE_CROSSCOMPILING) - if(APPLE_TARGET_TRIPLE) - set(cross_host "--host=${APPLE_TARGET_TRIPLE}") - elseif(ANDROID) - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) - set(cross_host "--host=x86_64-linux-android") - set(android_compiler_prefix x86_64) - set(android_compiler_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(cross_host "--host=i686-linux-android") - set(android_compiler_prefix i686) - set(android_compiler_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(cross_host "--host=armv7a-linux-androideabi") - set(android_compiler_prefix armv7a) - set(android_compiler_suffix linux-androideabi) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) - set(cross_host "--host=aarch64-linux-android") - set(android_compiler_prefix aarch64) - set(android_compiler_suffix linux-android) - else() - message(FATAL_ERROR "unknown android arch: ${CMAKE_ANDROID_ARCH_ABI}") - endif() - - string(REPLACE "android-" "" android_platform_num "${ANDROID_PLATFORM}") - set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}${android_platform_num}-clang") - else() - set(cross_host "--host=${ARCH_TRIPLET}") - if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) - set(cross_rc "WINDRES=${CMAKE_RC_COMPILER}") - endif() - endif() +function(add_static_subdirectory dir) + set(BUILD_SHARED_LIBS OFF) + add_subdirectory(${dir} ${ARGN}) +endfunction() + +session_dep(libsodium 1.0.21) +libsession_static_bundle(sessiondep::libsodium) + +if(NOT TARGET oxenc::oxenc) + set(OXENC_BUILD_TESTS OFF CACHE BOOL "") + set(OXENC_BUILD_DOCS OFF CACHE BOOL "") + sessiondep_or_submodule(liboxenc 1.6.0 session-router/external/oxen-libquic/external/oxen-encoding oxenc::oxenc) endif() -if(ENABLE_NETWORKING) - set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "") - libsession_system_or_submodule(OXENQUIC quic oxen::quic liboxenquic>=1.8 session-router/external/oxen-libquic) +if(ENABLE_NETWORKING_SROUTER) + set(SROUTER_FULL OFF CACHE BOOL "") + set(SROUTER_DAEMON OFF CACHE BOOL "") + set(SROUTER_NATIVE_BUILD OFF CACHE BOOL "") + set(SROUTER_JEMALLOC OFF CACHE BOOL "") + + add_static_subdirectory(session-router EXCLUDE_FROM_ALL) + libsession_static_bundle(session-router::core) endif() -libsession_system_or_submodule(OXENC oxenc oxenc::oxenc liboxenc>=1.5.0 session-router/external/oxen-libquic/external/oxen-encoding) +# The network library always needs oxen::quic. When session-router is built (SROUTER, above) it +# provides oxen::quic transitively; otherwise pull in oxen-libquic on its own. +if(NOT TARGET oxen::quic) + set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "") + sessiondep_or_submodule(liboxenquic 1.8.0 session-router/external/oxen-libquic oxen::quic) +endif() if(NOT TARGET oxen::logging) - # Find this one using an intermediate target alias (oxenlogging::oxenlogging) rather than the - # final oxen::logging alias target that we actually use: that way either we go into using it as - # a submodule (in which case the submodule sets up oxen::logging), or else we find via system - # lib, in which case we need an extra intermediate interface library that also brings in fmt and - # spdlog. - libsession_system_or_submodule(OXENLOGGING oxen-logging oxenlogging::oxenlogging liboxen-logging>=1.2.0 session-router/external/oxen-libquic/external/oxen-logging) - if(NOT TARGET oxen::logging) - # If we load oxen-logging via system lib then we won't necessarily have fmt/spdlog targets, - # but this script will give us them: - include(session-router/external/oxen-libquic/external/oxen-logging/cmake/load_fmt_spdlog.cmake) - - add_library(oxen-logging-fmt-spdlog INTERFACE) - target_link_libraries(oxen-logging-fmt-spdlog INTERFACE oxenlogging::oxenlogging ${OXEN_LOGGING_FMT_TARGET} ${OXEN_LOGGING_SPDLOG_TARGET}) - add_library(oxen::logging ALIAS oxen-logging-fmt-spdlog) + # oxen-logging can't go through sessiondep_or_submodule: its public headers expose fmt types and + # a system liboxen-logging's .pc doesn't pull in fmt/spdlog, so consumers must add them + # explicitly. oxen-logging ships a cmake/load.cmake that resolves oxen::logging for us -- either + # a system liboxen-logging bundled with suitable fmt/spdlog targets, or the submodule (which + # builds its own fmt/spdlog). This is the same loader oxen-libquic uses. We just force the + # submodule path when our other deps would also prefer submodules over system libs. + option(DEPS_FORCE_liboxen-logging_SUBMODULE "force using liboxen-logging submodule" OFF) + if(BUILD_STATIC_DEPS OR DEPS_FORCE_SUBMODULE OR DEPS_FORCE_liboxen-logging_SUBMODULE) + set(OXEN_LOGGING_FORCE_SUBMODULES ON) endif() + include(session-router/external/oxen-libquic/external/oxen-logging/cmake/load.cmake) endif() @@ -148,15 +84,6 @@ if(APPLE) endforeach() endif() -function(add_static_subdirectory dir) - set(BUILD_SHARED_LIBS OFF) - add_subdirectory(${dir} ${ARGN}) -endfunction() - -add_static_subdirectory(libsodium-internal) -libsession_static_bundle(libsodium::sodium-internal) - - set(protobuf_VERBOSE ON CACHE BOOL "" FORCE) set(protobuf_INSTALL ON CACHE BOOL "" FORCE) set(protobuf_WITH_ZLIB OFF CACHE BOOL "" FORCE) @@ -167,60 +94,52 @@ set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) set(protobuf_ABSL_PROVIDER "module" CACHE STRING "" FORCE) set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "") set(protobuf_BUILD_PROTOBUF_BINARIES ON CACHE BOOL "" FORCE) -libsession_system_or_submodule(PROTOBUF_LITE protobuf_lite protobuf::libprotobuf-lite protobuf-lite>=3.21 protobuf) -if(TARGET PkgConfig::PROTOBUF_LITE AND NOT TARGET protobuf::libprotobuf-lite) - add_library(protobuf::libprotobuf-lite ALIAS PkgConfig::PROTOBUF_LITE) -endif() +sessiondep_or_submodule(protobuf-lite 3.21 protobuf protobuf::libprotobuf-lite) -set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "") -set(ZSTD_BUILD_TESTS OFF CACHE BOOL "") -set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "") -set(ZSTD_BUILD_SHARED OFF CACHE BOOL "") -set(ZSTD_BUILD_STATIC ON CACHE BOOL "") -set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "") -add_subdirectory(zstd/build/cmake EXCLUDE_FROM_ALL) -# zstd's cmake doesn't properly set up include paths on its targets, so we have to wrap it in an -# interface target that does: -add_library(libzstd_static_fixed_includes INTERFACE) -target_include_directories(libzstd_static_fixed_includes INTERFACE zstd/lib zstd/lib/common) -target_link_libraries(libzstd_static_fixed_includes INTERFACE libzstd_static) -add_library(libzstd::static ALIAS libzstd_static_fixed_includes) -export( - TARGETS libzstd_static_fixed_includes - NAMESPACE libsession:: - FILE libsessionZstd.cmake -) -libsession_static_bundle(libzstd_static) - - -set(JSON_BuildTests OFF CACHE INTERNAL "") -set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use -libsession_system_or_submodule(NLOHMANN nlohmann_json nlohmann_json::nlohmann_json nlohmann_json>=3.7.0 session-router/external/nlohmann) - -if(ENABLE_NETWORKING AND ENABLE_NETWORKING_SROUTER) - set(SROUTER_FULL OFF CACHE BOOL "") - set(SROUTER_DAEMON OFF CACHE BOOL "") - set(SROUTER_NATIVE_BUILD OFF CACHE BOOL "") - set(SROUTER_JEMALLOC OFF CACHE BOOL "") +# Force a static libzstd: we want semi-stable compressed output, which means we want all session +# clients to use the same version (where that is guaranteed) as much as possible, so that duplicate +# configs get deduplicated at the swarm. +set(BUILD_STATIC_libzstd ON CACHE BOOL "" FORCE) +session_dep(libzstd 1.5) +libsession_static_bundle(sessiondep::libzstd) - add_library(sodium INTERFACE) - target_link_libraries(sodium INTERFACE libsodium::sodium-internal) - add_static_subdirectory(session-router EXCLUDE_FROM_ALL) - libsession_static_bundle(session-router::libsessionrouter) +# TODO FIXME: integrate this with the updated networking PR, which will end up right about here when +# merging. This can basically just get deleted once we are always building session-router. +if(NOT TARGET mlkem_native::mlkem768) + file(GLOB_RECURSE mlkem_sources + session-router/external/mlkem-native/mlkem/src/*.c + session-router/external/mlkem-native/mlkem/src/*.S) + + add_library(mlkem_native768 STATIC ${mlkem_sources}) + + target_compile_definitions(mlkem_native768 PUBLIC + MLK_CONFIG_NO_RANDOMIZED_API + MLK_CONFIG_PARAMETER_SET=768 + MLK_CONFIG_NAMESPACE_PREFIX=sr_mlkem768 + MLK_CONFIG_NO_SUPERCOP + MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + ) + + target_include_directories(mlkem_native768 PUBLIC session-router/external/mlkem-native/mlkem) + + add_library(mlkem_native::mlkem768 ALIAS mlkem_native768) endif() -set(JSON_BuildTests OFF CACHE INTERNAL "") -set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use +if(NOT TARGET nlohmann_json::nlohmann_json) + set(JSON_BuildTests OFF CACHE INTERNAL "") + set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use + sessiondep_or_submodule(nlohmann_json 3.7.0 nlohmann-json nlohmann_json::nlohmann_json) +endif() + + +session_dep(simdutf 7) +libsession_static_bundle(sessiondep::simdutf) + + +add_subdirectory(session-sqlite) -function(simdutf_subdir) - set(SIMDUTF_TESTS OFF CACHE BOOL "") - set(SIMDUTF_TOOLS OFF CACHE BOOL "") - set(BUILD_SHARED_LIBS OFF) - add_subdirectory(simdutf) -endfunction() -simdutf_subdir() -libsession_static_bundle(simdutf) # We need Howard Hinnant's header-only date library for now because the STL implementation of # std::chrono::parse() is spotty or broken on: diff --git a/external/libsodium-internal b/external/libsodium-internal deleted file mode 160000 index e12e612dc..000000000 --- a/external/libsodium-internal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e12e612dc735909a59be552f9bf03fe98e320703 diff --git a/external/session-router b/external/session-router index d2def4c91..654089444 160000 --- a/external/session-router +++ b/external/session-router @@ -1 +1 @@ -Subproject commit d2def4c91d024f9d896b43e817520f27a41a7995 +Subproject commit 654089444dd007a1d904d2601c6b12ab2582e3d5 diff --git a/external/session-sqlite b/external/session-sqlite new file mode 160000 index 000000000..57f308f3a --- /dev/null +++ b/external/session-sqlite @@ -0,0 +1 @@ +Subproject commit 57f308f3a46c1bfceb21ee9ec77fb287c6fdf122 diff --git a/external/simdutf b/external/simdutf deleted file mode 160000 index 7b3f5afca..000000000 --- a/external/simdutf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7b3f5afcae322391a03736809ef6eea0c2934388 diff --git a/external/zstd b/external/zstd deleted file mode 160000 index e47e674cd..000000000 --- a/external/zstd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e47e674cd09583ff0503f0f6defd6d23d8b718d3 diff --git a/include/session/attachments.hpp b/include/session/attachments.hpp index ba03e92e7..c04886d31 100644 --- a/include/session/attachments.hpp +++ b/include/session/attachments.hpp @@ -110,7 +110,7 @@ std::optional decrypted_max_size(size_t encrypted_size); /// /// - `data` -- the buffer of data to encrypt. /// -/// - `domain` -- domain separator; uploads of funamentally different types should use a different +/// - `domain` -- domain separator; uploads of fundamentally different types should use a different /// value, so that an identical upload used for different purposes will have unrelated key/nonce /// values. /// @@ -125,7 +125,7 @@ std::optional decrypted_max_size(size_t encrypted_size); /// Throws std::invalid_argument if `seed` is shorter than 32 bytes, or if data is larger than /// MAX_REGULAR_SIZE (unless `allow_large` is true). /// -std::pair, std::array> encrypt( +std::pair, cleared_b32> encrypt( std::span seed, std::span data, Domain domain, @@ -150,7 +150,7 @@ std::pair, std::array> encry /// /// Throws std::invalid_argument if `seed` is shorter than 32 bytes, or if data is larger than /// MAX_REGULAR_SIZE (unless `allow_large` is true). -std::array encrypt( +cleared_b32 encrypt( std::span seed, std::span data, Domain domain, @@ -173,7 +173,7 @@ std::array encrypt( /// /// Throws std::invalid_argument if `seed` is shorter than 32 bytes, or if the file is larger than /// MAX_REGULAR_SIZE. -std::pair, std::array> encrypt( +std::pair, cleared_b32> encrypt( std::span seed, const std::filesystem::path& file, Domain domain, @@ -197,7 +197,7 @@ std::pair, std::array> encry /// Throws std::invalid_argument if `seed` is shorter than 32 bytes, or if the file is larger than /// MAX_REGULAR_SIZE. /// Throws std::runtime_error if the file size changes between first and second passes. -std::array encrypt( +cleared_b32 encrypt( std::span seed, const std::filesystem::path& file, Domain domain, @@ -220,7 +220,7 @@ std::array encrypt( /// Throws std::invalid_argument if `seed` is shorter than 32 bytes, or if data is larger than /// MAX_REGULAR_SIZE (unless `allow_large` is given). Throws on I/O error. If decryption fails /// then any partially written output file will be removed. -std::array encrypt( +cleared_b32 encrypt( std::span seed, std::span data, Domain domain, @@ -264,6 +264,89 @@ size_t decrypt( std::span key, std::span out); +/// Sizes of the legacy attachment encryption scheme's pieces: a 32-byte AES-256 key followed by a +/// 32-byte HMAC-SHA256 key, a 16-byte CBC IV, and a full-length (untruncated) HMAC. +constexpr size_t LEGACY_KEY_SIZE = 64; +constexpr size_t LEGACY_IV_SIZE = 16; +constexpr size_t LEGACY_MAC_SIZE = 32; +constexpr size_t LEGACY_DIGEST_SIZE = 32; + +/// The largest encrypted attachment the file server will store, and so the most any legacy +/// attachment can be: unlike the stream scheme, legacy decryption has to hold the whole ciphertext +/// at once, because the MAC and digest cover all of it and must be checked before any of it is +/// decrypted. Anything larger has to use the stream scheme, which decrypts incrementally. +constexpr size_t LEGACY_MAX_ENCRYPTED_SIZE = 10223616; + +/// API: crypto/attachment::legacy_decrypt +/// +/// Decrypts an attachment encrypted with the scheme Session used before the stream one: AES-256-CBC +/// under a random key, authenticated by an HMAC over the IV and ciphertext, and again by a SHA-256 +/// digest carried separately in the AttachmentPointer. Every Session client still sends these, so +/// this is the path most received attachments take. +/// +/// The layout is `IV || AES-256-CBC(PKCS#7) || HMAC-SHA256(IV || ciphertext)`, with `digest` the +/// SHA-256 of all three. Both are checked, in constant time, before anything is decrypted. +/// +/// Inputs: +/// - `encrypted` -- the downloaded file, entire. At most LEGACY_MAX_ENCRYPTED_SIZE. +/// - `key` -- the 64-byte key from the pointer: AES key then HMAC key. +/// - `digest` -- the 32-byte digest from the pointer. +/// - `unpadded_size` -- the pointer's `size`, i.e. the sender's claim about how long the file is +/// before the zero padding that hides its true length. Zero means the sender did not say, which +/// only clients predating the field do, and leaves the padding in place; any other value must be +/// no larger than what was decrypted, or the pointer is lying and this throws. +/// +/// Outputs: +/// - std::vector of decrypted, de-padded data. +/// +/// Throws std::runtime_error if the input is too large or malformed, if either authenticator fails, +/// or if `unpadded_size` does not describe the decrypted data. +std::vector legacy_decrypt( + std::span encrypted, + std::span key, + std::span digest, + size_t unpadded_size); + +/// Sizes of the legacy *display picture* scheme, which is not the legacy attachment one above: +/// AES-256-GCM under a 32-byte key, with a 12-byte nonce and the 16-byte tag both carried in the +/// data. +constexpr size_t LEGACY_DISPLAY_PIC_KEY_SIZE = 32; +constexpr size_t LEGACY_DISPLAY_PIC_NONCE_SIZE = 12; +constexpr size_t LEGACY_DISPLAY_PIC_TAG_SIZE = 16; + +/// API: crypto/attachment::legacy_display_pic_decrypt +/// +/// Decrypts a display picture — a profile or group avatar — encrypted with the scheme Session used +/// before the stream one. +/// +/// "Display picture" rather than "profile picture" because it is both: the other clients apply this +/// to a group's avatar as well as a person's. It has nothing to do with picture *attachments*, +/// which are attachments and use the attachment scheme. +/// +/// This is a *third* format, unrelated to `legacy_decrypt` above despite both being "the old way". +/// Attachments used AES-256-CBC with a bolted-on HMAC, a 64-byte key and a digest carried +/// separately; display pictures used AES-256-GCM with a 32-byte key and nothing out of band. Same +/// file server, same clients, same era, two schemes — Session inherited both from Signal, where +/// attachments and profile material were unrelated subsystems, and the stream scheme is what +/// finally unified them. +/// +/// The layout is `nonce || AES-256-GCM(plaintext) || tag`, and nothing in it says which format it +/// is: that is decided by what was being downloaded and by whether its url carried the `d` fragment +/// that means stream encryption. Which is why choosing belongs in one place — see +/// `Client::_download_decrypted` — rather than at each call site. +/// +/// Inputs: +/// - `encrypted` -- the downloaded file, entire. +/// - `key` -- the 32-byte key from the profile pic or group info. +/// +/// Outputs: +/// - std::vector of decrypted data. No padding is involved in this scheme. +/// +/// Throws std::runtime_error if the tag does not verify or the input is too short to hold one. +std::vector legacy_display_pic_decrypt( + std::span encrypted, + std::span key); + /// API: crypto/attachment::Decryptor /// /// Object-based interfaced to streaming decryption. The basic usage is to construct the object @@ -288,7 +371,7 @@ class Decryptor { bool failed = false; bool finished = false; bool hit_final = false; - cleared_uc32 key; + cleared_b32 key; unsigned char st_data[52]; // crypto_secretstream_xchacha20poly1305_state data void process_header(std::span chunk); @@ -317,6 +400,140 @@ class Decryptor { [[nodiscard]] bool finalize(); }; +/// API: crypto/attachment::Encryptor +/// +/// Streaming two-phase encryptor for attachments. Encryption is deterministic: the same seed and +/// data always produce the same key, nonce, and ciphertext, which allows the file server to +/// deduplicate identical uploads. +/// +/// **Phase 1 (key derivation):** Construct the object and call `update()` with the plaintext data +/// (in any number of pieces). This hashes the data to derive the encryption key and nonce. +/// Normally this is the file contents itself (so that the same file always produces the same +/// encryption key); however, feeding different data (e.g. random bytes) is permitted for +/// non-deterministic encryption where deduplication is not desired. No encrypted output is +/// produced during this phase. +/// +/// **Phase 2 (encryption):** Call `start_encryption()` to finalize key derivation and transition to +/// encryption mode. Then call `next()` repeatedly to pull encrypted chunks (the encryptor reads +/// from the data source provided to `start_encryption()`). Each call returns a span of encrypted +/// output valid until the next `next()` call, or an empty span when encryption is complete. +/// +/// The `from_file()` factory handles the common case of encrypting a file: it opens the file, runs +/// phase 1, seeks back, and returns an Encryptor ready for `next()` calls with the file as the +/// data source. +class Encryptor { + alignas(64) std::byte hash_st_data[384]; // crypto_generichash_blake2b_state + cleared_array nonce_key; + std::byte ss_st_data[52]; // crypto_secretstream_xchacha20poly1305_state + + // Phase 1 state + size_t hashed_size = 0; + bool phase1_done = false; + // Set by the key-taking constructor: there is no key to derive, so phase 1 is skipped entirely + // and start_encryption() must not finalize a hash that was never started. + bool key_given = false; + + // Phase 2 state + std::function buffer)> source; + size_t encrypt_size = 0; + size_t encrypted_so_far = 0; + size_t padding = 0; + size_t padding_remaining = 0; + bool header_emitted = false; + bool done = false; + + // Internal buffers for producing encrypted output + std::vector plaintext_buf; + std::array out_buf; + size_t out_size = 0; + + // Produces the next chunk of encrypted output into out_buf. Returns false when done. + bool produce_next(); + + public: + /// Returns the data size: during phase 1 this is the number of bytes fed to update_key(); + /// after start_encryption() this is the target plaintext size for phase 2 (either the + /// phase 1 total, or the override if one was given to start_encryption()). + size_t data_size() const { return phase1_done ? encrypt_size : hashed_size; } + + /// Constructs an encryptor for the given seed and domain. + /// + /// `seed` must be at least 32 bytes; typically the user's Session seed. `domain` is the + /// domain separator (ATTACHMENT or PROFILE_PIC). + Encryptor(std::span seed, Domain domain); + + /// Constructs an encryptor that uses a key we choose rather than one derived from the content, + /// for encrypting something to our own disk rather than to a file server. + /// + /// Phase 1 does not apply and update_key() must not be called: there is nothing to derive. Go + /// straight to start_encryption(), which then *requires* its `encrypt_size` argument, since + /// without phase 1 nothing else knows how much is coming. + /// + /// The nonce is random per encryption rather than derived. That is not a detail: the same key + /// is used for every file, so a derived-from-content nonce would repeat the keystream for + /// anything encrypted twice, and a fixed one would repeat it for everything. The consequence + /// is that this is *not* deterministic — encrypting the same bytes twice gives different output + /// — which is the opposite of what the seed-based constructor is for, and is right here: file + /// server deduplication is exactly what a local cache does not want. + /// + /// Output is the same `'S'`-prefixed chunked format, so `decrypt(data, key)` reads it back + /// unchanged, padding included. Padding is kept rather than skipped: it hides a plaintext's + /// exact size from whoever holds the ciphertext, and a local disk is held by backups, disk + /// images and whoever ends up with the machine. An exact size identifies a file — against a + /// known image, or against an upload someone watched go out — so the reason for padding it on + /// the way to a file server applies here too. + explicit Encryptor(std::span key); + + /// Phase 1: feed plaintext data into key derivation (hashing). + /// The data is hashed to derive the encryption key; normally this should be the actual file + /// contents that will be encrypted in phase 2. + void update_key(std::span data); + + /// Transition from phase 1 to phase 2. Finalizes the key derivation and prepares for + /// encryption. + /// + /// `allow_large` permits data larger than MAX_REGULAR_SIZE. + /// + /// `encrypt_size` overrides the expected plaintext size for phase 2. If omitted, the size + /// from phase 1 (sum of update() calls) is used. + /// + /// `source` is a pull-based data source for phase 2: it is called with a buffer to fill and + /// must fill it completely; returning fewer bytes than requested signals the end of data. + /// Phase 2 is then driven by next() calls which pull from this source. + /// + /// Returns the decryption key (in a cleared buffer). + cleared_b32 start_encryption( + std::function buffer)> source, + bool allow_large = false, + std::optional encrypt_size = std::nullopt); + + /// Pull the next chunk of encrypted output. Returns a non-owning span that is valid until + /// the next call to next(). Returns an empty span when all data has been encrypted. + std::span next(); + + /// Runs both phases from a file: hashes the file contents (phase 1), then sets up + /// streaming encryption with the file as the data source (phase 2). After this call, + /// next() returns encrypted chunks. The file is held open internally for phase 2 reads. + /// + /// Must be called on a freshly constructed Encryptor (i.e. before any update_key() calls). + /// + /// If `progress` is provided, it is called periodically during phase 1 with (bytes_read, + /// total_size). If the callback throws, the operation is aborted and the exception + /// propagates to the caller. + cleared_b32 load_key_from_file( + const std::filesystem::path& file, + bool allow_large = false, + std::function progress = nullptr); + + /// Factory: constructs an Encryptor, runs load_from_file, and returns the ready Encryptor + /// along with the decryption key. + static std::pair from_file( + std::span seed, + Domain domain, + const std::filesystem::path& file, + bool allow_large = false); +}; + /// API: crypto/attachment::decrypt /// /// Decrypts an attachment allegedly produced by attachment::encrypt to an output file. Overwrites diff --git a/include/session/blinding.hpp b/include/session/blinding.hpp index fed3d8fff..e4d90de7d 100644 --- a/include/session/blinding.hpp +++ b/include/session/blinding.hpp @@ -4,6 +4,7 @@ #include #include +#include "crypto/ed25519.hpp" #include "platform.hpp" #include "sodium_array.hpp" @@ -59,13 +60,12 @@ namespace session { /// Returns the blinding factor for 15 blinding. Typically this isn't used directly, but is /// exposed for debugging/testing. Takes server pk in bytes, not hex. -std::array blind15_factor(std::span server_pk); +b32 blind15_factor(std::span server_pk); /// Returns the blinding factor for 25 blinding. Typically this isn't used directly, but is /// exposed for debugging/testing. Takes session id and server pk in bytes, not hex. session /// id can be 05-prefixed (33 bytes) or unprefixed (32 bytes). -std::array blind25_factor( - std::span session_id, std::span server_pk); +b32 blind25_factor(std::span session_id, std::span server_pk); /// Computes the two possible 15-blinded ids from a session id and server pubkey. Values accepted /// and returned are hex-encoded. @@ -76,8 +76,7 @@ std::array blind15_id(std::string_view session_id, std::string_v /// session_id here may be passed unprefixed (i.e. 32 bytes instead of 33 with the 05 prefix). Only /// the *positive* possible ID is returned: the alternative can be computed by flipping the highest /// bit of byte 32, i.e.: `result[32] ^= 0x80`. -std::vector blind15_id( - std::span session_id, std::span server_pk); +b33 blind15_id(std::span session_id, std::span server_pk); /// Computes the 25-blinded id from a session id and server pubkey. Values accepted and /// returned are hex-encoded. @@ -86,23 +85,22 @@ std::string blind25_id(std::string_view session_id, std::string_view server_pk); /// Same as above, but takes the session id and pubkey as byte values instead of hex, and returns a /// 33-byte value (instead of a 66-digit hex value). Unlike the string version, session_id here may /// be passed unprefixed (i.e. 32 bytes instead of 33 with the 05 prefix). -std::vector blind25_id( - std::span session_id, std::span server_pk); +b33 blind25_id(std::span session_id, std::span server_pk); /// Computes the 15-blinded id from a 32-byte Ed25519 pubkey, i.e. from the known underlying Ed25519 /// pubkey behind a (X25519) Session ID. Unlike blind15_id, knowing the true Ed25519 pubkey allows /// thie method to compute the correct sign and so using this does not require considering that the /// resulting blinded ID might need to have a sign flipped. /// -/// If the `session_id` is a non-null pointer then it must point at an empty string to be populated +/// If the `session_id` is a non-null pointer then it must point at a nullopt to be populated /// with the session_id associated with `ed_pubkey`. This is here for consistency with /// `blinded25_id_from_ed`, but unlike the 25 version, this value is not read if non-empty, and is /// not an optimization (that is: it is purely for convenience and is no more efficient to use this /// than it is to compute it yourself). -std::vector blinded15_id_from_ed( - std::span ed_pubkey, - std::span server_pk, - std::vector* session_id = nullptr); +b33 blinded15_id_from_ed( + std::span ed_pubkey, + std::span server_pk, + std::optional* session_id = nullptr); /// Computes the 25-blinded id from a 32-byte Ed25519 pubkey, i.e. from the known underlying Ed25519 /// pubkey behind a (X25519) Session ID. This will be the same as blind25_id (if given the X25519 @@ -110,56 +108,55 @@ std::vector blinded15_id_from_ed( /// known. /// /// The session_id argument is provided to optimize input or output of the session ID derived from -/// the Ed25519 pubkey: if already computed, this argument can be a pointer to a 33-byte string +/// the Ed25519 pubkey: if already computed, this argument can be a pointer to an optional b33 /// containing the precomputed value (to avoid needing to compute it again). If unknown but needed -/// then a pointer to an empty string can be given to computed and stored the value here. Otherwise +/// then a pointer to a nullopt can be given to compute and store the value here. Otherwise /// (if omitted or nullptr) then the value will temporarily computed within the function. -std::vector blinded25_id_from_ed( - std::span ed_pubkey, - std::span server_pk, - std::vector* session_id = nullptr); +b33 blinded25_id_from_ed( + std::span ed_pubkey, + std::span server_pk, + std::optional* session_id = nullptr); /// Computes a 15-blinded key pair. /// /// Takes the Ed25519 secret key (64 bytes, or 32-byte seed) and the server pubkey (in hex (64 /// digits) or bytes (32 bytes)). Returns the blinded public key and private key (NOT a seed). /// -/// Can optionally also return the blinding factor, k, by providing a pointer to a uc32 (or -/// cleared_uc32); if non-nullptr then k will be written to it. +/// Can optionally also return the blinding factor, k, by providing a pointer to a b32; if +/// non-nullptr then k will be written to it. /// /// It is recommended to pass the full 64-byte libsodium-style secret key for `ed25519_sk` (i.e. /// seed + appended pubkey) as with just the 32-byte seed the public key has to be recomputed. -std::pair, cleared_uc32> blind15_key_pair( - std::span ed25519_sk, - std::span server_pk, - std::array* k = nullptr); +std::pair blind15_key_pair( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + b32* k = nullptr); /// Computes a 25-blinded key pair. /// /// Takes the Ed25519 secret key (64 bytes, or 32-byte seed) and the server pubkey (in hex (64 /// digits) or bytes (32 bytes)). Returns the blinded public key and private key (NOT a seed). /// -/// Can optionally also return the blinding factor, k', by providing a pointer to a uc32 (or -/// cleared_uc32); if non-nullptr then k' will be written to it, where k' = ±k. Here, `k'` can be -/// negative to cancel out a negative in the true pubkey, which the remote client will always assume -/// is not present when it does a Session ID -> Ed25519 conversion for blinding purposes. +/// Can optionally also return the blinding factor, k', by providing a pointer to a b32; if +/// non-nullptr then k' will be written to it, where k' = ±k. Here, `k'` can be negative to cancel +/// out a negative in the true pubkey, which the remote client will always assume is not present +/// when it does a Session ID -> Ed25519 conversion for blinding purposes. /// /// It is recommended to pass the full 64-byte libsodium-style secret key for `ed25519_sk` (i.e. /// seed + appended pubkey) as with just the 32-byte seed the public key has to be recomputed. -std::pair, cleared_uc32> blind25_key_pair( - std::span ed25519_sk, - std::span server_pk, - std::array* k_prime = nullptr); +std::pair blind25_key_pair( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + b32* k_prime = nullptr); /// Computes a version-blinded key pair. /// /// Takes the Ed25519 secret key (64 bytes, or 32-byte seed). Returns the blinded public key and -/// blinded libsodium seed value. +/// blinded libsodium seed value (sensitive; uses cleared memory). /// /// It is recommended to pass the full 64-byte libsodium-style secret key for `ed25519_sk` (i.e. /// seed + appended pubkey) as with just the 32-byte seed the public key has to be recomputed. -std::pair, cleared_uc64> blind_version_key_pair( - std::span ed25519_sk); +std::pair blind_version_key_pair(const ed25519::PrivKeySpan& ed25519_sk); /// Computes a verifiable 15-blinded signature that validates with the blinded pubkey that would /// be returned from blind15_key_pair(). @@ -169,10 +166,15 @@ std::pair, cleared_uc64> blind_version_key_pair( /// /// It is recommended to pass the full 64-byte libsodium-style secret key for `ed25519_sk` (i.e. /// seed + appended pubkey) as with just the 32-byte seed the public key has to be recomputed. -std::vector blind15_sign( - std::span ed25519_sk, +b64 blind15_sign( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + std::span message); +/// String_view overload: accepts hex (64 digits) or raw bytes (32 bytes) as a string_view. +b64 blind15_sign( + const ed25519::PrivKeySpan& ed25519_sk, std::string_view server_pk_in, - std::span message); + std::span message); /// Computes a verifiable 25-blinded signature that validates with the blinded pubkey that would /// be returned from blind25_id(). @@ -182,10 +184,15 @@ std::vector blind15_sign( /// /// It is recommended to pass the full 64-byte libsodium-style secret key for `ed25519_sk` (i.e. /// seed + appended pubkey) as with just the 32-byte seed the public key has to be recomputed. -std::vector blind25_sign( - std::span ed25519_sk, +b64 blind25_sign( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + std::span message); +/// String_view overload: accepts hex (64 digits) or raw bytes (32 bytes) as a string_view. +b64 blind25_sign( + const ed25519::PrivKeySpan& ed25519_sk, std::string_view server_pk, - std::span message); + std::span message); /// Computes a verifiable version-blinded signature that validates with the version-blinded pubkey /// that would be returned from blind_version_key_pair. @@ -193,20 +200,20 @@ std::vector blind25_sign( /// Takes the Ed25519 secret key (64 bytes, or 32-byte seed), unix timestamp, method, path, and /// optional body. /// Returns the version-blinded signature. -std::vector blind_version_sign_request( - std::span ed25519_sk, +b64 blind_version_sign_request( + const ed25519::PrivKeySpan& ed25519_sk, uint64_t timestamp, std::string_view method, std::string_view path, - std::optional> body); + std::optional> body); /// Computes a verifiable version-blinded signature that validates with the version-blinded pubkey /// that would be returned from blind_version_key_pair. /// /// Takes the Ed25519 secret key (64 bytes, or 32-byte seed), current platform and unix timestamp. /// Returns the version-blinded signature. -std::vector blind_version_sign( - std::span ed25519_sk, Platform platform, uint64_t timestamp); +b64 blind_version_sign( + const ed25519::PrivKeySpan& ed25519_sk, Platform platform, uint64_t timestamp); /// Takes in a standard session_id and returns a flag indicating whether it matches the given /// blinded_id for a given server_pk. diff --git a/include/session/client.hpp b/include/session/client.hpp new file mode 100644 index 000000000..0d2240b78 --- /dev/null +++ b/include/session/client.hpp @@ -0,0 +1,1264 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client/schema/schema_registry.hpp" + +/// `session::client::Client` is the conversation-level data model of a Session client: the +/// conversation list, message history, unread state and drafts, built on top of a `core::Core` +/// which owns the synced account state (keys, configs, polling, decryption). +/// +/// Client owns a Core rather than being part of one. The consequence that matters is the one the +/// compiler enforces: Core has no dependency on Client, and builds, tests and runs with none of +/// this in existence. A bot that wants raw protocol events constructs a bare Core; an application +/// that wants conversations constructs a Client and reaches through `client.core` for the account +/// state. +/// +/// Client is where interpretation lives. Core hands up an authenticated sender and a span of +/// decrypted bytes; everything after that is here — parsing the payload, deciding which +/// conversation it belongs to, whether it is one of ours, what it does to unread state and to the +/// order of the list. Composing outbound messages is the same job in reverse, which is why setting +/// and clearing protocol fields such as `syncTarget` happens at this layer and not below it. +/// +/// The rule, in one line: if a question can be answered without reading what a message *says*, it +/// belongs in Core; if answering it means interpreting the payload, it belongs here. +/// +/// session::client::Client client{ +/// std::filesystem::path{"/path/to/session.db"}, +/// session::sqlite::argon2id_password{"correct horse battery staple"}}; +/// +/// session::client::Client client{ +/// std::filesystem::path{"/path/to/session.db"}, +/// session::client::callbacks{ +/// .conversation_updated = [&](const auto& convo) { redraw(convo); }, +/// .message_added = [&](const auto& id, const auto& msg) { append(id, msg); }, +/// }}; +/// +/// for (const auto& convo : client.conversations()) +/// std::cout << convo.name_or_id() << ": " << convo.unread << " unread\n"; +/// +/// Client shares Core's database — the same file and the same connection pool, not a second +/// database — so a write from a Client handler joins whatever transaction Core already has open on +/// that thread. +// Forward declared rather than included: the generated protobuf headers are large and this one is +// public. Only referenced by private members below. +namespace SessionProtos { +class Content; +class DataExtractionNotification; +class UnsendRequest; +} // namespace SessionProtos + +namespace session::client { + +using namespace std::literals; + +class Client { + friend class session::TestHelper; // for unit tests + + // A conversation *is* part of Client's interface, split off rather than added to: what these + // call is the same private machinery the methods here do. + friend class Conversation; + friend class DM; + + public: + /// Constructs a Client and, internally, the Core it sits on. Takes the options `core::Core` + /// takes (database encryption, predefined_seed, …) and forwards them — with the one exception + /// of `core::callbacks`, which a Client's application cannot supply: those are Client's own + /// wiring, and the static_assert below rejects an attempt to pass a set rather than letting it + /// be silently overwritten. + /// + /// What an application is told is `client::callbacks`, given as the `cbs` argument of the + /// overloads below and reported through the dispatcher. Anything an application needs that + /// only Core knows is Client's job to handle and re-report there; if something Core reports has + /// no `client::callbacks` equivalent, that is a gap to fill here rather than a reason to reach + /// past Client for it. + template + explicit Client(std::filesystem::path db_path, Opts&&... opts) : + Client{std::move(db_path), callbacks{}, std::forward(opts)...} {} + + /// As below, additionally taking the dispatcher every handler is delivered through. See + /// `dispatcher`; without one, handlers run on Core's event loop. + template + explicit Client( + std::filesystem::path db_path, callbacks cbs, dispatcher dispatch, Opts&&... opts) : + Client{std::move(db_path), std::move(cbs), std::forward(opts)...} { + set_dispatcher(std::move(dispatch)); + } + + /// As above, additionally taking the change notifications to deliver. They are fixed for the + /// life of the Client, exactly as core::callbacks are, and are the only way an application is + /// told anything -- see `callbacks`, and read the startup ordering note there before using it. + template + explicit Client(std::filesystem::path db_path, callbacks cbs, Opts&&... opts) : + _cbs{std::make_shared(std::move(cbs))}, + core{std::move(db_path), + _core_callbacks(), + core::schema_extension{"client", schema::MIGRATIONS, schema::FULL_SCHEMA}, + std::forward(opts)...} { + static_assert( + (!std::same_as, core::callbacks> && ...), + "A Client's Core callbacks are its own wiring and cannot be supplied: what an " + "application is told is client::callbacks, passed as this constructor's `cbs`."); + _init(); + } + + ~Client(); + + // -- Conversations and messages --------------------------------------------------------------- + // + // None of these touches the database on the calling thread: they hand the work to Core's event + // loop, which is the only thread that ever touches it. + // + // Not because doing otherwise is unsafe -- the connection pool hands each thread its own + // connection, and WAL lets a reader run alongside the loop's writer -- but because of what it + // costs and what it cannot promise. A first read from the calling thread opens a second + // encrypted connection to keep for the life of that thread; a write contends with the loop's + // writer for the single WAL write lock and gives up after the 5s busy timeout; and a read gets + // a snapshot whose relationship to the notifications the application has been given is + // undefined, so two reads either side of a callback can disagree. Going through the loop makes + // all three questions not arise. + // + // A caller that genuinely wants to read on its own thread anyway can reach the pool through + // `core.database()` and will not corrupt anything. It is just answering a different, weaker + // question than these do. + // + // Each returns immediately and invokes `cb` when the work is done -- through the dispatcher if + // one was given, so on the application's own thread. A caller that would rather block on the + // answer than be handed it passes `await` in place of the handler and takes the return value. + // + // **Every `cb` is invoked exactly once**, unless the Client is destroyed before its work runs. + // That is what its leading `error` argument is for: unset when the call succeeded, and + // otherwise carrying what went wrong, so that a caller is never left waiting on an answer that + // will not come. The other arguments then mean only what they always meant -- an unset + // `std::optional` says the conversation does not exist, and never that we could + // not find out. + // + // The message is the thrown exception's, which is generally SQLite's, and is passed along as-is + // rather than reduced to a category of our invention: what went wrong is worth more in a log or + // a bug report than an enumeration that discarded it. + // + // Argument validation happens on the calling thread, before anything is dispatched, so misuse + // still throws where the mistake is, rather than arriving later as an error argument. + + /// The conversation list: pinned first, then most recently active. + /// + /// Message requests are not in it — see `message_requests()` — and neither are hidden + /// conversations. + void conversations(failable_function)> cb); + std::vector conversations(await_t); + + /// The message requests: accounts that have written to us and that we have never written to, + /// most recently active first. + /// + /// Disjoint from `conversations()`, and the same objects otherwise — a request has history, a + /// name and an unread count, and `Conversation::request` is true on every one of these. It + /// stops being a request when we answer it, since writing to someone is what approving them + /// is; there is no separate accept, and no way back short of deleting the contact. + void message_requests(failable_function)> cb); + std::vector message_requests(await_t); + + /// One conversation, or nullopt if we have no such conversation. + /// + /// What you get back carries the operations as well as the values — see `Conversation` — so + /// this is how a caller holding only an id reaches everything that can be done to it. + /// + /// client.conversation(id, [](auto err, auto convo) { + /// if (convo) convo->mark_read(cb); + /// }); + /// + /// `dm()` is the same question asked of a kind you already know, so that what comes back is a + /// `DM` and needs no narrowing. + /// + /// @throws std::invalid_argument, from `dm()`, if the id is not a one-to-one conversation. + void conversation( + const ConversationId& id, failable_function)> cb); + void dm(const ConversationId& id, failable_function)> cb); + std::optional conversation(const ConversationId& id, await_t); + std::optional dm(const ConversationId& id, await_t); + + /// The conversation with someone, made if it was not there already — "open a chat with this + /// account", which is the one thing `conversation()` cannot express. + /// + /// Cannot answer "no such conversation": that is the difference from `dm()`, and it is why the + /// waiting form hands back a `DM` rather than an optional. Opening one that already exists + /// changes nothing and simply gives it to you. + /// + /// The handler form still carries an optional, because a call that has been dispatched can + /// still fail — a disk error — and has nowhere to throw. It is unset only when `error` is set, + /// never because the conversation was missing. + /// + /// Only DMs so far, and so only this one; groups and communities are joined and created rather + /// than opened, and will say so in their own words when they arrive. + /// + /// @throws std::invalid_argument if the id is not a one-to-one conversation. + void open_dm(const ConversationId& id, failable_function)> cb); + DM open_dm(const ConversationId& id, await_t); + + /// True if this is our own account's session ID. + /// + /// Touches no database and is not dispatched, so it is callable from any thread: it compares + /// against our own ID, which is fixed once the account exists. False rather than throwing when + /// there is no account yet — with no identity, nothing can be us. + bool is_me(std::span session_id); + + /// True if `id` is the conversation with our own account — Session's "Note to Self". Same + /// answer as `Conversation::note_to_self`, for a caller holding only an id. + /// + /// Unlike the accessors around it this touches no database and is not dispatched onto Core's + /// loop: it compares against our session ID, which is fixed once the account exists. So it is + /// callable from any thread, including a render loop, and does not need a callback form. + /// + /// False rather than throwing when there is no account yet: with no identity, nothing can be a + /// conversation with ourselves. + bool is_note_to_self(const ConversationId& id); + + /// A single message by its Client-assigned id, or nullopt if it does not exist. + void message(int64_t id, failable_function)> cb); + std::optional message(int64_t id, await_t); + + /// Blocks or unblocks an account named by id. + /// + /// The same operation as `DM::set_blocked`, and forwards to it; it is here as well because + /// blocking is a fact about a *relationship* rather than about a conversation, and there is not + /// always a conversation to hang it off — blocking someone whose name you found in a group has + /// nothing to open first, and opening one would be the wrong thing to do about it. + /// + /// @throws std::invalid_argument if the id is not a one-to-one conversation, or is our own. + void set_blocked(const ConversationId& id, bool blocked, failable_function cb); + void set_blocked(const ConversationId& id, bool blocked, await_t); + + /// Sends to a conversation named by id, creating it if it does not exist. + /// + /// The same operation as `Conversation::send_message`, and forwards to it; it is here as well + /// because this is the one thing that cannot require a conversation to already exist — + /// messaging an account you have never spoken to is how the conversation begins. With one in + /// hand, send through it and skip the lookup. + /// + /// See `Conversation::send_message` for what the arguments mean and what `on_upload` reports. + /// + /// @throws std::invalid_argument if the conversation is not a DM (groups and communities are + /// not implemented yet), if an attachment cannot be read, or if `reply_to` names a message that + /// does not exist or belongs to another conversation; thrown on the calling thread, before + /// anything is stored or dispatched. + void send_message( + const ConversationId& id, + OutgoingMessage msg, + Conversation::upload_progress on_upload, + failable_function cb); + void send_message( + const ConversationId& id, + OutgoingMessage msg, + failable_function cb); + int64_t send_message( + const ConversationId& id, + OutgoingMessage msg, + Conversation::upload_progress on_upload, + await_t); + int64_t send_message(const ConversationId& id, OutgoingMessage msg, await_t); + + /// Sends a failed message again, resuming rather than restarting: attachments that already + /// reached the file server are left alone and only the ones that did not are uploaded, because + /// what each upload achieved is recorded against the message. A message whose files all got + /// up but whose send failed is simply dispatched again, uploading nothing. + /// + /// Returns false, having done nothing, if the message cannot be retried: it does not exist, it + /// is not one of ours, it is not in a failed state, or it is SendState::unsendable — that last + /// being the one an application should offer deletion for rather than a retry, since what it + /// needs is gone rather than merely unreachable. + /// + /// `on_upload` reports as it does for send_message, indexed the same way, so a display built + /// for the original send works unchanged for the retry. Note that a retry is where a file + /// that has since been deleted is discovered: an attachment that has gone moves the message to + /// SendState::unsendable, which is terminal. + void retry_send( + int64_t message_id, + std::function< + void(size_t index, int64_t sent, int64_t total, std::optional result)> + on_upload, + failable_function cb); + bool retry_send(int64_t message_id, Conversation::upload_progress on_upload, await_t); + bool retry_send(int64_t message_id, await_t); + + /// Deletes one message from this device: its body, its decrypted content and everything it + /// recorded about its attachments go, and what is left says only that someone said something + /// here and when. + /// + /// The row stays, marked `Deletion::here`, and this is not squeamishness about the data. The + /// swarm still holds the message, and the swarm hash on that row is the only thing that + /// recognises it if it is delivered again -- which a storage server makes likely rather than + /// hypothetical, since it stops honouring a `last_hash` once that has expired and answers the + /// next poll with the whole retention window. Delete the row and the message comes back + /// looking new. + /// + /// Files are not touched. An attachment's path names a file the user chose -- one they picked + /// to send, or a place they asked a download to be put -- so it is theirs in both directions + /// and nothing here has ever unlinked one. The rows that described the attachments do go. + /// + /// Deleting an unread incoming message makes it read, since there is no longer anything to + /// read; the conversation's unread count follows. + /// + /// This is the whole of "delete for me": nothing is sent, so nothing tells the other side or + /// our own other devices. A message deleted here can still be deleted everywhere afterwards, + /// which is why how far it went is recorded rather than merely that it happened. + /// + /// Returns false, having done nothing, if there is no such message. Deleting one already + /// deleted here is not an error and changes nothing. + void delete_message(int64_t message_id, failable_function cb); + bool delete_message(int64_t message_id, await_t); + + /// Deletes a message we sent, here and everywhere else it reached. + /// + /// Does everything `delete_message` does, and then two things more: removes our own swarm's + /// copy — the one our other devices read — and asks the recipient to remove theirs. + /// + /// **Only for messages we sent.** Not a policy choice: an unsend request is honoured only from + /// the message's author or from one of your own devices, so asking a stranger to delete their + /// own message is asking for something the other end will ignore. Returns false for an + /// incoming message, having done nothing — a caller wanting that message gone locally wants + /// `delete_message`. + /// + /// **The remote half cannot be confirmed, and is not reported.** `cb` fires once the local + /// deletion is done, which is immediate and certain; the swarm delete and the request to the + /// recipient are dispatched and not waited on. There is no acknowledgement to wait for — a + /// recipient may be offline for a week, may be running a client that ignores unsend requests, + /// and may have already read and screenshotted it. Reporting "deleted everywhere" as a + /// completed fact would be a promise nothing can keep, so this reports what it did rather than + /// what it achieved. + /// + /// The message is marked `Deletion::everywhere` locally either way, since that records what we + /// asked for, and it is what stops a client offering the same deletion twice. + void delete_message_everywhere(int64_t message_id, failable_function cb); + bool delete_message_everywhere(int64_t message_id, await_t); + + /// Shows, or stops showing, a message as a gallery. + /// + /// Returns false, having done nothing, if the message does not exist or is not + /// `gallery_viewable` — a message that cannot be shown that way cannot be asked to be, so the + /// stored decision can never disagree with the rule that governs it. Clearing is always + /// allowed. + /// + /// Starts nothing. Turning gallery mode on for a conversation that does not auto-download + /// leaves its images unfetched, and getting them is the caller's move — `attachment_data` for + /// each — because a setter that reached for the network would surprise whoever called it. + void set_gallery(int64_t message_id, bool gallery, failable_function cb); + bool set_gallery(int64_t message_id, bool gallery, await_t); + + /// An attachment's contents, decrypted and whole. + /// + /// For showing a file rather than keeping it: a gallery needs the bytes, and `save_attachment` + /// would have it write files it then reads back and deletes. + /// + /// Served from the cache when it is there, and fetched *and cached* when it is not — which is + /// the difference from `save_attachment`, which reads the cache but never fills it. A save has + /// a home of its own to put the file in; a display does not, and would otherwise re-fetch on + /// every scroll. + /// + /// Not subject to the auto-download size limit. That governs what arrives unasked, and this is + /// asked for. + void attachment_data( + int64_t message_id, + size_t index, + std::function on_progress, + failable_function)> cb); + + /// Removes one deleted message's leftover row. The conversation-wide form, and the reason a + /// deletion leaves a row at all — including how this can bring a message back — are on + /// `Conversation::purge_deleted`. + /// + /// Returns false, having done nothing, if the message does not exist or has not been deleted. + /// Refusing a live message is the point rather than a nicety: this is the one operation here + /// that removes history outright, and it is only ever entitled to remove what a deletion left. + void purge_deleted_message(int64_t message_id, failable_function cb); + bool purge_deleted_message(int64_t message_id, await_t); + + /// The message as it arrived on the wire, rendered as indented text: one line per field that is + /// set, named, with nested messages beneath their field and enums by name. + /// + /// For seeing what a client actually sent, which nothing else here answers. A message is + /// stored twice — as the columns this schema models, and as the whole decrypted `Content` it + /// came in — and everything else reads the first of those. + /// + /// Rendered here rather than handed over as bytes, because the wire format is this layer's to + /// interpret: nothing above Client parses a protobuf, and the generated headers are kept out of + /// the public ones deliberately. + /// + /// Deliberately not a field on `Message`: this is asked for one message at a time by someone + /// looking, and putting it on the row would drag the whole wire form through the query that + /// draws a conversation. + /// + /// Returns nullopt when there is nothing to show — which is not an error, and covers three + /// cases a viewer is free to render alike: a message composed here that was never given a + /// stored wire form, one whose content has been deleted, and one whose content some later + /// pruning removed. Also nullopt if the stored bytes no longer parse, which is corruption + /// rather than absence and is logged as such. + void message_debug(int64_t message_id, failable_function)> cb); + std::optional message_debug(int64_t message_id, await_t); + + /// Fetches one of a message's attachments and writes it to `dest`, decrypting it on the way. + /// + /// Nothing is downloaded until this is called. An arriving message records where its files are + /// and what they are called, and stops there: whether a file is worth the bandwidth is the + /// application's decision, and on a metered connection it is the user's. + /// + /// `dest` is where the caller *asked* for it, and is not remembered. Where a file went is the + /// application's business — it chose the location and can move it afterwards — so nothing here + /// would stay true. Saving the same attachment twice to two places is therefore fine and means + /// what it says. The file is written whole or not at all: it lands at a temporary name beside + /// `dest` and is renamed only once it has been decrypted and verified, so an interrupted save + /// leaves no half-file that looks finished. + /// + /// **`cb` reports where it actually went**, which is not always `dest`. Whether `dest` was + /// free is something the caller decided when it asked its user; the rename happens when the + /// download finishes, which may be minutes later, and anything that has appeared there in + /// between is a file nobody agreed to lose. So by default the finished file takes the next + /// free `name (2)` instead — before the extension, since only that still opens on a + /// double-click. + /// + /// A caller whose user has *already* been shown what is there and said replace it passes + /// `replace` and gets `dest` whatever has happened since. That is not the same decision and + /// must not be guessed at: renaming an approved overwrite would leave the file it was meant to + /// replace sitting there, which discards the answer the user gave rather than protecting a + /// stranger's file. + /// + /// `on_progress` reports as `send_message`'s `on_upload` does and is indexed the same way, so + /// a display built for sending works unchanged in the other direction: `result` unset means + /// under way with `done`/`total` in encrypted bytes, `result == 0` means written and verified, + /// and anything else is the failure's status code. + /// + /// `notify_sender` tells the person who sent it that we saved their file, which is what + /// Session's other clients do and so what a recipient expects. Passing false keeps this + /// particular save to ourselves. Only ever sent for a message we received: saving from our own + /// message notifies nobody, and neither does a failed save. + /// + /// The account's own preference — `UserProfile::get_notify_media_saved`, which follows it + /// between devices — can refuse the notification but cannot require one. So a client that has + /// not grown a setting for this still honours one made elsewhere, and a caller that passes + /// false is not overruled. + /// + /// Which of Session's two attachment encryptions applies is read from the url, so a caller + /// neither chooses nor needs to know: current clients still send the legacy scheme, and files + /// we send use the stream one. + /// + /// The error a failure reports is worth showing rather than a generic one: an attachment that + /// the file server no longer holds, one whose sender described it wrongly, and one that failed + /// to authenticate are different problems, and only the first is worth retrying. + /// + /// @throws std::invalid_argument if `dest` names a directory or its parent does not exist; + /// thrown on the calling thread, before anything is fetched. + void save_attachment( + int64_t message_id, + size_t index, + std::filesystem::path dest, + std::function on_progress, + failable_function cb, + bool notify_sender = true, + bool replace = false); + + /// Sets, replaces or removes the dispatcher every handler is delivered through, which a caller + /// whose loop does not exist yet when the Client is built needs: an application typically opens + /// its database, constructs this, and only then creates the window that owns the loop. + /// + /// Safe to call while Core is running and from any thread. A handler already on its way either + /// goes to the old dispatcher or the new one; there is no flush and none is needed, since + /// passing nullptr simply means what no dispatcher has always meant -- handlers run on Core's + /// event loop. An application shutting its loop down therefore has the choice of unsetting + /// this or having its own dispatcher run the work inline once posting stops arriving anywhere. + void set_dispatcher(dispatcher d); + + /// Gives us somewhere to keep what we download, and turns caching on. + /// + /// Unset by default, and with it unset nothing is cached: every fetch goes to the file server. + /// That is the safe default rather than a limitation — we are handed a database file, not + /// permission to write beside it, and an embedder that has somewhere in mind (an Android cache + /// dir, an XDG cache home) should be the one to say where. + /// + /// **The directory must be ours outright.** Freeing what nothing references any more works by + /// listing the directory and unlinking what is not in the list, which is only safe somewhere + /// nothing else writes. Point this at a directory with other things in it and they will go. + /// + /// Calling this starts one such pass, in the background, against what the database says should + /// be there. It is a net for what no code of ours was running to see -- a crash between + /// writing a file and recording it, a contact whose row went by cascade -- so it is done once, + /// here, rather than on a timer: those are things that happen while we are not looking, and + /// this is the moment we look. It runs off the event loop and reports only to the log. + /// + /// Contents are encrypted under a key generated once and kept in the database, so they outlive + /// the message or config entry whose key originally opened them — and so the files are not + /// readable by whoever ends up with the disk. That protection is only as good as the + /// database's: with an unencrypted database the key sits in plaintext beside them. + void set_cache_dir(std::filesystem::path dir); + + /// A conversation's picture, decrypted and ready to decode. + /// + /// Served from the cache when it is there, and fetched, decrypted and cached when it is not. + /// Which of those happened is deliberately not reported: it is the same picture either way, and + /// a caller that had to know would end up implementing the caching decision itself. + /// + /// `on_progress` reports the download as `save_attachment` does — `done`/`total` in encrypted + /// bytes while it runs, `result == 0` when it is in, anything else a failure status. It is not + /// called at all on a cache hit: there is no progress to draw when the bytes are already here, + /// and a progress bar that flashes for a cached read is worse than none. + /// + /// Returns nullopt when there is no picture to fetch — nobody has told us of one, or this is a + /// group or community, whose pictures are real but not wired up yet. An actual failure to + /// fetch reports through `cb`'s error rather than as nullopt, so "there isn't one" and "we + /// could not get it" stay apart. + /// + /// Needs `set_cache_dir` to cache; without it every call fetches. Nothing is cached for a + /// picture we could not decrypt, since what we would be storing is not the picture. + void profile_picture( + const ConversationId& id, + std::function result)> on_progress, + failable_function>)> cb); + void profile_picture( + const ConversationId& id, + failable_function>)> cb); + + /// How much disk the cached attachments may occupy in total, or nullopt for no limit. + /// + /// Measured as bytes on disk, so it counts what caching actually costs: the files are encrypted + /// and padded, which makes the total larger than the same attachments would be as plain files. + /// That is the right way round for a disk limit — 5GB means 5GB of disk. + /// + /// Display pictures are not counted and are never evicted for this. They are one small file + /// per contact, replaced rather than accumulated, and a contact you have not spoken to in years + /// should not lose the last picture you had of them; freeing them is the reference sweep's job. + /// + /// Going over evicts least-recently-*used* entries until it fits — used rather than oldest, so + /// something opened weekly does not lose to something downloaded once and never looked at. + /// + /// Persisted and device-local: a limit set in a settings screen should outlive the process, and + /// how much disk to spend is a property of this machine rather than of the account. Unlike + /// `set_cache_dir`, which is the application's to decide every run — a stored path would be the + /// wrong one the moment the database moved. + void set_attachment_cache_limit(std::optional bytes, failable_function cb); + void set_attachment_cache_limit(std::optional bytes, await_t); + void attachment_cache_limit(failable_function)> cb); + std::optional attachment_cache_limit(await_t); + + /// The largest attachment that will be fetched *unasked*, or nullopt for no limit. + /// + /// Compared against the size in the pointer, which is the file's own length — so a limit of 2MB + /// admits files up to 2MB, with no allowance for padding or framing. That is a claim by the + /// sender, and one that is separately held to: a transfer whose contents turn out to be a + /// different size from what was declared fails. + /// + /// Only ever applies to automatic downloads. `save_attachment` and `attachment_data` are + /// somebody asking for one particular file, and are never refused for being large. + /// + /// Persisted and device-local, for the same reasons as the cache limit. + void set_auto_download_max_size(std::optional bytes, failable_function cb); + void set_auto_download_max_size(std::optional bytes, await_t); + void auto_download_max_size(failable_function)> cb); + std::optional auto_download_max_size(await_t); + + // -- Our own account ---------------------------------------------------------------------- + // + // These read and write the UserProfile config, which follows the account between devices. They + // are here rather than left to `client.core.configs.user_profile()` because that has to be + // touched on Core's loop -- a config read racing a merge is not safe -- and an application + // drawing a settings screen is on its own thread. One place doing the hop correctly beats + // every caller discovering the rule, or not. + + /// Our own display name: what we publish about ourselves, empty until one is set. + /// + /// Not to be confused with a conversation's `display_name`, which is what we call somebody + /// else. This is account state and has no conversation: reading it off the note-to-self + /// conversation works only once that conversation exists, which is a bug waiting for a fresh + /// account. + void display_name(failable_function cb); + std::string display_name(await_t); + void set_display_name(std::string_view name, failable_function cb); + void set_display_name(std::string_view name, await_t); + + /// Whether to tell somebody when we save a file they sent us. + /// + /// Follows the account, so turning it off on one device turns it off everywhere. Composes with + /// `save_attachment`'s `notify_sender` in one direction only: this can refuse a notification + /// and cannot require one, so a caller passing false is never overruled, and a client with no + /// setting of its own still honours a choice made elsewhere. + void notify_media_saved(failable_function cb); + bool notify_media_saved(await_t); + void set_notify_media_saved(bool notify, failable_function cb); + void set_notify_media_saved(bool notify, await_t); + + /// How often a handler that reports continuously — such as attachment upload progress — is + /// allowed to fire, per thing being reported on. Defaults to 100ms; zero lets every update + /// through. + /// + /// A transfer reports far faster than a display can use, and with a dispatcher each of those + /// reports is a job handed to the application's loop, which for some toolkits means a repaint. + /// Squelching the ones in between costs nothing, because each update supersedes the last. + /// Whatever a caller must not miss — an upload starting, finishing, or failing — is reported + /// outside this and is never squelched. + /// + /// Safe to call while Core is running and from any thread; it applies to transfers begun after + /// it takes effect. + void set_high_freq_dispatch_interval(std::chrono::milliseconds interval); + + // -- Change notification ------------------------------------------------------------------ + // + // Handlers are given at construction; there is no way to add or remove one afterwards. See + // `callbacks` for what each reports, and note in particular that the initial conversation list + // must be taken *after* construction, never gathered before it: + // + // Client client{path, std::move(cbs)}; // 1. notifications start here + // auto convos = client.conversations(); // 2. install as the starting point + // + // Anything arriving between 1 and 2 is applied on top of the snapshot and lands on the right + // answer, because each handler carries the whole of the new state rather than a delta, so + // applying one twice changes nothing. + + private: + // Declared above `core`, which is declared last: see the note there. + // Shared rather than held, so that a handler queued to another thread stays valid if the + // Client is destroyed before it runs. + std::shared_ptr _cbs; + + // Read and written only on the loop, which is what set_dispatcher hops onto rather than + // synchronising. Unset means run on the loop. + dispatcher _dispatcher; + + // As above, and for the same reason. + std::chrono::milliseconds _high_freq_dispatch_interval{100}; + + // Core's send ids are per-process (its counter restarts at 1 on every run), so this mapping + // must not be persisted or a stale row would capture a later run's status updates. + struct OutgoingSend { + int64_t client_id; + // A note to self goes to our own swarm, so its swarm hash is one we can act on later; a + // send to someone else deposits on their swarm, and that hash is theirs to expire. + bool own_swarm; + }; + std::unordered_map _send_ids; // core send id -> the send it belongs to + + // Status updates that arrived from send_dm() before it returned, i.e. before we knew the core + // send id to map. Drained by send_message() once the mapping is registered. + struct EarlyStatus { + core::MessageSendStatus status; + std::optional swarm_hash; + }; + std::unordered_map _early_status; + + // Core send ids belonging to the copy of an outgoing message deposited in our own swarm. Their + // delivery status is deliberately not reported: what the application waits on is the copy going + // to the recipient, and a sync copy that fails costs an entry on our other devices rather than + // the message itself. Tracked rather than merely unregistered so that a late status does not + // accumulate in _early_status forever. + std::unordered_map _sync_sends; // core send id -> client message id + + // Sends whose outcome nobody is waiting for -- the media-saved notification is the only one so + // far. Tracked rather than left unregistered so that their statuses are dropped as they + // arrive, instead of accumulating in _early_status against ids that will never be claimed. + std::unordered_set _quiet_sends; + + // The actual work, all of it assuming it is already on the loop thread. The public methods + // above are dispatches onto that thread and nothing else; these are where the database is + // touched, and are also what Client's own handlers call, since those already run there. + // + std::span _self_or_none(); + std::vector _conversations(); + std::vector _message_requests(); + std::optional _conversation(const ConversationId& id); + AnyConversation _create_conversation(const ConversationId& id); + void _mark_read(const ConversationId& id, std::optional up_to); + void _set_priority(const ConversationId& id, int priority); + void _set_marked_unread(const ConversationId& id, bool unread); + void _set_notifications(const ConversationId& id, config::notify_mode mode); + void _set_mute_until(const ConversationId& id, std::chrono::sys_seconds until); + void _set_expiry( + const ConversationId& id, config::expiration_mode mode, std::chrono::seconds timer); + void _set_auto_download(const ConversationId& id, AutoDownload mode); + void _set_nickname(const ConversationId& id, std::string_view nickname); + + // One conversation-row column, updated where it differs and re-derived into the config if it + // did. Templated on the value only so the caller need not name the bind type. + template + void _set_conversation_setting(const ConversationId& id, std::string_view column, T value); + void _set_blocked(const ConversationId& id, bool blocked); + void _clear_messages(const ConversationId& id); + void _delete_conversation(const ConversationId& id, bool keep_messages); + void _delete_contact(const ConversationId& id); + bool _delete_message(int64_t message_id, Deletion how_far); + bool _delete_message_everywhere(int64_t message_id); + // Handles an inbound unsend request: finds what it names, if it names exactly one thing we + // hold, and deletes it. + void _on_unsend_request( + std::span sender, const SessionProtos::UnsendRequest& req); + // A download that is already happening, and everyone waiting on it. + // + // Keyed by the cache name -- the hashed base url -- because that is what identifies the *file*, + // so two messages quoting the same attachment share one transfer rather than racing to write + // one cache entry twice. + // + // Without this, a conversation opening while its attachments are auto-downloading would fetch + // every one of them a second time: the cache is still empty, so a display asking for bytes sees + // a miss and starts its own. Joining instead means a display never has to know whether + // something is already under way -- it asks for the bytes and gets them, whoever started it. + // + // Only transfers that *accumulate* are in here, which means everything except a save. A save + // streams decrypted bytes to the destination as they arrive and keeps none of them -- which is + // what stops a large file sitting in memory -- so by the time a second caller could join, the + // first half of the file has already gone to disk and is not ours to hand over. A save may + // therefore join something already accumulating, and costs nothing extra when it does, but is + // never itself joinable: a display asking during a save fetches the file again. + // + // **Only ever touched on the loop.** Download callbacks arrive on the network thread, so + // everything that reads or writes this hops first; the application's own callbacks then hop + // again, out through the dispatcher. + struct InFlight { + // The last figures reported, so somebody joining midway can be told where it has got to + // rather than being left with nothing to draw until the next chunk lands. + int64_t done = 0, total = 0; + std::shared_ptr> plain; + std::vector)>> progress; + std::vector)>> waiting; + }; + std::unordered_map _in_flight; + + // What an attachment row says about where its file is and how to open it. + struct StoredPointer { + std::string url; + std::vector key, digest; + std::optional size; + }; + // Throws if there is no such attachment, or if its sender gave no url. + StoredPointer _attachment_pointer(int64_t message_id, size_t index); + + // Writes `data` into the attachment cache under `url`, and records it. The row is an index + // over the file, so it is written after the file exists. + void _cache_attachment( + const std::string& url, + std::span key, + std::span data); + // Marks a cache entry as used now, which is what makes eviction least-recently-used. + void _touch_cached(const std::string& name); + + // Removes least-recently-used entries until the cache fits its limit, never touching `keep` -- + // which is whatever was just written, so that a download cannot complete and immediately + // vanish. Does nothing when no limit is set. + void _evict_cache(const std::string& keep); + + // Where a profile reached us from, which is what a field it does not carry means. + enum class ProfileSource { + config, // States the whole profile: no picture means they have none. + message, // Mentions one in passing: no picture means it did not say, so keep what we have. + }; + + // Writes a profile onto an account and returns whether anything changed, dropping the cached + // picture the account has stopped using. + // + // One helper rather than the same UPDATE at each place a profile arrives from: the cleanup + // belongs wherever the url changes, and a site that omitted it would strand a file that nothing + // afterwards can attribute to anyone. + bool _update_profile( + sqlite::Connection& c, + int64_t account, + const std::optional& name, + const std::optional& pic_url, + const std::optional>& pic_key, + int64_t updated, + ProfileSource source); + + // Unlinks a cached profile picture unless some account still names that url. Unlike an + // attachment there is no size limit and no expiry, so a picture nobody points at is reclaimed + // only by someone noticing that nobody points at it. + void _drop_unused_picture(sqlite::Connection& c, std::string_view url); + + // Starts one pass of reconciling the cache directories against the database, in the background. + // + // A net, not a mechanism. Everything that caches a file records it in the same breath, and + // everything that stops referencing one drops it. What collects here is what no code of ours + // was running to see: a crash between writing a file and recording it, an account row that went + // by cascade, a database restored from a backup older than the directory. Nothing else can + // ever attribute those files to anything, so nothing else can ever remove them. + void _sweep_cache(); + + // The deciding half of `_sweep_cache`, on the loop. Takes the directory listings because + // taking them is the slow part and does not belong here. + void _reconcile_cache(std::vector attachments, std::vector pictures); + + // Runs the listing half of a sweep. Joined before anything it touches goes away, which is why + // it hands its result back with `call_get`: joining a thread that had merely *posted* a job + // would not wait for the job. + std::thread _sweeper; + + void _attachment_data( + int64_t message_id, + size_t index, + std::function on_progress, + failable_function)> cb); + + // Decides what an arriving message's attachments are worth fetching unasked, sets whether it is + // shown as a gallery, and starts whatever it decided on. Does nothing without a cache + // directory: the point of fetching early is to have the file to hand, and with nowhere to keep + // it the download would be thrown away. + void _auto_download(const ConversationId& convo, int64_t message_id); + + bool _set_gallery(int64_t message_id, bool gallery); + bool _purge_deleted_message(int64_t message_id); + size_t _purge_deleted(const ConversationId& id); + std::optional _message_debug(int64_t message_id); + + // Where downloads are cached, and the key they are encrypted under. Empty path means no + // caching. The key is read (or generated) on first use rather than at construction, so an + // account that never caches anything never grows one. + std::filesystem::path _cache_dir; + std::optional _cache_key; + // Must be called on the loop: it touches globals. + const b32& _cache_encryption_key(); + void _profile_picture( + const ConversationId& id, + std::function)> on_progress, + failable_function>)> cb); + std::vector _messages( + const ConversationId& id, + int limit, + std::optional before, + bool include_deleted); + std::optional _message(int64_t id); + int64_t _send_message(const ConversationId& id, const OutgoingMessage& msg); + int64_t _send_message( + const ConversationId& id, + const OutgoingMessage& msg, + std::function)> on_upload); + + // Uploads the message's first attachment that has no url yet and, when there are none left, + // finishes the send. Each upload's completion calls this again, so the chain runs one file at + // a time and resumes wherever it was left -- which is also what a retry does. + void _upload_next( + int64_t client_id, + std::function)> on_upload); + + // Rebuilds the message's content with its now-uploaded attachments named in it, replaces what + // was stored, and dispatches it. Rebuilt from the database rather than from what send_message + // was given, so that this is reachable for a message whose uploads finished in an earlier run. + void _finish_attachment_send(int64_t client_id); + + // Marks a message as failed because its attachments could not be uploaded, and reports it. + // `permanent` distinguishes a file that is gone, which no amount of retrying will fix, from a + // transfer that merely did not work this time. + void _fail_attachment_send(int64_t client_id, bool permanent = false); + + bool _retry_send( + int64_t client_id, + std::function)> on_upload); + + // Wraps a progress callback so each report reaches the application through the dispatcher, and + // returns an empty function when given one — so a download can skip reporting entirely rather + // than call something whose whole body is a check that there is nothing to do. + // + // A caller reporting something narrower than "this download" — one attachment of several, say — + // binds that in first and hands the result here; what is shared is the hop and the emptiness, + // not what the numbers are about. + std::function)> _dispatch_progress( + std::function)> cb); + + // What is being downloaded. Only ever consulted to pick between the two *legacy* formats, + // which are different for no reason anyone chose — see `attachment::legacy_display_pic_decrypt` + // — and which nothing in the bytes distinguishes. + enum class DownloadKind { + attachment, ///< A file sent with a message. + display_pic, ///< A profile picture or a group avatar. + }; + + // Downloads `url`, decrypts it, and hands the plaintext to `on_plain` — possibly in pieces, and + // on the network's thread. Whatever wants the bytes decides what to do with them: write them + // to a file the user chose, keep them in memory, put them in the cache. + // + // **This is the one place that chooses between Session's three at-rest formats**, and no caller + // above it learns there was a choice. The rule: + // + // - the url carries a `d` fragment -> the stream scheme, whatever is being fetched. That + // fragment means stream encryption universally; it is the one honest discriminator here. + // - otherwise, `kind` decides: an attachment is AES-CBC with an HMAC and a separate digest; + // a display picture is AES-GCM with the nonce and tag inline. + // - no key at all -> plaintext. Community images are stored that way. + // + // `kind` is a parameter rather than something inferred from the key's length because the caller + // knows which it asked for, and inference would be a guess standing in for a fact: it happens + // to work today only because the two legacy key sizes differ, and would misroute silently the + // first time something else turned up with a 32-byte key and no `d`. + // + // The stream scheme decrypts as it arrives; both legacy ones have to accumulate, because their + // authentication covers the whole ciphertext and cannot be checked until all of it is here. + // + // Anything that goes wrong stops the transfer rather than being noted while the rest is + // received and thrown away. The stream scheme is what makes that worth doing: it authenticates + // each chunk as it arrives, so a failure surfaces when the bad chunk does -- which may be the + // first or may be most of the way in, but is not "once the whole file is here". + // + // `on_progress` reports in encrypted bytes, unindexed; a caller that reports per-attachment + // adds its own index. `on_done` fires exactly once, with the failure if there was one. + // + // Throws, before starting anything, if the url is not a download url, if no network is + // attached, or if the key or digest is the wrong length for the scheme that resolves to. + void _download_decrypted( + const std::string& url, + DownloadKind kind, + std::vector key, + std::vector digest, + std::optional claimed_size, + std::function plaintext)> on_plain, + std::function result)> on_progress, + std::function error)> on_done); + + // What a fetch needs to know about the file it is after, independent of who wants it. + struct FetchTarget { + std::string url; + std::vector key, digest; + std::optional claimed_size; + DownloadKind kind; + std::string_view dir; // cache::ATTACHMENT_DIR or cache::PROFILE_DIR + }; + + // Serves a file from the cache, joins a fetch of it already running, or starts one. + // + // The joining is what this exists for. Two things want the same file routinely -- an arrival + // starts a download and the display then asks for the very thing being downloaded -- so a + // second request attaches to the first, picking up its progress from wherever it has reached, + // rather than fetching the same bytes twice and caching them twice. + // + // `on_hit` runs when the cache answered and `store` after a fetch completes, both on the loop. + // They are the whole of the difference between a cached attachment, which is indexed and + // evictable, and a cached picture, which is neither. + void _fetch_cached( + FetchTarget target, + std::function result)> progress, + failable_function)> cb, + std::function on_hit, + std::function)> store); + + // The `store` a picture fetch wants, or nothing when there is nowhere to keep it. + std::function)> _store_picture(std::string url); + + // Queues a fetch of a picture we have just learned the url of, so that it is to hand before + // anything asks to draw it. Unconditional: unlike an attachment there is no setting, because a + // picture is small and is wanted the moment the conversation is shown. + // + // Takes the connection because it must read the key on the one whose transaction just wrote it, + // and hands the fetch values rather than an account to look up again: the write it is reacting + // to is not visible to any other connection until a commit that happens above this call. + void _prefetch_picture(sqlite::Connection& c, int64_t account, const std::string& url); + + // Fetches a display picture into the cache for nobody in particular, reporting to + // `display_picture_progress` as it goes. + void _fetch_picture(const ConversationId& id, std::string url, std::vector key); + + // Starts the download behind save_attachment. Everything after the row lookup happens off the + // loop, on the network's thread: the file is decrypted and written there, and nothing about it + // is recorded, so this is the one attachment path that never comes back to the database. + void _save_attachment( + int64_t message_id, + size_t index, + std::filesystem::path dest, + std::function on_progress, + failable_function cb, + bool notify_sender, + bool replace); + + // Tells a message's sender that we saved one of its attachments. Fire and forget: nothing + // waits on it and a failure is logged rather than reported, since it is a courtesy to them + // rather than part of what the caller asked for. + void _notify_media_saved(int64_t message_id, size_t index); + + // Whether *we* are the recipient of a message, which is what makes our own save of one of its + // attachments worth recording. True for anything incoming, and for a note to self, where the + // message is outgoing but the recipient is still us. + bool _saved_by_recipient(int64_t message_id); + + // Records that the recipient of a message saved one of its attachments: us, for one we + // received, and them for one we sent. An unset `index` means all of the message's + // attachments, which is what a peer saving several at once reports. + // + // Told rather than deciding: this is reached both by our own save and by a peer's notification, + // and only the caller knows which of those it is -- for a message we sent, our save is not the + // recipient's and theirs is. + void _record_saved(int64_t message_id, std::optional index, sys_ms when); + + // A peer telling us they saved a file we sent them. `when` is the notification's own + // timestamp -- when they saved it -- rather than the timestamp identifying which message it is + // about, which is generally older. + void _on_media_saved( + std::span sender, + const SessionProtos::DataExtractionNotification& note, + sys_ms when); + + // Hands a stored message to Core: the recipient's copy and, unless it is a note to self, the + // copy for our own swarm. Shared by the plain and attachment-carrying sends. + void _dispatch_sends( + int64_t client_id, + const ConversationId& id, + const SessionProtos::Content& content, + const SessionProtos::Content& synced, + sys_ms now, + bool to_self); + + // Runs `work` on the loop thread, logging rather than propagating anything it throws: an + // exception escaping there has no caller to reach and would take the loop with it. + void _dispatch(std::function work); + + void _require_dm(std::string_view op, const ConversationId& id); + void _require_contact(std::string_view op, const ConversationId& id); + void _require_page(std::string_view op, int limit); + void _require_readable(const std::vector& attachments); + + // Everything a send must be able to reject before storing anything: the conversation kind, the + // attachment files, and the message being replied to. Together in one place so that a caller + // gets one answer rather than discovering the next problem after fixing the first, and so that + // every send overload is guarded identically. + // + // On the calling thread, so caller error surfaces at the call site rather than inside the loop + // where a callback form could only log it. + void _require_sendable( + std::string_view op, const ConversationId& id, const OutgoingMessage& msg); + + core::callbacks _core_callbacks(); + void _init(); + + void _on_message_received(core::ReceivedMessage&& msg); + void _on_send_status( + int64_t core_id, + core::MessageSendStatus status, + std::optional swarm_hash); + + // Applies what a config merge changed to the tables that answer queries about it. + // + // Every one of these compares against what we already hold and writes only where they differ, + // rather than trusting the notification to say what moved. That is what makes them + // self-correcting: the config cannot describe how far behind our tables are -- a merge can + // cross several updates at once, and a crash between merging and reconciling leaves them behind + // by an amount nothing recorded -- so anything a previous pass missed is picked up by the next. + void _on_configs_changed(std::span changed); + + // Reconciles every config, whether or not anything reported a change. + // + // A change notification only arrives for state that changes *after* someone is listening, which + // leaves three ways for the tables to fall behind with nothing to announce it: a config merged + // by a version that did not yet know how to reconcile it, a crash between merging and + // reconciling, and a dump restored from an older state. In each the database is behind by an + // amount nothing recorded, and no further notification is owed -- a contact list that has + // stopped changing would stay unreconciled indefinitely. + // + // Safe to run when nothing has changed, because reconciliation compares rather than replays: it + // costs a pass over the configs and writes nothing. + void _reconcile_all(); + + // Records that a note-to-self conversation now exists, by moving its priority off hidden. + // + // For a contact, the presence of an entry in the Contacts config is what says the conversation + // exists, and priority separately says whether it is shown. Note to self has no such entry to + // be present or absent: UserProfile exists from the moment the account does, because it holds + // the account's own name. So priority does both jobs there, and a negative value is the only + // way that config can express "there is no note-to-self conversation" -- which is why a new + // account writes one, and why putting a message in it has to write the value back. + // + // Does nothing if it is already visible, so a pin the user chose is left alone. + void _reveal_note_to_self(const ConversationId& id); + + // Everything the Contacts config says about the people we know. + // + // Each entry projects onto three tables, because it carries three different kinds of fact: who + // someone is goes on `accounts` (which anyone we have merely *seen* also has), the relationship + // goes on `contacts`, and how the conversation with them behaves goes on `conversations`. + // + // Whose name wins is decided by `profile_updated`, not by which source spoke last: the config's + // name is applied only when its stamp is at least as new as the one we already hold, so a + // profile observed on a message that arrived out of order cannot overwrite a newer one. + void _reconcile_contacts(); + + // Re-derives every contact we hold into the Contacts config. + // + // A no-op wherever the config already agrees, because assigning a config field its existing + // value does not dirty it -- so this only writes where the config was actually missing + // something. That is what makes it safe to run before the deletion pass, and why it has to be: + // creating a contact commits the row and updates the config in memory, but the *dump* happens + // later, so a crash in between leaves a row that the config has never heard of. Reconciled + // inward first, that row looks like a contact deleted elsewhere and is destroyed along with its + // history. Derived outward first, it is simply published. + void _sync_all_contacts(); + + // Writes what our tables say about one account back into the Contacts config. + // + // Re-derived from the rows rather than applied alongside each change, so the mapping lives in + // one place and cannot drift from the tables it describes: a caller has to remember to call + // this, but it cannot remember to call it *wrongly*. Idempotent -- assigning a config field + // its existing value does not dirty it -- so it is safe to call whenever a row might have + // moved. + // + // Not for our own account: our profile is UserProfile's, and we are not a contact. + void _sync_contact(const ConversationId& id); + + // As above, for whichever config carries the conversation rather than for a contact + // specifically. The dispatch is the point: something that has changed a conversation should + // not have to know that note to self lives in UserProfile while everybody else lives in + // Contacts. + void _sync_conversation(const ConversationId& id); + + // Read state — the watermark and the marked-unread flag — from ConvoInfoVolatile. + // + // Deliberately without the deletion pass that Contacts has. That config is pruned by age + // rather than by anyone noticing a removal, so an entry absent from it means only that nothing + // has been read in that conversation lately, and treating absence as a deletion would destroy + // conversations for having been quiet. + // + // Runs after whatever might have created a conversation, because read state about one we do not + // have is nothing we can apply and nothing we should create a conversation from. + void _reconcile_convo_volatile(); + + // The other direction, for one conversation and for all of them. + // + // The watermark only ever moves forwards, in both directions. The config does not enforce that + // — it permits a value to be written backwards on purpose — and a conflict between two devices + // at the same seqno resolves by a tie-break that knows nothing about which value is newer, so + // without this a stale device would make read messages unread everywhere. + void _sync_convo_volatile(const ConversationId& id); + void _sync_all_convo_volatile(); + + // Records a delete-before instruction in the config that carries this conversation, so that + // every device deletes the same history rather than only the one the user was looking at. + // + // Never moves the instruction backwards. Two devices clearing at different moments merge to + // one value, and the one that destroys more is the one that was asked for. + void _set_delete_before(const ConversationId& id, sys_ms before); + + // Our own name and picture, and the note-to-self conversation's settings. + // + // This one runs the opposite way round to the others: UserProfile is authoritative and the + // `accounts` row is a projection of it, because the config holds structure the row does not + // model -- a second picture slot, carrying the same image at a fresh URL, which exists so a + // linked device that already has those bytes can skip re-downloading them. Deriving the config + // back from the row would flatten the two slots and destroy another device's reupload. + void _reconcile_user_profile(); + + // `swarm_hash` is recorded against the message only when it names a copy in our own swarm; the + // caller passes nullopt for the copy sent to someone else, whose hash is meaningless here. + void _apply_send_status( + int64_t client_id, + core::MessageSendStatus status, + bool sync, + std::optional swarm_hash); + + // Runs `invoke` against the application's handlers, swallowing and logging anything it throws: + // a broken listener is not something a data model can do anything about. + void _emit(std::function invoke); + + // Hands anything Client says outward to the dispatcher, or runs it here if there is none. + // Everything the application supplied goes through this: the change notifications, and the + // handlers given to individual calls. Only called on the loop, which is what lets the + // dispatcher itself be an ordinary member. + void _dispatch_out(std::function job); + + // Calls `cb` with `args`, on the application's thread, if it gave us one. + template + void _report(Cb& cb, A... args) { + if (!cb) + return; + _dispatch_out([cb, args = std::make_tuple(std::move(args)...)]() mutable { + std::apply(cb, std::move(args)); + }); + } + + // Runs `produce` on the loop and reports what it produced to `cb`, or reports the reason it + // could not. This is what makes "`cb` is invoked exactly once" true: the work is database + // access, which throws on a disk error, and by then the caller's stack is gone -- so the + // callback they gave us is the only way left to tell them. Logging it and returning would + // leave them waiting for an answer that is never coming. + template + void _async(Produce produce, Cb cb) { + loop.call([this, produce = std::move(produce), cb = std::move(cb)]() mutable { + using Result = decltype(produce()); + try { + if constexpr (std::is_void_v) { + produce(); + _report(cb, std::optional{}); + } else + _report(cb, std::optional{}, produce()); + } catch (const std::exception& e) { + log_operation_failure(e); + // Whatever a default value is: the caller is being told not to read it. + if constexpr (std::is_void_v) + _report(cb, std::optional{std::string{e.what()}}); + else + _report(cb, std::optional{std::string{e.what()}}, Result{}); + } + }); + } + + // Out of line so that the logging category does not have to be reachable from this header. + static void log_operation_failure(const std::exception& e); + + void _emit_conversation_added(const ConversationId& id); + void _emit_conversation_removed(const ConversationId& id); + void _emit_lists_replaced(); + void _emit_history_replaced(const ConversationId& id); + // Reports a message, and then reports every message that replies to it. + // + // A reply resolves what it answers on each read, so anything that happens to a message changes + // the replies pointing at it: one arriving makes them resolve, one being deleted changes what + // they show. Cascading here rather than at the ten call sites means none of them can forget + // it, which would show up only as a display that quietly stops matching the database. + void _emit_message(bool added, const ConversationId& id, int64_t message_id); + + // The above without the cascade, which is what the cascade itself uses. + // + // One level is enough, and is why this terminates: reporting B refreshes what A shows for the + // message it replied to, but reporting A cannot change what anything shows for *A*, because a + // message reached through a reply carries the reference to what it answered and never the + // answer itself. So there is no visited set, and two messages claiming to reply to each other + // cannot loop. + void _emit_message_alone(bool added, const ConversationId& id, int64_t message_id); + + // Conversations whose settled state still has to be reported. A conversation is marked here + // rather than reported immediately so that a poll delivering fifty messages to one conversation + // reports it once; `_flush_pending` is scheduled on the loop and runs when the work that + // dirtied them is finished. + std::vector _dirty; + bool _flush_scheduled = false; + void _touch(const ConversationId& id); + void _flush_pending(); + + public: + /// The account state this Client is built on: keys, device group, configs, polling. A + /// Client-based application uses this for everything below the conversation layer. + /// + /// Declared last, which is load-bearing rather than stylistic: members are destroyed in reverse + /// declaration order, so this is destroyed *first*. Core's callbacks and loop jobs capture + /// `this` and reach the members above -- the signal registry, the send-id map, the pending + /// flush -- and Core's polling thread can be part-way through delivering one when a Client is + /// destroyed. `quic::Loop::~Loop()` joins that thread, so once this member is gone no callback + /// can fire, and everything destroyed after it is unreachable by definition. + /// + /// Put anything a Core callback touches *above* this, never below. + core::Core core; + + /// Helper reference to Core's event loop, which is where this class does its work. + oxen::quic::Loop& loop{core.loop()}; + + private: + // Client's own queue on Core's loop, rather than the loop's shared one, so that work deferred + // here is *cancelled* if the Client is destroyed with it still outstanding. Running it instead + // would mean reporting a change to the subscribers of a Client that is going away, against a + // Core whose database is already being torn down. + // + // Declared after `core` -- the one thing that belongs below it -- because a JobQueue needs its + // loop alive in order to stop, so it has to be destroyed while Core still exists. + oxen::quic::JobQueue _jq{loop}; +}; + +} // namespace session::client diff --git a/include/session/client/attachment.hpp b/include/session/client/attachment.hpp new file mode 100644 index 000000000..859c4f092 --- /dev/null +++ b/include/session/client/attachment.hpp @@ -0,0 +1,145 @@ +#pragma once + +#include +#include +#include +#include + +namespace session::client { + +/// Reported through send_message's upload handler when an attachment's file is no longer at the +/// path it was attached from, which can happen between attaching and sending, or to a message +/// being resumed in a later run. Deliberately outside the network layer's code space: nothing +/// about this came from the network, and the message becomes SendState::unsendable rather than +/// merely failed. +constexpr int ATTACHMENT_FILE_MISSING = -20001; + +/// Reported through save_attachment's progress handler when the file arrived but could not be +/// turned back into the file it claims to be: it failed to authenticate, its sender described it +/// wrongly, or it could not be written. Distinct from a transfer failure, and unlike one it is not +/// worth retrying -- the bytes on the file server will be the same bytes next time. +constexpr int ATTACHMENT_UNREADABLE = -20002; + +/// A file to attach to an outgoing message. Attaching costs nothing: the file is read, encrypted +/// and uploaded when the message is sent, not when it is attached, so a caller can hold these +/// against a draft for as long as the user takes to write it. +struct OutgoingAttachment { + /// The file to send. Read at send time, so it must still be there and unchanged by then. + std::filesystem::path path; + + /// MIME type to advertise. Recipients use it to decide how to display the attachment; when + /// unset it is inferred from the filename's extension. + std::optional content_type; + + /// Name to advertise, defaulting to `path`'s filename. Worth setting explicitly when the + /// local file is a temporary whose name means nothing to the recipient. + std::optional filename; + + /// Text shown with the attachment. Distinct from the message body, which is shown as its own + /// message. + std::optional caption; + + /// Marks this as a recorded voice message rather than an ordinary audio file, which clients + /// present differently. + bool voice_message = false; + + /// Pixel dimensions, for visual media. Recipients use them to lay out a placeholder before + /// the file itself has been fetched, so supplying them avoids the layout jumping. + /// + /// TODO: these are the caller's to supply because libsession cannot read them -- deriving them + /// means either an image library or hand-written header parsing of untrusted files. Worth + /// revisiting if libsession takes on an image dependency for other reasons (thumbnailing, say), + /// at which point every client stops needing its own. + std::optional width; + std::optional height; +}; + +/// An attachment on a stored message, in either direction, as reported on `Message::attachments`. +/// +/// The descriptive fields are the same ones `OutgoingAttachment` supplies, and on an incoming +/// attachment they are the sender's claims: nothing here has been checked against the file, which +/// has usually not been fetched at all. In particular `content_type` and `filename` are chosen by +/// whoever sent it, so treat them as display hints rather than as facts about the bytes. +/// What a conversation fetches without being asked. +/// +/// `image_attachments` rather than "images" because a display picture is an image too and is not +/// governed by this: it is always fetched, whatever this says. This is only ever about files sent +/// with a message. +enum class AutoDownload : int { + none = 0, ///< Nothing; every attachment waits to be asked for. + image_attachments = 1, ///< Attachments whose content type is an image. + all = 2, ///< Every attachment. +}; + +/// How an attachment transfer is going. +/// +/// Reported two ways, for the two kinds of transfer: handed directly to whoever called +/// `save_attachment`, and — for a download nobody asked for — broadcast through +/// `callbacks::attachment_progress`, since a background fetch has no caller to hand anything to. +struct AttachmentProgress { + int64_t message_id; + size_t index; + + /// Encrypted bytes so far, and how many are expected. Encrypted rather than the file's own + /// size because that is what is actually being moved and therefore what a proportion should be + /// computed from; the two differ by padding and framing. + /// + /// `total` is 0 until the server has said how big it is, which is not known when a transfer + /// starts — so the first report of any transfer is 0 of 0, meaning "beginning". + int64_t done = 0; + int64_t total = 0; + + /// Unset while it is running, 0 once the file is here and verified, and otherwise the status of + /// whatever went wrong. Exactly one report per transfer carries a value. + std::optional result; +}; + +struct Attachment { + /// Position within the message's attachment list. This is the index `send_message`'s upload + /// handler reports progress against, and what `save_attachment` takes. + size_t index; + + std::optional content_type; + std::optional filename; + std::optional caption; + + /// A recorded voice message rather than an ordinary audio file, which clients present + /// differently. + bool voice_message = false; + + std::optional width; + std::optional height; + + /// The file's size in bytes, before encryption -- what the file server holds is larger, since + /// it carries the stream's per-chunk overhead and the padding that hides the true length. + /// + /// Unset on an outgoing attachment that has not been uploaded yet. On an incoming one this is + /// the sender's claim: for a legacy-encrypted attachment it is load-bearing, being what the + /// padding is trimmed by, and a wrong value there shows up as a corrupt file. + std::optional size; + + /// Whether the file is on the file server: for an outgoing attachment, that its upload + /// finished, which is how a partly-uploaded message reports which of its files got through. + /// Always true for an incoming attachment, which is where it came from. + /// + /// Where the file is *locally* is deliberately not here. A local path is an argument to + /// sending or saving, not a property of the attachment: the application chose it and knows it, + /// and anything recorded here would go stale as soon as the file was moved. + bool uploaded = false; + + /// When the *recipient* of this message last saved this attachment -- us, on an incoming one, + /// and the other party on one we sent. The same fact from either end, so it does not have to + /// be read differently depending on `Message::outgoing`. + /// + /// What it is for is knowing whether offering "save" again is pointless, and — on a message we + /// sent — whether the file reached a person rather than merely a file server. + /// + /// Unset means **not known to have been saved**, which is not the same as not saved, and must + /// not be shown as though it were. On an outgoing attachment it depends entirely on the other + /// end volunteering a notification: a client that sends none, or one too old to say which + /// message it means, leaves this unset no matter how many times its user saved the file. A + /// sender reading an absent value as "they never got it" would be wrong. + std::optional saved_at; +}; + +} // namespace session::client diff --git a/include/session/client/callbacks.hpp b/include/session/client/callbacks.hpp new file mode 100644 index 000000000..4a6316d4a --- /dev/null +++ b/include/session/client/callbacks.hpp @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::client { + +/// Notifications of everything the conversation layer changes, so that an application never has to +/// ask. A caller sets the handlers it cares about and leaves the rest empty; an unset handler is +/// simply not called. +/// +/// Every handler is given the new state outright rather than an identifier to go and fetch, which +/// is what makes a display bindable without reading anything back. It also makes applying one +/// twice harmless, which in turn makes startup race-free — see the Client constructor. +/// +/// Handed to Client at construction and fixed thereafter, exactly as core::callbacks is. There is +/// deliberately no way to register a second set: a process that wants to fan these out to somewhere +/// else — a notification daemon, a log — does that fanning out itself, which it has to anyway once +/// the other end is a separate process. +/// +/// **Handlers run on Core's event loop**, not the caller's thread. A handler must not block and +/// must not throw (an escaping exception is caught and logged, and the change is not redelivered). +/// What a handler receives is its own: it was read for this delivery and nothing else holds it, so +/// a UI moves it into its own queue and wakes its render thread. The signatures say which is +/// which — what is given is taken by rvalue reference, what is lent by `const&`. The two progress +/// handlers are the ones that lend, because they report repeatedly against one captured id and a +/// handler that moved from it would empty what the next report needs. +/// +/// A handler may declare such a parameter as `T&&`, `const T&` or `T`, whichever suits it: only the +/// last constructs anything, and a handler that just reads pays nothing. (`T&` is the one form +/// that will not bind.) The `&&` is not perfect forwarding despite the spelling — `std::function` +/// is not a template on its argument — it is a promise by the caller that the object is spent +/// afterwards. +/// +/// The conversation list an application maintains from these is expected to be *complete*: ordering +/// is a comparison against every other conversation, so a partial list cannot be sorted. Showing +/// only part of it is fine, holding only part of it is not. +struct callbacks { + /// A conversation now exists that did not before. + std::function conversation_added; + + /// A conversation's contents changed: a new or edited message, a name, an unread count, its + /// last activity. Fired once with the conversation's settled state rather than once per + /// underlying change, so a poll that delivers fifty messages to one conversation fires this + /// once. + std::function conversation_updated; + + /// A conversation is gone and should be dropped from the list. + std::function conversation_removed; + + /// Priorities changed — a pin, unpin, hide or unhide — carrying the whole list in its new + /// order. A replacement rather than a description of what moved, because one config update + /// from another device can repin, reveal and hide arbitrarily many conversations at once, and + /// because a replacement cannot leave the application subtly out of step the way a missed + /// delta would. + std::function&&)> conversation_list_replaced; + + /// The message requests changed, carrying the whole list of them, for the same reasons and with + /// the same guarantees as the above. + /// + /// The two lists are disjoint and a conversation moves between them, so approving one fires + /// both: it left the requests and joined the conversations. `conversation_added` and the rest + /// are shared between them — a request is a conversation in every respect except which list it + /// belongs to — and `Conversation::request` is what says which one a given handler is about. + std::function&&)> request_list_replaced; + + /// A message was added, whether received or sent from here. + std::function message_added; + + /// An existing message changed — currently only its send state. + std::function message_updated; + + /// Messages were deleted from a conversation, and anything displaying its history should read + /// it again. + /// + /// Unlike `conversation_list_replaced` this carries only the conversation and not the messages + /// themselves: a history is unbounded, and an application showing one page of it has no use for + /// the rest. What deletes messages is a delete-before instruction, which can take any number + /// of them at once and is not otherwise describable as a sequence of removals. + std::function history_replaced; + + /// An attachment is being fetched that nobody asked for — see `Conversation::auto_download`. + /// + /// Only for background fetches. A `save_attachment` reports to the caller that started it, and + /// does not come through here: this handler means "something is happening you did not ask for", + /// which is exactly what a display has no other way of learning. + /// + /// The first report of a transfer arrives when it is *started*, before anything has been sent + /// to a server, carrying 0 of 0 — so a row can show that a fetch is beginning rather than + /// appearing to do nothing until the first bytes land. Exactly one report carries a `result`. + /// + /// Reports are rate limited (see `set_dispatch_interval`) to keep the cost off the + /// application's thread. That is all the limiting is for: how often a spinner turns is the + /// application's own business, and it should not be reading motion into the arrival of these. + std::function attachment_progress; + + /// The same, for a display picture, which belongs to a conversation rather than to a message — + /// so it carries no message or index and gets its own handler rather than a struct with two + /// fields that are never filled in. + /// + /// Display pictures are always fetched, with no setting to turn that off, so this fires for + /// every one that is not already cached. + std::function result)> + display_picture_progress; +}; + +} // namespace session::client diff --git a/include/session/client/conversation.hpp b/include/session/client/conversation.hpp new file mode 100644 index 000000000..05b27dc70 --- /dev/null +++ b/include/session/client/conversation.hpp @@ -0,0 +1,588 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::client { + +class Client; + +/// Enough of the most recent message to draw a conversation-list row, without reading the message. +/// +/// A summary rather than the message itself, because a list row is not a message view: it wants a +/// line of text and a hint of what else is there, and handing over whole `Message`s to fill in a +/// column of one-liners costs a read per row for detail nobody draws. +/// +/// The fields are independent, not alternatives — a message can carry a body *and* attachments, and +/// a row showing "look at this 📎2" needs both — so there is deliberately no single "kind" enum +/// here to switch on. +/// +/// **Expect this to gain fields.** Message kinds that are not yet modelled — call metadata, a +/// typing indicator — will describe themselves by adding to this struct, and adding a field is the +/// intended way to grow it: designated initialisers and defaults keep that source-compatible. Two +/// consequences for a caller: +/// +/// - Do not treat the fields you know as exhaustive. "Empty body and no attachments" means *this +/// build* has nothing to say about the message, which today implies there is nothing to show, but +/// will later be how an unhandled kind looks. A row that draws nothing in that case degrades +/// gracefully; one that asserts on it does not. +/// - Do not persist a preview, and do not compare two for equality to decide whether to redraw. It +/// is a description of a message as this version understands it, and both of those turn a gained +/// field into a stale cache or a missed repaint. +struct MessagePreview { + /// The message body, whole and untruncated. Empty when the message carries no text, which is + /// normal: a message can be attachments alone. Truncation is left to the caller, since how + /// much fits is a property of the row it is being drawn into and not of the message. + std::string body; + + /// The name of each attachment, in the order the sender listed them, and empty for the whole + /// message when it has none. + /// + /// One entry per attachment, so `filenames.size()` *is* the attachment count and the entries + /// line up with `Attachment::index`. An entry is an empty string when that attachment carries + /// no name — the sender simply omits the field — so the names cannot be counted as a proxy for + /// the attachments, and a row drawing them needs a fallback for the empty ones. + /// + /// Names but not types or sizes: a single-attachment row saying "invoice.pdf" is worth far more + /// than one saying "1 file", while the rest of what an attachment is remains + /// `Conversation::messages()`'s business. + std::vector filenames; + + /// True if we sent it, for a row that prefixes "You: ". + bool outgoing = false; + + /// True if the attachments are a voice message, which a row usually names rather than counts. + bool voice_message = false; + + /// True if there is at least one attachment and every one of them is an image, so a row can say + /// "3 images" where it would otherwise say "3 files". False when there are none at all. + bool all_images = false; +}; + +/// A conversation: what it looks like, and everything you can do to it. +/// +/// **The values are a snapshot; the operations are not.** The fields were read when this was +/// handed to you, and the conversation may have moved on since — another device can rename, pin or +/// delete it at any moment. The operations do not read the fields; they act through `id`, which +/// stays meaningful however old this object is. So a stale object still does the right thing when +/// acted on, and only what it *says* can be out of date. To see current values, ask for it again. +/// +/// There is no way to obtain one that was never populated: `Client::conversation()` fetches one, +/// `conversations()` lists them, `open_dm()` makes one, and the callbacks hand them over — each of +/// those has read the database. So an empty `display_name` means nobody knows their name, never +/// that this object has not looked yet. +/// +/// Every operation comes in two forms: one taking a handler, which returns immediately and reports +/// later, and one taking `await`, which blocks the calling thread and throws instead of reporting. +/// See `await_t` for which to use where. +/// +/// Cheap to copy and safe to hold, including on another thread — but it names a Client and must not +/// outlive it. +class Conversation { + public: + ConversationId id; + + /// Best known display name, or empty if none is known yet. + /// + /// For a DM this is the nickname we gave them if there is one, and otherwise the name they gave + /// themselves — learned either from the `LokiProfile` on a message or from the Contacts config, + /// whichever is the fresher. Empty is normal and expected: an account we have seen but know + /// nothing else about has no name, and the caller decides how to render that. + std::string display_name; + + /// The most recent message that still says something, summarised for a list row. + /// + /// Unset when there is nothing to preview — the conversation has no messages, or every one it + /// has was deleted. So an *empty* `body` on a preview that is set means the message carries no + /// text rather than that there is no message, which is the distinction a bare string could not + /// make and the reason this is an optional. + /// + /// Deleted messages are skipped rather than reported: taking the newest one regardless would + /// blank the row whenever the last thing said was later deleted, and what a list wants there is + /// the last thing that was actually said. A caller that wants the deleted message itself is + /// asking about history, and should read it — `messages()` with `include_deleted`. + std::optional last_preview; + + /// Timestamp of the most recent message, or the conversation's creation time if it has none. + sys_ms last_activity; + + /// Count of incoming messages newer than the read watermark. + int unread = 0; + + /// Deliberately marked unread — "come back to this" — rather than having unread messages. + /// + /// Independent of `unread`: it survives having read everything, which is what it is for, and + /// reading the conversation is what clears it. Synced, so it arrives from other devices too. + bool marked_unread = false; + + /// Pinning, numerically identical to the value the Contacts and UserGroups configs sync: 0 is + /// unpinned, a positive value is pinned with higher values first, and a negative value is + /// hidden. + /// + /// Hidden is a statement about the *list*: `conversations()` omits them, so this is never + /// negative on anything reached that way. Naming a conversation still reaches one, since that + /// is not the same as asking what the list contains — and it is how a hidden conversation is + /// reached at all. + /// + /// Conversations sort by priority before recency, and equal priorities form a block that sorts + /// among itself by last_activity — so pinning several conversations together keeps them at the + /// top while still letting the most recently active of them lead. + int priority = 0; + + /// When to notify about this conversation. `defaulted` means the application's own default + /// applies; `mentions_only` is a group notion and reads as `all` on a DM. + /// + /// Independent of `mute_until`, and overridden by it while that is in the future. + config::notify_mode notifications = config::notify_mode::defaulted; + + /// Notifications are suppressed until this moment, whatever `notifications` says. The epoch + /// means not muted — which is not the same as `notify_mode::disabled`, since a mute expires and + /// a UI shows it as "muted until …" rather than as off. + std::chrono::sys_seconds mute_until{}; + + /// Disappearing messages. The timer is meaningless when the mode is `none`, and the mode is + /// what a UI has to say out loud: after-send and after-read are a different promise to the + /// person you are talking to, not two spellings of the same one. + config::expiration_mode exp_mode = config::expiration_mode::none; + std::chrono::seconds exp_timer{0}; + + /// Where this conversation's picture is, if we know of one: the url it was uploaded to and the + /// key that decrypts it. Empty url means there is no picture, or none we have been told about. + /// + /// Enough to know whether there is one and whether it has changed — the url is what changes + /// when somebody updates their picture — without fetching anything. `Client::profile_picture` + /// is what turns it into bytes. + /// + /// On the base rather than on `DM` because every kind of conversation can have one; only a DM's + /// is wired up today, so this is empty for a group or community until that lands. + config::profile_pic picture; + + /// What to fetch from this conversation without being asked. + /// + /// Unset means nobody has been asked yet, which is a different thing from having been asked and + /// said no: Session asks within a conversation the first time and remembers the answer, and a + /// client can only do that if it can tell the two apart. Nothing is fetched while it is unset. + /// + /// Device-local and never synced. Auto-downloading everything on a desktop and nothing on a + /// phone is the case this exists for. + std::optional auto_download; + + /// The display name if known, otherwise the conversation's string id — a reasonable default + /// for a caller that has no better fallback of its own. + std::string name_or_id() const { return display_name.empty() ? id.to_string() : display_name; } + + Conversation(Client& client, ConversationId id) : id{std::move(id)}, _client{&client} {} + + // -- Reading ------------------------------------------------------------------------------ + + /// A window of history, newest first. Pass the `cursor()` of the last message of a page as + /// `before` to fetch the next (older) page; the message at the cursor is not repeated. + /// + /// Deleted messages are left out unless `include_deleted` is passed. Out by default because a + /// deleted message carries no body, so a caller that has not thought about `Message::deleted` + /// would draw an empty row that looks like a bug. A client that wants to show "message + /// deleted" asks for them, and gets them in place, in order. + /// + /// Filtered in the query rather than left to the caller, because the alternative breaks paging: + /// a page of 50 that is mostly deleted would hand back a handful of rows with nothing to say + /// that another page is warranted. + /// + /// @throws std::invalid_argument if `limit` is not positive. In particular there is no value + /// meaning "all of it" -- SQLite reads `LIMIT -1` that way, this does not. History grows + /// without bound, so reading all of it means paging with `before` until a page comes back + /// short. + void messages(failable_function)> cb) const; + void messages(int limit, failable_function)> cb) const; + void messages( + int limit, + std::optional before, + failable_function)> cb) const; + void messages( + int limit, + std::optional before, + bool include_deleted, + failable_function)> cb) const; + std::vector messages(await_t) const; + std::vector messages(int limit, await_t) const; + std::vector messages(int limit, std::optional before, await_t) const; + std::vector messages( + int limit, std::optional before, bool include_deleted, await_t) const; + + /// Removes every deleted message's leftover row from this conversation, and says how many went. + /// + /// A deletion leaves a row behind deliberately — see `Client::delete_message` — and this is + /// what finally removes them, for a client that would rather not accumulate them or show them. + /// + /// **This can bring a message back.** A message deleted only here is still in our swarm, and + /// that row's swarm hash is the only thing that recognises it if it is delivered again — which + /// a storage server makes likely rather than hypothetical, since it stops honouring a + /// `last_hash` once that has expired and answers the next poll with the whole retention window. + /// Removing the row removes the memory of it, and the message returns looking new. + /// + /// The alternative — deleting our swarm copy too, so there is nothing to come back — is worse + /// and is deliberately not done: that copy is what our *other devices* poll, so destroying it + /// would turn "delete for me" into "delete on every device I own", including devices that have + /// never seen the message and then never would. + /// + /// Messages we sent and deleted everywhere are not exposed to this: their swarm copy is already + /// gone, so there is nothing left to return. + void purge_deleted(failable_function cb); + size_t purge_deleted(await_t); + + // -- Read state --------------------------------------------------------------------------- + + /// Marks incoming messages up to and including `up_to` as read, moving the unread watermark + /// forward. Passing nullopt marks everything currently stored as read — which is not the same + /// as parking the watermark at infinity: a message that arrives afterwards is still unread, + /// even if its timestamp is older than the one we just read to. + /// + /// Never moves the watermark backwards. Clears `marked_unread`, since reading the conversation + /// is the thing that was being asked for. + void mark_read(failable_function cb); + void mark_read(std::optional up_to, failable_function cb); + void mark_read(await_t); + void mark_read(std::optional up_to, await_t); + + /// Marks the conversation unread, or clears that — the deliberate "I want to come back to this" + /// rather than a count of messages. Synced, so it follows you between devices. + void set_marked_unread(bool unread, failable_function cb); + void set_marked_unread(bool unread, await_t); + + // -- Settings ----------------------------------------------------------------------------- + + /// Sets pinning: 0 unpinned, positive pinned with higher values first, negative hidden. + /// + /// Reported to subscribers as `conversation_list_replaced`, not as an update to the one + /// conversation, because hiding removes a conversation from the list and unhiding returns it — + /// so what changed is the list, not a row in it. + void set_priority(int priority, failable_function cb); + void set_priority(int priority, await_t); + + /// Sets when to notify, and until when to stay quiet. Separate calls because they are separate + /// decisions: muting until Monday does not change what you want notified after Monday, and a UI + /// that offered them together would have to invent an answer for the other one. + /// + /// A `mute_until` in the past, or the epoch, is not muted. + void set_notifications(config::notify_mode mode, failable_function cb); + void set_notifications(config::notify_mode mode, await_t); + void set_mute_until(std::chrono::sys_seconds until, failable_function cb); + void set_mute_until(std::chrono::sys_seconds until, await_t); + + /// Sets the disappearing-message mode and timer together, because neither means anything + /// alone: a timer with no mode does not expire, and a mode with no timer has nothing to count. + /// `expiration_mode::none` clears the timer whatever is passed with it. + void set_expiry( + config::expiration_mode mode, std::chrono::seconds timer, failable_function cb); + void set_expiry(config::expiration_mode mode, std::chrono::seconds timer, await_t); + + /// Sets what this conversation fetches without being asked. + /// + /// Device-local: unlike every other setting here, this one does not follow the account, because + /// it is about what this machine's bandwidth and disk are for. Auto-downloading everything on + /// a desktop and nothing on a phone is the case it exists for. + /// + /// Calling this is also what records that the question has been asked at all, so a client that + /// prompts on first use should call it with whatever answer it was given — `none` included, + /// since that is not the same as never having asked, and only the latter should prompt again. + void set_auto_download(AutoDownload mode, failable_function cb); + void set_auto_download(AutoDownload mode, await_t); + + // -- Sending ------------------------------------------------------------------------------ + + /// Sends a message, storing it immediately and dispatching it via Core. Yields the Client + /// message id of the stored row, which is what subsequent `message_updated` handlers carry as + /// delivery progresses. + /// + /// See `OutgoingMessage` for what a message can carry. With no attachments this stores and + /// dispatches in one step; with any, sending becomes two stages, described below. + /// + /// Sending a message carrying files turns sending into two stages: each attachment is + /// encrypted and uploaded to the file server, and only then is the message — now able to name + /// where those files live — dispatched to the swarms. + /// + /// Returns as soon as the row is stored, as the plain overload does, so the message is + /// displayable immediately; it sits in `SendState::uploading` until the uploads finish and + /// moves on to the ordinary send states from there. An upload that fails fails the message, + /// leaving it in `SendState::failed` with nothing sent. + /// + /// `on_upload` follows one attachment, identified by `index`, its position in `attachments`: + /// + /// - `result` unset — under way. `sent`/`total` are encrypted bytes. The first such report + /// for an attachment is always 0/0 and means it has started rather than that it has sent + /// nothing: it comes before the transfer is established, which is what tells an attachment + /// being worked on from one still waiting its turn. The size is not known until then, so a + /// progress bar is sized from the first report carrying a non-zero total. + /// - `result == 0` — done, and the file server gave us an id for it. Note this is the only + /// thing that means done: `sent == total` merely means the last byte was acknowledged, and + /// the server's decision to accept the file arrives after that. + /// - anything else — that attachment failed, with the file server's status code or one of the + /// network layer's own negative codes. The message fails as a whole, but this is reported + /// per attachment, so a list of them can mark the one that broke rather than all of them. + /// + /// Reports for one attachment arrive in order — progress, then exactly one result — but + /// reports for different attachments may interleave, and a failure does not stop the others + /// being reported. An attachment that never produces a result was not attempted, or had not + /// finished when the message failed; either way that is not something to read as success. + /// + /// It is taken here rather than through `callbacks` because it belongs to this call: those + /// handlers report what the network did to us, whereas this reports how something we asked for + /// is going, and a caller passing it here can bind whatever it wants to update without matching + /// an id back to it. + /// + /// Progress fires as often as the transfer reports, which on a fast upload is often; coalescing + /// is the caller's to do, since dropping intermediate values loses nothing. Like the + /// `callbacks` handlers it runs on Core's event loop, so it must not block or throw — and note + /// it can fire before this function returns, so it must not depend on anything the caller sets + /// up afterwards. + /// + /// The files are read at this point, not when they were attached, so they must still exist. + /// Nothing about them is stored: what persists is the message, whose content names the uploaded + /// copies. Re-sending a failed message therefore uploads again. + /// + /// @throws std::invalid_argument if any attachment's file cannot be opened or exceeds the file + /// server's limit, or if `reply_to` names a message that does not exist or belongs to another + /// conversation; thrown on the calling thread, before anything is stored or dispatched. + using upload_progress = std::function result)>; + void send_message( + OutgoingMessage msg, + upload_progress on_upload, + failable_function cb); + void send_message(OutgoingMessage msg, failable_function cb); + int64_t send_message(OutgoingMessage msg, upload_progress on_upload, await_t); + int64_t send_message(OutgoingMessage msg, await_t); + + // -- Destroying --------------------------------------------------------------------------- + // + // Named for what a user is choosing rather than for what happens underneath — "delete + // conversation" and "delete contact" differ by what they leave behind, not by how they delete. + // They are composed here, and not left to a UI to assemble out of smaller pieces, because every + // Session client has to compose them the same way: which fields a deletion resets, and what it + // tells other devices, is part of what the operation *is*, and three clients each deriving it + // separately is three chances to diverge. + // + // The deletion is *synced as an instruction*, not as a local act. Clearing records the moment + // it was cleared, so every device deletes what it holds from before then — including messages + // this one never had, and messages that arrive afterwards but are older than the instruction. + // A device that was offline for the whole thing catches up when it merges, rather than keeping + // a history its owner told it to destroy. + + /// Deletes the messages, keeping the conversation itself and everything that describes it — the + /// contact, the nickname, the pin. + void clear_messages(failable_function cb); + void clear_messages(await_t); + + /// Removes the conversation from the list without forgetting who it is with: the contact entry + /// stays, so a nickname and an approval survive, and a new message brings the conversation + /// back. + /// + /// `keep_messages` distinguishes the two things a UI calls this for. Deleting a conversation + /// destroys its history; hiding one — which is what "Hide" on Note to Self does — leaves the + /// history to come back with it. + void delete_conversation(failable_function cb); + void delete_conversation(bool keep_messages, failable_function cb); + void delete_conversation(await_t); + void delete_conversation(bool keep_messages, await_t); + + protected: + /// Never null, and not owned: the Client this came from, which must outlive it. + Client* _client; +}; + +/// A one-to-one conversation with another account. +class DM : public Conversation { + public: + explicit DM(Conversation base) : Conversation{std::move(base)} {} + + /// True while this is a message request rather than a conversation: someone we have never + /// written to has written to us. + /// + /// Approval is not something anyone sets — it is recorded by messages flowing. Writing to + /// someone approves them, so answering a request is what accepts it, and there is no way back: + /// what un-requests a conversation is deleting the contact, not clearing a flag. + /// + /// Requests are conversations in every other respect — they have history, an unread count and a + /// name — which is why this is a property of one rather than a kind of its own. What differs + /// is which list it belongs to: `conversations()` omits them and `message_requests()` returns + /// only them, so this is never true on anything the former returned. + /// + /// Never true for note to self. + bool request = false; + + /// The mirror of `request`: we have written to someone who has never written back, so they have + /// us in *their* message requests and have not answered. + /// + /// Set from the same evidence, read the other way round — nobody sends anything to say they + /// accepted, so the only thing that clears this is a message from them. A display showing + /// "waiting for them to accept" wants this; the conversation is otherwise ordinary and is in + /// `conversations()` like any other, because it is one we chose to start. + /// + /// The two are mutually exclusive: their having written to us is exactly what makes this false + /// and what can make `request` true. + bool awaiting_approval = false; + + /// True for the conversation with our own account — Session's "Note to Self". It is an + /// ordinary DM rather than a kind of its own, so this is the only thing distinguishing it, and + /// it is reported here so that displaying it differently does not require the caller to know + /// our session ID or to compare it themselves. What to call it is still the caller's + /// decision: `display_name` is whatever our own profile says, not a localised label. + bool note_to_self = false; + + /// Whether their messages are refused on arrival. The other half of `set_blocked`: a toggle + /// whose state cannot be read is not a toggle, and a UI needs this to draw it as on, to offer + /// "unblock" rather than "block", and to stop offering to block somebody already blocked. + bool blocked = false; + + /// The two halves `display_name` merges, for a screen that *edits* the name rather than + /// rendering it. + /// + /// `name` is what they call themselves — from the `LokiProfile` on a message, or from the + /// Contacts config, whichever is fresher. `nickname` is what we call them, which is ours and + /// syncs to our other devices. A list row wants the merge and should keep using + /// `display_name`; only a screen offering to change the nickname needs to show "they call + /// themselves X, you call them Y", and it cannot when only the resolved answer arrives. + /// + /// Either may be empty: an account we have seen but know nothing else about has no name, and + /// most contacts have no nickname. + std::string name; + std::string nickname; + + /// Sets or clears the name we have given them, which is ours rather than theirs and follows us + /// between devices. An empty nickname removes it, so `display_name` falls back to `name`. + void set_nickname(std::string_view nickname, failable_function cb); + void set_nickname(std::string_view nickname, await_t); + + /// Blocks or unblocks the account. Blocking is synced, so it takes effect on every device, and + /// what they send is refused on arrival rather than hidden when drawing. + /// + /// Blocking someone we hold no contact entry for creates one, since being blocked is a fact + /// about a relationship and there is nowhere else to record it. It does not approve them, and + /// unblocking does not undo anything else the block implied. + void set_blocked(bool blocked, failable_function cb); + void set_blocked(bool blocked, await_t); + + /// Deletes the contact along with the conversation and its history, everywhere. + /// + /// This is the strong one: the entry goes from the Contacts config, so every device drops the + /// conversation rather than merely hiding it, and what the entry held — the nickname, the + /// approval in both directions, the block — goes with it. Deleting a blocked contact therefore + /// unblocks them; the block lived in the entry that was removed. + /// + /// What survives is the account itself, because we may still see them in a group or a + /// community and need their name to render it. A message from them afterwards arrives as a + /// message request, exactly as one from a stranger would. + /// + /// Takes no `keep_messages`: there would be no conversation left to keep them in. + /// + /// @throws std::invalid_argument for our own account, which is not a contact of ours. + void delete_contact(failable_function cb); + void delete_contact(await_t); +}; + +/// A closed group. Nothing of its own yet: what distinguishes a group — its membership, who +/// administers it, whether we have accepted the invitation — arrives with UserGroups. +class Group : public Conversation { + public: + explicit Group(Conversation base) : Conversation{std::move(base)} {} +}; + +/// A community (open group) room. As above: moderation and room metadata arrive with the config +/// that carries them. +class Community : public Conversation { + public: + explicit Community(Conversation base) : Conversation{std::move(base)} {} +}; + +/// A conversation of whichever kind it turned out to be. +/// +/// The kind is a variant rather than a pointer so that a list of these is one contiguous block with +/// no allocation per row — a conversation list is re-read on every redraw, which makes it the one +/// place where that matters. Reading a field common to every kind costs a compare against the +/// variant's discriminant and nothing else: each alternative holds its `Conversation` at the same +/// offset, so the compiler folds the dispatch away entirely. +/// +/// **Keep `Conversation` the first base of any kind added here.** A second base ahead of it moves +/// the shared fields to a different offset in that one alternative, and the accessors below quietly +/// become a runtime selection instead of nothing at all. +class AnyConversation { + public: + std::variant kind; + + AnyConversation(DM c) : kind{std::move(c)} {} + AnyConversation(Group c) : kind{std::move(c)} {} + AnyConversation(Community c) : kind{std::move(c)} {} + + /// Everything every kind has, values and operations alike. The shorthands below reach through + /// this; it is here for anything they do not cover. + Conversation& base() { + return std::visit([](auto& c) -> Conversation& { return c; }, kind); + } + const Conversation& base() const { + return std::visit([](const auto& c) -> const Conversation& { return c; }, kind); + } + + const ConversationId& id() const { return base().id; } + const std::string& display_name() const { return base().display_name; } + const std::optional& last_preview() const { return base().last_preview; } + sys_ms last_activity() const { return base().last_activity; } + int unread() const { return base().unread; } + bool marked_unread() const { return base().marked_unread; } + int priority() const { return base().priority; } + config::notify_mode notifications() const { return base().notifications; } + std::chrono::sys_seconds mute_until() const { return base().mute_until; } + config::expiration_mode exp_mode() const { return base().exp_mode; } + std::chrono::seconds exp_timer() const { return base().exp_timer; } + const config::profile_pic& picture() const { return base().picture; } + std::optional auto_download() const { return base().auto_download; } + std::string name_or_id() const { return base().name_or_id(); } + + /// The kind, or nullptr if it is not that one. This is how a caller asks a question only one + /// kind can answer — whether a conversation is a message request, who administers a group — + /// and the nullptr is what stops that question being asked of a kind it means nothing to. + DM* dm() { return std::get_if(&kind); } + Group* group() { return std::get_if(&kind); } + Community* community() { return std::get_if(&kind); } + const DM* dm() const { return std::get_if(&kind); } + const Group* group() const { return std::get_if(&kind); } + const Community* community() const { return std::get_if(&kind); } + +// Forwarded rather than reimplemented, so that an overload added to Conversation is reachable here +// without anything being added below. +#define SESSION_CONVO_FORWARD(name) \ + template \ + decltype(auto) name(A&&... a) { \ + return base().name(std::forward(a)...); \ + } \ + template \ + decltype(auto) name(A&&... a) const { \ + return base().name(std::forward(a)...); \ + } + + SESSION_CONVO_FORWARD(messages) + SESSION_CONVO_FORWARD(mark_read) + SESSION_CONVO_FORWARD(set_marked_unread) + SESSION_CONVO_FORWARD(set_priority) + SESSION_CONVO_FORWARD(set_notifications) + SESSION_CONVO_FORWARD(set_mute_until) + SESSION_CONVO_FORWARD(set_expiry) + SESSION_CONVO_FORWARD(set_auto_download) + SESSION_CONVO_FORWARD(send_message) + SESSION_CONVO_FORWARD(clear_messages) + SESSION_CONVO_FORWARD(delete_conversation) + SESSION_CONVO_FORWARD(purge_deleted) + +#undef SESSION_CONVO_FORWARD +}; + +} // namespace session::client diff --git a/include/session/client/conversation_id.hpp b/include/session/client/conversation_id.hpp new file mode 100644 index 000000000..c89a02a44 --- /dev/null +++ b/include/session/client/conversation_id.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace session::client { + +/// Opaque identifier for a conversation. +/// +/// A conversation is keyed differently depending on its kind — a DM by the remote session ID, a +/// closed group by the group ID, a community by server URL plus room token — so this deliberately +/// does not decay to any one of those. Extracting the underlying value requires naming the kind +/// you expect (session_id(), group_id(), community()), which throws if the conversation is not of +/// that kind. +/// +/// The string form from to_string() round-trips through parse() and is stable across releases: it +/// is what gets stored in the database and what an application may persist (a CLI argument, a +/// bookmark). It is *not* guaranteed to be a good display string. +class ConversationId { + public: + enum class Type : int { + dm = 0, ///< One-to-one conversation, keyed on the remote 0x05 session ID + group = 1, ///< Closed group, keyed on the 0x03 group ID + community = 2, ///< Community (open group), keyed on server base URL + room token + }; + + /// Constructs a DM conversation id from a 33-byte 0x05-prefixed session ID. + /// @throws std::invalid_argument if the first byte is not 0x05. + static ConversationId dm(std::span session_id); + + /// Constructs a closed group conversation id from a 33-byte 0x03-prefixed group ID. + /// @throws std::invalid_argument if the first byte is not 0x03. + static ConversationId group(std::span group_id); + + /// Constructs a community conversation id. `base_url` is normalised (lowercased, trailing + /// slash removed) and `room` is lowercased, so that the same community reached by differently + /// spelled URLs compares equal. + /// @throws std::invalid_argument if either argument is empty, or `room` contains a '/'. + static ConversationId community(std::string_view base_url, std::string_view room); + + /// Parses the form produced by to_string(). + /// @throws std::invalid_argument if the string is not a valid conversation id. + static ConversationId parse(std::string_view s); + + Type type() const { return _type; } + + /// The remote 0x05-prefixed session ID. @throws std::logic_error unless type() == dm. + std::span session_id() const; + + /// The 0x03-prefixed group ID. @throws std::logic_error unless type() == group. + std::span group_id() const; + + /// The community server base URL and room token. + /// @throws std::logic_error unless type() == community. + std::pair community() const; + + /// The stable, round-trippable string form: + /// - dm: the 66-character hex session ID, e.g. "05abc…" + /// - group: the 66-character hex group ID, e.g. "03abc…" + /// - community: "community:/" + /// + /// The dm and group forms are exactly the session/group IDs a user would paste, which is what + /// makes them usable directly as CLI arguments. + std::string to_string() const; + + std::strong_ordering operator<=>(const ConversationId&) const = default; + bool operator==(const ConversationId&) const = default; + + private: + ConversationId(Type t, std::string key) : _type{t}, _key{std::move(key)} {} + + static ConversationId _from_prefixed( + std::span id, std::byte want, Type type); + + Type _type; + // For dm/group: the 33 raw ID bytes. For community: "/", already normalised. + std::string _key; +}; + +} // namespace session::client + +namespace std { +template <> +struct hash { + size_t operator()(const session::client::ConversationId& c) const { + return hash{}(c.to_string()); + } +}; +} // namespace std diff --git a/include/session/client/handler.hpp b/include/session/client/handler.hpp new file mode 100644 index 000000000..cdad8cd12 --- /dev/null +++ b/include/session/client/handler.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include + +namespace session::client { + +/// Passed where a handler would go, to say "wait until this is done and give me the answer" +/// instead. +/// +/// Every asynchronous method has a blocking twin taking one of these. The work is the same and +/// happens in the same place -- on Core's loop -- so the only difference is who waits: the twin +/// blocks the calling thread until the answer is ready, and *throws* what the handler form would +/// have reported through its `error` argument. +/// +/// A tag rather than a second class, and rather than an overload with no handler at all, because +/// the point is that it be visible where it is used. Blocking is a decision about the calling +/// thread, so it belongs at the call site: a render loop must not do it, and a review can grep for +/// it, neither of which works when the choice was made wherever the variable was declared. +/// +/// Calling one from a Client handler is safe rather than a deadlock -- the loop runs the work +/// inline when it is already the current thread -- but it is still waiting, and anything else the +/// loop owes is waiting behind it. +struct await_t {}; +inline constexpr await_t await{}; + +/// Runs a job on the application's own thread. +/// +/// Client does its work on Core's event loop and, given one of these, hands everything it has to +/// say to the application over to it rather than calling from that loop. An application with a +/// loop of its own -- Qt, node, GTK -- supplies the one-line transfer it already has, and is then +/// free to touch its own state in every handler without marshalling in each one. +/// +/// It must not run the job inline, which would defeat the point, and it must preserve the order +/// jobs are given to it: what Client reports is a sequence, and delivering it out of order shows +/// an older state on top of a newer one. +/// +/// Everything Client calls outward goes through it -- the change notifications and the handlers +/// passed to individual calls alike -- so an application never has to reason about which of its +/// handlers arrive on which thread. Without one, everything runs on Core's loop, which is what a +/// program with no loop of its own wants. +using dispatcher = std::function)>; + +namespace detail { + template + struct failable_function; + + template + struct failable_function { + using type = std::function error, A...)>; + }; +} // namespace detail + +/// A handler an application passes to one of the asynchronous methods, written in terms of what +/// that method produces: `failable_function` is a handler taking a +/// message id. +/// +/// What it adds is the leading `error` argument every one of them carries -- unset when the call +/// succeeded, and otherwise saying what went wrong. Every such handler is invoked exactly once, +/// unless the Client is destroyed before its work runs, so a caller is never left waiting on an +/// answer that is not coming; the error argument is how a failure says so, since a call that has +/// been dispatched has no caller left to throw to. +/// +/// Written as an alias rather than spelled out at each declaration so that the convention is stated +/// once and the argument cannot be forgotten or put in the wrong place. +template +using failable_function = typename detail::failable_function::type; + +} // namespace session::client diff --git a/include/session/client/message.hpp b/include/session/client/message.hpp new file mode 100644 index 000000000..f8c62f09b --- /dev/null +++ b/include/session/client/message.hpp @@ -0,0 +1,216 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::client { + +/// Delivery state of an outgoing message. Incoming messages have no send state. +enum class SendState : int { + /// Queued locally; Core has not yet been able to dispatch it (typically waiting on a PFS key + /// fetch for the recipient). + pending = 0, + + /// Handed to the swarm; awaiting confirmation. + sending = 1, + + /// Accepted by a swarm node. + sent = 2, + + /// Terminal failure: the swarm rejected it, there was no network, or encryption failed. + failed = 3, + + /// The application exited while this send was in flight, so its outcome is unknown — it may or + /// may not have reached the swarm. Set at startup for anything left in pending or sending, + /// because Core's in-flight send queue does not survive a restart. + interrupted = 4, + + /// Terminal, and unlike `failed` not worth trying again: something the message needs is gone + /// rather than merely unreachable. Currently that means an attachment whose file is no longer + /// where it was — retrying re-reads the same path and will fail identically every time. + /// + /// Kept apart from `failed` so that an application knows which failures to offer a retry for, + /// and which to offer only deletion. + unsendable = 6, + + /// Attachments are being uploaded to the file server. The message exists and is displayable, + /// but nothing has gone to a swarm yet: it cannot, because the message has to carry the + /// pointers the upload is still producing. How far along that is, is reported to whoever asked + /// for the send, through the progress handler they passed to send_message. + uploading = 5, +}; + +/// A position in a conversation's message history, used to page backwards. Ordering is by +/// timestamp then message id, so this is a stable cursor even when several messages share a +/// timestamp. +struct MessageCursor { + sys_ms timestamp; + int64_t id; + std::strong_ordering operator<=>(const MessageCursor&) const = default; +}; + +/// How far a deletion went, for a message whose content has been removed. +enum class Deletion : int { + here = 1, ///< Deleted on this device; the copies elsewhere are untouched. + everywhere = 2 ///< Deleted here, and asked to be deleted wherever else it reached. +}; + +/// A message to send: everything about it that is the caller's to decide. +/// +/// One struct rather than an overload per combination, so that gaining an optional field costs a +/// field rather than doubling the overload set — which it had already done once. Designated +/// initialisers make a call say which parts it is using: +/// +/// convo->send_message({.body = "hi"}, await); +/// convo->send_message({.body = "look", .attachments = {...}}, on_upload, await); +/// convo->send_message({.body = "agreed", .reply_to = other_id}, await); +struct OutgoingMessage { + std::string body; + + /// Files to send with it. See the attachment overloads' notes on what sending these costs and + /// how progress is reported: with any attachment present, sending becomes two-stage. + std::vector attachments; + + /// The message this answers, by local id. + /// + /// An id rather than the author and timestamp the wire wants, because everything beyond the id + /// is ours to look up — and a caller holding a message it is looking at has the id and nothing + /// else, so asking for the rest would be asking it to guess. + /// + /// @throws std::invalid_argument, on the calling thread, if this names a message that does not + /// exist or that belongs to another conversation. + std::optional reply_to; +}; + +struct Message; + +/// What a message is a reply to. +/// +/// The wire addresses a message by who sent it and when, never by an id, so `author` and +/// `timestamp` are always known even when the message itself is not — which is what lets a client +/// name the author over an "original message not found" line. +struct Reply { + /// Who wrote the replied-to message, and when they sent it. + b33 author; + sys_ms timestamp; + + /// Whether we have that message at all. Unset means we do not — never that there is no reply; + /// `Message::reply` being unset means that. + /// + /// It can become set on a later read, once the message arrives, so do not cache it alongside + /// the message. You are told when it changes: the quoting message is re-reported through + /// `message_updated`. + std::optional message_id; + + /// The message itself, when it was loaded. + /// + /// The whole message rather than a summary of it, because every summary is missing whatever a + /// caller needs next: a quoted message with an image and no text has nothing to show from a + /// body alone, and `deleted`, `gallery` and the attachment list are all things a reply line may + /// legitimately want to draw. + /// + /// A `shared_ptr` rather than an `optional` because `Message` contains this struct: an optional + /// would need a complete type, and a `unique_ptr` would make `Message` non-copyable when it is + /// copied into every callback and every page. + /// + /// **Null in a nested reply, even when `message_id` is set.** Loading goes one level deep, so + /// reading a message cannot walk an arbitrarily long chain of replies — a message reached + /// *through* a reply carries the reference to what it answered but not the answer itself. That + /// is why `message_id` is a separate field rather than read off this one: "we do not have it" + /// and "it was not loaded here" are different answers, and a caller wanting the second resolved + /// has the id to ask with. + /// + /// This is our own stored copy of the message, never the sender's. The wire has fields for the + /// sender's snippet of it, which current clients do not populate and which we would not trust + /// if they did: a forged one would put chosen words on screen attributed to someone else. + std::shared_ptr message; +}; + +struct Message { + /// Client-assigned, database-local message id. Stable for the life of the message; not the + /// swarm hash and not Core's send id. + int64_t id; + + ConversationId conversation; + + /// 0x05-prefixed session ID of the sender; our own account ID for outgoing messages. + b33 sender; + + bool outgoing; + + /// The message's authenticated timestamp (the Content `sigTimestamp`), which is what history + /// is ordered by — not the swarm's upload time. + sys_ms timestamp; + + /// The message body. Empty for a message that carries no text (e.g. attachments only). + std::string body; + + /// Files attached to the message, in the order they appear in it. Populated for every message + /// a query returns, so a caller never has to ask a second time to find out whether there are + /// any -- an attachments-only message is one with an empty `body` and a non-empty list here. + std::vector attachments; + + /// Delivery of the copy sent to the recipient — what "did it arrive" ordinarily means, and + /// what a message list normally shows. Unset on an incoming message. + std::optional send_state; + + /// Delivery of the copy deposited in our own swarm, which is how our other devices come to see + /// a message we sent. A separate send, retried separately, so it can still be pending when the + /// recipient already has the message — worth surfacing somewhere detailed rather than in the + /// message list, since it says nothing about whether the message arrived. + /// + /// Unset on an incoming message, and on a note to self, where the recipient's swarm is our own + /// and `send_state` already describes the only send there was. + std::optional sync_send_state; + + /// Swarm-assigned hash of the copy of this message held in *our own* swarm: for an incoming + /// message the one we retrieved, and for an outgoing one the copy we deposit for our other + /// devices (which for a note to self is the only copy there is). The copy sent to someone + /// else is stored in their swarm under a different hash, which is not reported here. + /// + /// Unset for an outgoing message that has not been stored yet, and for one stored by a build + /// whose storage server did not report a hash. + std::optional hash; + + /// Whether this message *can* be shown as a gallery rather than as a list of attachments. + /// + /// Derived, never stored, and the rule is ours: today it is "has attachments, and every one of + /// them is an image", but it may narrow to particular formats or gain a size ceiling. Deriving + /// it means a change to that rule takes effect on old messages too, rather than leaving stored + /// answers from whatever the rule used to be. + bool gallery_viewable = false; + /// Whether it *is* being shown that way. + /// + /// Stored, because it is a decision rather than a property: it is made when the message is + /// processed — on if the conversation was auto-downloading then — and the conversation's + /// setting may have changed since, so recomputing it later would not give the same answer. + /// + /// Never true when `gallery_viewable` is false: a stored decision that no longer agrees with + /// the current rule is dropped rather than honoured, so a message that qualified under an older + /// definition stops claiming to. + bool gallery = false; + + /// What this message is a reply to, or unset if it is not one. + std::optional reply; + + /// Set once the message has been deleted, saying how far the deletion went. The row survives + /// so that a redelivery cannot resurrect it, and so that a message deleted only here can still + /// be deleted everywhere afterwards. + /// + /// Everything the message said is gone when this is set: `body` is empty and `attachments` is + /// empty, whatever it carried before. What remains is who sent it and when, which is what a + /// client needs to draw the gap where it was — and `hash`, which is what keeps it deleted. + std::optional deleted; + + MessageCursor cursor() const { return {timestamp, id}; } +}; + +} // namespace session::client diff --git a/include/session/client/schema/schema_registry.hpp b/include/session/client/schema/schema_registry.hpp new file mode 100644 index 000000000..f4e5f2c6e --- /dev/null +++ b/include/session/client/schema/schema_registry.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include + +/// Migrations for Client's tables, generated from src/client/schema/ and applied via Core's +/// schema_extension option under the owner name "client". +/// +/// These run from Core::apply_migrations(), before any Core component's init() and long before the +/// Client that owns them exists, so they may depend on Core's tables but on nothing of Client's. +namespace session::client::schema { + +extern const std::span MIGRATIONS; + +/// See session::core::schema::FULL_SCHEMA. +extern const std::string_view FULL_SCHEMA; + +} // namespace session::client::schema diff --git a/include/session/clock.hpp b/include/session/clock.hpp new file mode 100644 index 000000000..f8941240c --- /dev/null +++ b/include/session/clock.hpp @@ -0,0 +1,117 @@ +#pragma once + +#include +#include +#include +#include + +namespace session { + +/// A clock satisfying the C++ Clock named requirement whose time_point is identical to +/// std::chrono::system_clock::time_point. Returns system time adjusted by a configurable offset, +/// making it suitable for both production use (where the offset is learned from the network) and +/// unit testing (where the offset can be set to any desired value). +struct AdjustedClock { + using duration = std::chrono::system_clock::duration; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::system_clock::time_point; + static constexpr bool is_steady = false; + + /// Returns the current time, adjusted by the current offset. + static time_point now() noexcept { + return std::chrono::system_clock::now() + duration{_offset.load(std::memory_order_relaxed)}; + } + + /// Sets the clock offset. Accepts any duration implicitly convertible to + /// system_clock::duration (e.g. std::chrono::milliseconds, seconds, nanoseconds). + static void set_offset(duration offset) { + _offset.store(offset.count(), std::memory_order_relaxed); + } + + /// Returns the current clock offset. + static duration get_offset() { return duration{_offset.load(std::memory_order_relaxed)}; } + + private: + inline static std::atomic _offset{0}; +}; + +// Returns the current time from AdjustedClock, optionally floored to the given precision. +// E.g. clock_now() gives a timepoint with seconds precision (aka +// std::chrono::sys_seconds). +template +inline std::chrono::sys_time clock_now() { + return std::chrono::floor(AdjustedClock::now()); +} +// Shortcut for clock_now(); +inline std::chrono::sys_seconds clock_now_s() { + return clock_now(); +} +using sys_ms = std::chrono::sys_time; +// Shortcut for clock_now(); +inline sys_ms clock_now_ms() { + return clock_now(); +} + +// Returns the duration count of the given duration cast into ToDuration. Example: +// duration_count(30000ms) // returns 30 +// This function requires that the target type is no more precise than d, that is, it will not allow +// you to cast from seconds to milliseconds because such a cast indicates that the sub-second +// precision has already been lost. +template + requires std::is_convertible_v> +constexpr int64_t duration_count(const std::chrono::duration& d) { + return std::chrono::duration_cast(d).count(); +} +// Returns the seconds count of the given duration +template + requires std::is_convertible_v> +constexpr int64_t duration_seconds(const std::chrono::duration& d) { + return duration_count(d); +} +// Returns the milliseconds count of the given duration +template + requires std::is_convertible_v> +constexpr int64_t duration_ms(const std::chrono::duration& d) { + return duration_count(d); +} + +// Returns the time-since-epoch count of the given time point, cast into ToDuration. The given time +// point must be at least as precise as ToDuration, i.e. this will not allow you to cast to a more +// precise time point as that would mean the intended precision has already been lost by an earlier +// cast. +template + requires std::is_convertible_v +constexpr int64_t epoch_count(const std::chrono::time_point& t) { + return duration_count(t.time_since_epoch()); +} +// Returns the seconds-since-epoch count of the given time point. The given time point must be at +// least as precise as seconds. +template + requires std::is_convertible_v +constexpr int64_t epoch_seconds(const std::chrono::time_point& t) { + return duration_seconds(t.time_since_epoch()); +} +// Returns the milliseconds-since-epoch count of the given time point. The given time point must +// have at least milliseconds precision. +template + requires std::is_convertible_v +constexpr int64_t epoch_ms(const std::chrono::time_point& t) { + return duration_ms(t.time_since_epoch()); +} + +// Inverse of epoch_count/epoch_seconds/epoch_ms: reconstruct a sys_time with the given duration +// precision from a raw integer count of that duration since the epoch. +template +inline std::chrono::sys_time from_epoch(int64_t t) { + return std::chrono::sys_time{Duration{t}}; +} +// Shortcuts for the common cases: +inline std::chrono::sys_seconds from_epoch_s(int64_t t) { + return from_epoch(t); +} +inline sys_ms from_epoch_ms(int64_t t) { + return from_epoch(t); +} + +} // namespace session diff --git a/include/session/config.hpp b/include/session/config.hpp index 1be5f2834..f0129ed94 100644 --- a/include/session/config.hpp +++ b/include/session/config.hpp @@ -12,6 +12,7 @@ #include #include "types.hpp" +#include "util.hpp" namespace session::config { @@ -50,7 +51,7 @@ constexpr inline const dict_variant& unwrap(const dict_value& v) { return static_cast(v); } -using hash_t = std::array; +using hash_t = std::array; using seqno_hash_t = std::pair; class MutableConfigMessage; @@ -102,9 +103,9 @@ class ConfigMessage { /// Seqno and hash of the message; we calculate this when loading. Subclasses put the hash here /// (so that they can return a reference to it). - seqno_hash_t seqno_hash_{0, {0}}; + seqno_hash_t seqno_hash_{0, {}}; - std::optional> verified_signature_; + std::optional verified_signature_; // This will be set during construction from configs based on the merge result: // nullopt means we had to merge one or more configs together into a new merged config @@ -121,11 +122,10 @@ class ConfigMessage { /// the message when loading multiple messages, but can still continue with other messages; /// throwing aborts the entire construction). using verify_callable = std::function data, std::span signature)>; + std::span data, std::span signature)>; /// Signing function: this is passed the data to be signed and returns the 64-byte signature. - using sign_callable = - std::function(std::span data)>; + using sign_callable = std::function(std::span data)>; ConfigMessage(); ConfigMessage(const ConfigMessage&) = default; @@ -138,7 +138,7 @@ class ConfigMessage { /// Initializes a config message by parsing a serialized message. Throws on any error. See the /// vector version below for argument descriptions. explicit ConfigMessage( - std::span serialized, + std::span serialized, verify_callable verifier = nullptr, sign_callable signer = nullptr, int lag = DEFAULT_DIFF_LAGS, @@ -174,7 +174,7 @@ class ConfigMessage { /// `[](size_t, const auto& e) { throw e; }` can be used to make any parse error of any message /// fatal. explicit ConfigMessage( - const std::vector>& configs, + const std::vector>& configs, verify_callable verifier = nullptr, sign_callable signer = nullptr, int lag = DEFAULT_DIFF_LAGS, @@ -229,9 +229,7 @@ class ConfigMessage { /// verified signature when it was parsed. Returns nullopt otherwise (e.g. not loaded from /// verification at all; loaded without a verification function; or had no signature and a /// signature wasn't required). - const std::optional>& verified_signature() { - return verified_signature_; - } + const std::optional& verified_signature() { return verified_signature_; } /// Constructs a new MutableConfigMessage from this config message with an incremented seqno. /// The new config message's diff will reflect changes made after this construction. @@ -245,11 +243,10 @@ class ConfigMessage { /// typically for a local serialization value that isn't being pushed to the server). Note that /// signing is always disabled if there is no signing callback set, regardless of the value of /// this argument. - virtual std::vector serialize(bool enable_signing = true); + virtual std::vector serialize(bool enable_signing = true); protected: - std::vector serialize_impl( - const oxenc::bt_dict& diff, bool enable_signing = true); + std::vector serialize_impl(const oxenc::bt_dict& diff, bool enable_signing = true); }; // Constructor tag @@ -297,7 +294,7 @@ class MutableConfigMessage : public ConfigMessage { /// constructor only increments seqno once while the indirect version would increment twice in /// the case of a required merge conflict resolution. explicit MutableConfigMessage( - const std::vector>& configs, + const std::vector>& configs, verify_callable verifier = nullptr, sign_callable signer = nullptr, int lag = DEFAULT_DIFF_LAGS, @@ -307,7 +304,7 @@ class MutableConfigMessage : public ConfigMessage { /// take an error handler and instead always throws on parse errors (the above also throws for /// an erroneous single message, but with a less specific "no valid config messages" error). explicit MutableConfigMessage( - std::span config, + std::span config, verify_callable verifier = nullptr, sign_callable signer = nullptr, int lag = DEFAULT_DIFF_LAGS); @@ -355,7 +352,7 @@ class MutableConfigMessage : public ConfigMessage { protected: /// Internal version of hash() that takes the already-serialized value, to avoid needing a call /// to `serialize()` when such a call has already been done for other reasons. - const hash_t& hash(std::span serialized); + const hash_t& hash(std::span serialized); void increment_impl(); }; @@ -396,7 +393,7 @@ class MutableConfigMessage : public ConfigMessage { void verify_config_sig( oxenc::bt_dict_consumer dict, const ConfigMessage::verify_callable& verifier, - std::optional>* verified_signature = nullptr, + std::optional* verified_signature = nullptr, bool trust_signature = false); } // namespace session::config diff --git a/include/session/config/base.hpp b/include/session/config/base.hpp index fad7c7d76..f4b9a6a86 100644 --- a/include/session/config/base.hpp +++ b/include/session/config/base.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include +#include "../crypto/ed25519.hpp" #include "../hash.hpp" #include "../logging.hpp" #include "../sodium_array.hpp" @@ -54,8 +56,8 @@ enum class ConfigState : int { Waiting = 2, }; -using Ed25519PubKey = std::array; -using Ed25519Secret = sodium_array; +using Ed25519PubKey = b32; +using Ed25519Secret = sodium_vector; // Helper base class for holding a config signing keypair class ConfigSig { @@ -73,7 +75,7 @@ class ConfigSig { // be 64 bytes or less, and should generally be unique for each key use case. // // Throws if a secret key hasn't been set via `set_sig_keys`. - std::array seed_hash(std::string_view key) const; + cleared_b32 seed_hash(std::string_view key) const; virtual void set_verifier(ConfigMessage::verify_callable v) = 0; virtual void set_signer(ConfigMessage::sign_callable v) = 0; @@ -83,8 +85,8 @@ class ConfigSig { // // Throws if given invalid data (i.e. wrong key size, or mismatched pubkey/secretkey). void init_sig_keys( - std::optional> ed25519_pubkey, - std::optional> ed25519_secretkey); + std::optional> ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey); public: virtual ~ConfigSig() = default; @@ -112,7 +114,7 @@ class ConfigSig { /// Inputs: /// - `secret` -- the 64-byte sodium-style Ed25519 "secret key" (actually the seed+pubkey /// concatenated together) that sets both the secret key and public key. - void set_sig_keys(std::span secret); + void set_sig_keys(const ed25519::PrivKeySpan& secret); /// API: base/ConfigSig::set_sig_pubkey /// @@ -122,7 +124,7 @@ class ConfigSig { /// /// Inputs: /// - `pubkey` -- the 32 byte Ed25519 pubkey that must have signed incoming messages - void set_sig_pubkey(std::span pubkey); + void set_sig_pubkey(std::span pubkey); /// API: base/ConfigSig::get_sig_pubkey /// @@ -132,7 +134,7 @@ class ConfigSig { /// /// Outputs: /// - reference to the 32-byte pubkey, or `std::nullopt` if not set. - const std::optional>& get_sig_pubkey() const { return _sign_pk; } + const std::optional& get_sig_pubkey() const { return _sign_pk; } /// API: base/ConfigSig::clear_sig_keys /// @@ -159,7 +161,7 @@ class ConfigBase : public ConfigSig { // Contains the base key(s) we use to encrypt/decrypt messages. If non-empty, the .front() // element will be used when encrypting a new message to push. When decrypting, we attempt each // of them, starting with .front(), until decryption succeeds. - using Key = std::array; + using Key = std::array; sodium_vector _keys; // Contains the current active message hash(es), as fed into us in `confirm_pushed()`. @@ -173,12 +175,11 @@ class ConfigBase : public ConfigSig { std::unordered_set _old_hashes; struct PartialMessage { - int index; // 0-based index of this part - std::string message_id; // storage server message hash of this part - std::vector data; // Data chunk + int index; // 0-based index of this part + std::string message_id; // storage server message hash of this part + std::vector data; // Data chunk - PartialMessage( - int index, std::string_view message_id, std::span data) : + PartialMessage(int index, std::string_view message_id, std::span data) : index{index}, message_id{message_id}, data{data.begin(), data.end()} {} }; struct PartialMessages { @@ -202,7 +203,7 @@ class ConfigBase : public ConfigSig { done = true; size = 0; parts.clear(); - expiry = std::chrono::system_clock::now() + lifetime; + expiry = clock_now() + lifetime; } }; @@ -230,8 +231,8 @@ class ConfigBase : public ConfigSig { // // For new parts that don't complete a set, errors, and already seen messages the optional // value will be nullopt. - std::pair, std::vector>>> - _handle_multipart(std::string_view msg_id, std::span message); + std::pair, std::vector>>> + _handle_multipart(std::string_view msg_id, std::span message); // Writes multipart data into the sub-dict of the dump data. void _dump_multiparts(oxenc::bt_dict_producer&& multi) const; @@ -252,9 +253,9 @@ class ConfigBase : public ConfigSig { // verification of incoming messages using the associated pubkey, and will be signed using the // secretkey (if a secret key is given). explicit ConfigBase( - std::optional> dump = std::nullopt, - std::optional> ed25519_pubkey = std::nullopt, - std::optional> ed25519_secretkey = std::nullopt); + std::optional> dump = std::nullopt, + std::optional> ed25519_pubkey = std::nullopt, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey = std::nullopt); // Initializes the base config object with dump data and keys; this is typically invoked by the // constructor, but is exposed to subclasses so that they can delay initial processing by @@ -265,9 +266,9 @@ class ConfigBase : public ConfigSig { // // This method must not be called outside derived class construction! void init( - std::optional> dump = std::nullopt, - std::optional> ed25519_pubkey = std::nullopt, - std::optional> ed25519_secretkey = std::nullopt); + std::optional> dump = std::nullopt, + std::optional> ed25519_pubkey = std::nullopt, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey = std::nullopt); // Tracks whether we need to dump again; most mutating methods should set this to true (unless // calling set_state, which sets to to true implicitly). @@ -551,20 +552,19 @@ class ConfigBase : public ConfigSig { /// - `std::string*` -- Returns a pointer to the string if one exists const std::string* string() const { return get_clean(); } - /// API: base/ConfigBase::DictFieldProxy::uview + /// API: base/ConfigBase::DictFieldProxy::bview /// - /// Returns the value as a std::span, if it exists and is a string; + /// Returns the value as a std::span, if it exists and is a string; /// nullopt otherwise. /// /// Inputs: None /// /// Outputs: - /// - `std::optional>` -- Returns a value as a view if it - /// exists - std::optional> uview() const { + /// - `std::optional>` -- Returns a value as a view if it exists + std::optional> bview() const { if (auto* s = get_clean()) - return std::span{ - reinterpret_cast(s->data()), s->size()}; + return std::span{ + reinterpret_cast(s->data()), s->size()}; return std::nullopt; } @@ -703,15 +703,13 @@ class ConfigBase : public ConfigSig { /// API: base/ConfigBase::DictFieldProxy::operator=(std::span) /// - /// Replaces the current value with the given std::span. This also + /// Replaces the current value with the given std::span. This also /// auto-vivifies any intermediate dicts needed to reach the given key, including replacing /// non-dict values if they currently exist along the path (this makes a copy). /// /// Inputs: - /// - `value` -- replaces current value with given std::span - /// - /// Same as above, but takes a std::span - void operator=(std::span value) { + /// - `value` -- replaces current value with given std::span + void operator=(std::span value) { *this = std::string{reinterpret_cast(value.data()), value.size()}; } @@ -932,7 +930,7 @@ class ConfigBase : public ConfigSig { /// and processed as a config message, even if it was too old to be useful (or was already /// known to be included). std::unordered_set _merge( - std::span>> configs); + std::span>> configs); /// API: base/ConfigBase::extra_data /// @@ -974,7 +972,7 @@ class ConfigBase : public ConfigSig { /// /// Inputs: /// - `ed25519_secret_key` -- key is loaded for encryption - void load_key(std::span ed25519_secretkey); + void load_key(const ed25519::PrivKeySpan& ed25519_secretkey); public: virtual ~ConfigBase() = default; @@ -1064,9 +1062,9 @@ class ConfigBase : public ConfigSig { /// Declaration: /// ```cpp /// std::unordered_set merge( - /// const std::vector>>& configs); + /// const std::vector>>& configs); /// std::unordered_set merge( - /// const std::vector>>& configs); + /// const std::vector>>& configs); /// ``` /// /// Inputs: @@ -1083,12 +1081,29 @@ class ConfigBase : public ConfigSig { /// parts that do not complete a message set, inclusion in the return value is based only on /// whether the multipart part itself looked valid. std::unordered_set merge( - const std::vector>>& configs); + const std::vector>>& configs); - // Same as above, but takes values as std::spans (because sometimes that is + // Same as above, but takes values as std::spans (because sometimes that is // more convenient). std::unordered_set merge( - const std::vector>>& configs); + const std::vector>>& configs); + + /// API: base/ConfigBase::seqno + /// + /// Returns the current sequence number of the config data. + /// + /// This advances when the config changes: on a local modification, and on a merge that either + /// adopted a higher-numbered config from elsewhere or had to resolve a conflict. A merge that + /// changed nothing -- because what arrived was stale, or was already what we held -- leaves it + /// alone. Comparing it either side of a `merge()` is therefore how a caller asks whether that + /// merge actually did anything, which the returned hash set does not answer: a hash is reported + /// as parsed regardless of whether it turned out to be useful. + /// + /// Inputs: None + /// + /// Outputs: + /// - `seqno_t` -- the current sequence number + seqno_t seqno() const; /// API: base/ConfigBase::is_dirty /// @@ -1248,12 +1263,12 @@ class ConfigBase : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::tuple, std::vector>` - Returns a + /// - `std::tuple, std::vector>` - Returns a /// tuple containing /// - `seqno_t` -- sequence number - /// - `std::vector` -- data message to push to the server + /// - `std::vector` -- data message to push to the server /// - `std::vector` -- list of known message hashes - virtual std::tuple>, std::vector> + virtual std::tuple>, std::vector> push(); /// API: base/ConfigBase::confirm_pushed @@ -1290,8 +1305,8 @@ class ConfigBase : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::vector` -- Returns binary data of the state dump - std::vector dump(); + /// - `std::vector` -- Returns binary data of the state dump + std::vector dump(); /// API: base/ConfigBase::make_dump /// @@ -1302,8 +1317,8 @@ class ConfigBase : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::vector` -- Returns binary data of the state dump - std::vector make_dump() const; + /// - `std::vector` -- Returns binary data of the state dump + std::vector make_dump() const; /// API: base/ConfigBase::needs_dump /// @@ -1366,14 +1381,14 @@ class ConfigBase : public ConfigSig { /// Will throw a std::invalid_argument if the key is not 32 bytes. /// /// Inputs: - /// - `std::span key` -- 32 byte binary key + /// - `std::span key` -- 32 byte binary key /// - `high_priority` -- Whether to add to front or back of key list. If true then key is added /// to beginning and replace highest-priority key for encryption /// - `dirty_config` -- if true then mark the config as dirty (incrementing seqno and needing a /// push) if the first key (i.e. the key used for encryption) is changed as a result of this /// call. Ignored if the config is not modifiable. void add_key( - std::span key, + std::span key, bool high_priority = true, bool dirty_config = false); @@ -1407,7 +1422,7 @@ class ConfigBase : public ConfigSig { /// /// Outputs: /// - `bool` -- Returns true if found and removed - bool remove_key(std::span key, size_t from = 0, bool dirty_config = false); + bool remove_key(std::span key, size_t from = 0, bool dirty_config = false); /// API: base/ConfigBase::replace_keys /// @@ -1421,7 +1436,7 @@ class ConfigBase : public ConfigSig { /// requiring a repush) if the old and new first key are not the same. Ignored if the config /// is not modifiable. void replace_keys( - const std::vector>& new_keys, bool dirty_config = false); + const std::vector>& new_keys, bool dirty_config = false); /// API: base/ConfigBase::get_keys /// @@ -1435,8 +1450,8 @@ class ConfigBase : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::vector>` -- Returns vector of encryption keys - std::vector> get_keys() const; + /// - `std::vector>` -- Returns vector of encryption keys + std::vector> get_keys() const; /// API: base/ConfigBase::key_count /// @@ -1457,7 +1472,7 @@ class ConfigBase : public ConfigSig { /// /// Outputs: /// - `bool` -- Returns true if it does exist - bool has_key(std::span key) const; + bool has_key(std::span key) const; /// API: base/ConfigBase::key /// @@ -1469,10 +1484,10 @@ class ConfigBase : public ConfigSig { /// - `i` -- keys position in key list /// /// Outputs: - /// - `std::span` -- binary data of the key - std::span key(size_t i = 0) const { + /// - `std::span` -- binary data of the key + std::span key(size_t i = 0) const { assert(i < _keys.size()); - return {_keys[i].data(), _keys[i].size()}; + return _keys[i]; } }; @@ -1509,56 +1524,8 @@ struct internals final { const ConfigT& operator*() const { return *operator->(); } }; -template , int> = 0> -inline internals& unbox(config_object* conf) { - return *static_cast*>(conf->internals); -} -template , int> = 0> -inline const internals& unbox(const config_object* conf) { - return *static_cast*>(conf->internals); -} - -template -void copy_c_str(char (&dest)[N], std::string_view src) { - if (src.size() >= N) - src.remove_suffix(src.size() - N - 1); - std::memcpy(dest, src.data(), src.size()); - dest[src.size()] = 0; -} - -// Wraps a labmda and, if an exception is thrown, sets an error message in the internals.error -// string and updates the last_error pointer in the outer (C) config_object struct to point at it. -// -// No return value: accepts void and pointer returns; pointer returns will become nullptr on error -template -decltype(auto) wrap_exceptions(config_object* conf, Call&& f) { - using Ret = std::invoke_result_t; - - try { - conf->last_error = nullptr; - return std::invoke(std::forward(f)); - } catch (const std::exception& e) { - copy_c_str(conf->_error_buf, e.what()); - conf->last_error = conf->_error_buf; - } - if constexpr (std::is_pointer_v) - return static_cast(nullptr); - else - static_assert(std::is_void_v, "Don't know how to return an error value!"); -} - -// Same as above but accepts callbacks with value returns on errors: returns `f()` on success, -// `error_return` on exception -template -Ret wrap_exceptions(config_object* conf, Call&& f, Ret error_return) { - try { - conf->last_error = nullptr; - return std::invoke(std::forward(f)); - } catch (const std::exception& e) { - copy_c_str(conf->_error_buf, e.what()); - conf->last_error = conf->_error_buf; - } - return error_return; -} +// Internal helper: attempts zstd compression of `msg` in-place (with a 'z' prefix byte); leaves +// `msg` unchanged if compression does not reduce the size. `level` of 0 disables compression. +void compress_message(std::vector& msg, int level); } // namespace session::config diff --git a/include/session/config/community.hpp b/include/session/config/community.hpp index 232e53e37..ee2f3f582 100644 --- a/include/session/config/community.hpp +++ b/include/session/config/community.hpp @@ -30,7 +30,7 @@ struct community { community( std::string_view base_url, std::string_view room, - std::span pubkey); + std::span pubkey); // Same as above, but takes pubkey as an encoded (hex or base32z or base64) string. community(std::string_view base_url, std::string_view room, std::string_view pubkey_encoded); @@ -92,13 +92,13 @@ struct community { /// /// Declaration: /// ```cpp - /// void set_pubkey(std::span pubkey); + /// void set_pubkey(std::span pubkey); /// void set_pubkey(std::string_view pubkey); /// ``` /// /// Inputs: /// - `pubkey` -- Pubkey to be stored - void set_pubkey(std::span pubkey); + void set_pubkey(std::span pubkey); void set_pubkey(std::string_view pubkey); /// API: community/community::base_url @@ -140,8 +140,8 @@ struct community { /// Inputs: None /// /// Outputs: - /// - `const std::vector&` -- Returns the pubkey - const std::vector& pubkey() const { return pubkey_; } + /// - `const std::vector&` -- Returns the pubkey + const std::vector& pubkey() const { return pubkey_; } /// API: community/community::pubkey_hex /// @@ -199,9 +199,7 @@ struct community { /// Outputs: /// - `std::string` -- Returns the Full URL static std::string full_url( - std::string_view base_url, - std::string_view room, - std::span pubkey); + std::string_view base_url, std::string_view room, std::span pubkey); /// API: community/community::canonical_url /// @@ -269,8 +267,8 @@ struct community { /// - `std::tuple` -- Tuple of 3 components of the url /// - `std::string` -- canonical url, normalized /// - `std::string` -- room name, *not* normalized - /// - `std::vector` -- binary of the server pubkey - static std::tuple> parse_full_url( + /// - `std::vector` -- binary of the server pubkey + static std::tuple> parse_full_url( std::string_view full_url); /// API: community/community::parse_partial_url @@ -285,9 +283,9 @@ struct community { /// - `std::tuple` -- Tuple of 3 components of the url /// - `std::string` -- canonical url, normalized /// - `std::string` -- room name, *not* normalized - /// - `std::optional>` -- optional binary of the server pubkey if + /// - `std::optional>` -- optional binary of the server pubkey if /// present - static std::tuple>> + static std::tuple>> parse_partial_url(std::string_view url); protected: @@ -297,7 +295,7 @@ struct community { // `someroom` and this could `SomeRoom`). Omitted if not available. std::optional localized_room_; // server pubkey - std::vector pubkey_; + std::vector pubkey_; // Construction without a pubkey for when pubkey isn't known yet but will be set shortly // after constructing (or when isn't needed, such as when deleting). @@ -349,8 +347,12 @@ struct comm_iterator_helper { continue; } - std::span pubkey{ - reinterpret_cast(pubkey_raw->data()), pubkey_raw->size()}; + if (pubkey_raw->size() != 32) { + next_server(); + continue; + } + auto pubkey = std::span{ + reinterpret_cast(pubkey_raw->data()), 32}; if (!it_room) { if (auto rit = server_info_dict->find("R"); diff --git a/include/session/config/contacts.h b/include/session/config/contacts.h index 7fd175c48..409f0bb84 100644 --- a/include/session/config/contacts.h +++ b/include/session/config/contacts.h @@ -36,7 +36,14 @@ typedef struct contacts_contact { int64_t created; // unix timestamp (seconds) - session_protocol_pro_profile_bitset profile_bitset; + // Messages in this conversation older than these are to be deleted (and arriving ones older + // than them dropped); 0 for no such instruction. `delete_attach_before` covers the attachments + // alone, leaving the messages. Both unix timestamps in seconds, matching the group info + // config's fields of the same names. + int64_t delete_before; + int64_t delete_attach_before; + + uint64_t profile_bitset; // Mask of SESSION_PROTOCOL_PRO_PROFILE_FEATURE_* bits } contacts_contact; @@ -55,7 +62,7 @@ typedef struct contacts_blinded_contact { bool legacy_blinding; int64_t created; // unix timestamp (seconds) - session_protocol_pro_profile_bitset profile_bitset; + uint64_t profile_bitset; // Mask of SESSION_PROTOCOL_PRO_PROFILE_FEATURE_* bits } contacts_blinded_contact; @@ -371,7 +378,7 @@ LIBSESSION_EXPORT bool contacts_set_blinded( /// /// Outputs: /// - `bool` -- True if erasing was successful -LIBSESSION_EXPORT bool contacts_erase_blinded_contact( +LIBSESSION_EXPORT bool contacts_erase_blinded( config_object* conf, const char* community_base_url, const char* blinded_id); typedef struct contacts_iterator { diff --git a/include/session/config/contacts.hpp b/include/session/config/contacts.hpp index 7490c2021..ff09704e1 100644 --- a/include/session/config/contacts.hpp +++ b/include/session/config/contacts.hpp @@ -49,6 +49,11 @@ namespace session::config { /// equivalent "j"oined field). Omitted if 0. /// t - The `profile_updated` unix timestamp (seconds) for this contacts profile information. /// f - session pro profile features bitset for this contact +/// d - "delete before" unix timestamp (seconds): messages in this conversation older than this +/// are to be deleted, and arriving ones older than it are to be dropped. Omitted if 0. +/// Named to match the group info config's equivalent field. +/// D - "delete attachments before" unix timestamp (seconds): as above but for attachments +/// alone, leaving the messages themselves. Omitted if 0. /// /// b - dict of blinded contacts. This is a nested dict where the outer keys are the BASE_URL of /// the community the blinded contact originated from and the outer value is a dict containing: @@ -96,7 +101,21 @@ struct contact_info { std::chrono::seconds exp_timer{0}; // The expiration timer (in seconds) int64_t created = 0; // Unix timestamp (seconds) when this contact was added - ProProfileBitset profile_bitset = {}; + /// Messages in this conversation older than this are to be deleted, and arriving ones older + /// than it dropped. This is what makes clearing a conversation, and deleting one, mean the + /// same thing on every device: the instruction is recorded rather than inferred from when some + /// config happened to be written. Epoch (the default) means no such instruction. + std::chrono::sys_seconds delete_before{}; + + /// As above, but only the attachments: the messages themselves stay. Attachments are most of + /// what there is to reclaim, so they are worth being able to drop on their own. + /// + /// Only stored while it says something `delete_before` does not: deleting a message takes its + /// attachments with it, so a value at or before `delete_before` is dropped when the contact is + /// stored rather than kept as a redundant instruction. + std::chrono::sys_seconds delete_attach_before{}; + + ProProfileFlags profile_flags = ProProfileFlags::None; explicit contact_info(std::string sid); @@ -141,12 +160,12 @@ struct blinded_contact_info { bool legacy_blinding; std::chrono::sys_seconds created{}; // Unix timestamp (seconds) when this contact was added - ProProfileBitset profile_bitset = {}; + ProProfileFlags profile_flags = ProProfileFlags::None; blinded_contact_info() = default; explicit blinded_contact_info( std::string_view community_base_url, - std::span community_pubkey, + std::span community_pubkey, std::string_view blinded_id); // Internal ctor/method for C API implementations: @@ -187,8 +206,8 @@ struct blinded_contact_info { /// Inputs: None /// /// Outputs: - /// - `const std::vector&` -- Returns the pubkey - const std::vector& community_pubkey() const { return comm.pubkey(); } + /// - `const std::vector&` -- Returns the pubkey + const std::vector& community_pubkey() const { return comm.pubkey(); } /// API: contacts/blinded_contact_info::community_pubkey_hex /// @@ -212,7 +231,7 @@ struct blinded_contact_info { /// into this struct void set_base_url(std::string_view base_url); void set_room(std::string_view room); - void set_pubkey(std::span pubkey); + void set_pubkey(std::span pubkey); void set_pubkey(std::string_view pubkey); }; @@ -239,8 +258,8 @@ class Contacts : public ConfigBase { /// Outputs: /// - `Contact` - Constructor Contacts( - std::span ed25519_secretkey, - std::optional> dumped); + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: contacts/Contacts::storage_namespace /// @@ -443,7 +462,7 @@ class Contacts : public ConfigBase { /// Inputs: /// - `session_id` -- hex string of the session id /// - `features` -- The updated profile features to use - void set_pro_features(std::string_view session_id, ProProfileBitset features); + void set_pro_features(std::string_view session_id, ProProfileFlags features); /// API: contacts/contacts::erase /// @@ -482,8 +501,7 @@ class Contacts : public ConfigBase { protected: // Drills into the nested dicts to access community details DictFieldProxy blinded_contact_field( - const blinded_contact_info& bc, - std::span* get_pubkey = nullptr) const; + const blinded_contact_info& bc, std::span* get_pubkey = nullptr) const; public: /// API: contacts/Contacts::blinded diff --git a/include/session/config/convo_info_volatile.h b/include/session/config/convo_info_volatile.h index 001327b25..6a501a8d4 100644 --- a/include/session/config/convo_info_volatile.h +++ b/include/session/config/convo_info_volatile.h @@ -15,10 +15,10 @@ typedef struct convo_info_volatile_1to1 { bool unread; // true if the conversation is explicitly marked unread bool has_pro_revocation_tag; // Flag indicating if hash is set - bytes32 pro_revocation_tag; // Opaque revocation tag identifying this proof (from the Session + cbytes32 pro_revocation_tag; // Opaque revocation tag identifying this proof (from the Session // Pro backend) - int64_t pro_expiry_ts; // Unix epoch timestamp (seconds) until which this contact's - // entitlement to Session Pro features is valid + int64_t pro_expiry_ts; // Unix epoch timestamp (seconds) until which this contact's entitlement + // to Session Pro features is valid } convo_info_volatile_1to1; typedef struct convo_info_volatile_community { @@ -53,10 +53,10 @@ typedef struct convo_info_volatile_blinded_1to1 { bool unread; // true if the conversation is explicitly marked unread bool has_pro_revocation_tag; // Flag indicating if hash is set - bytes32 pro_revocation_tag; // Opaque revocation tag identifying this proof (from the Session + cbytes32 pro_revocation_tag; // Opaque revocation tag identifying this proof (from the Session // Pro backend) - int64_t pro_expiry_ts; // Unix epoch timestamp (seconds) until which this contact's - // entitlement to Session Pro features is valid + int64_t pro_expiry_ts; // Unix epoch timestamp (seconds) until which this contact's entitlement + // to Session Pro features is valid } convo_info_volatile_blinded_1to1; /// API: convo_info_volatile/convo_info_volatile_init diff --git a/include/session/config/convo_info_volatile.hpp b/include/session/config/convo_info_volatile.hpp index 69afc45f5..f4285c36c 100644 --- a/include/session/config/convo_info_volatile.hpp +++ b/include/session/config/convo_info_volatile.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "base.hpp" @@ -87,11 +88,11 @@ namespace convo { struct pro_base : base { /// Opaque revocation tag identifying this proof (from the Session Pro backend) - std::optional pro_revocation_tag; + std::optional pro_revocation_tag; /// Unix epoch timestamp (seconds) until which this proof's entitlement to Session Pro /// features is valid - sys_seconds pro_expiry_at{}; + std::chrono::sys_seconds pro_expiry_at{}; protected: using base::base; @@ -230,8 +231,8 @@ class ConvoInfoVolatile : public ConfigBase { /// - `dumped` -- either `std::nullopt` to construct a new, empty object; or binary state data /// that was previously dumped from an instance of this class by calling `dump()`. ConvoInfoVolatile( - std::span ed25519_secretkey, - std::optional> dumped); + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: convo_info_volatile/ConvoInfoVolatile::storage_namespace /// @@ -284,12 +285,12 @@ class ConvoInfoVolatile : public ConfigBase { /// Inputs: None /// /// Outputs: - /// - `std::tuple, std::vector>` - Returns a + /// - `std::tuple, std::vector>` - Returns a /// tuple containing /// - `seqno_t` -- sequence number - /// - `std::vector>` -- data message(s) to push to the server + /// - `std::vector>` -- data message(s) to push to the server /// - `std::vector` -- list of known message hashes - std::tuple>, std::vector> push() + std::tuple>, std::vector> push() override; /// API: convo_info_volatile/ConvoInfoVolatile::get_1to1 @@ -427,7 +428,7 @@ class ConvoInfoVolatile : public ConfigBase { /// std::string_view base_url, std::string_view room, std::string_view pubkey_hex) /// const; /// convo::community get_or_construct_community( - /// std::string_view base_url, std::string_view room, std::span + /// std::string_view base_url, std::string_view room, std::span /// pubkey) const; /// ``` /// @@ -443,7 +444,7 @@ class ConvoInfoVolatile : public ConfigBase { convo::community get_or_construct_community( std::string_view base_url, std::string_view room, - std::span pubkey) const; + std::span pubkey) const; /// API: convo_info_volatile/ConvoInfoVolatile::get_or_construct_community(full_url) /// @@ -507,7 +508,7 @@ class ConvoInfoVolatile : public ConfigBase { // Drills into the nested dicts to access community details; if the second argument is // non-nullptr then it will be set to the community's pubkey, if it exists. DictFieldProxy community_field( - const convo::community& og, std::span* get_pubkey = nullptr) const; + const convo::community& og, std::span* get_pubkey = nullptr) const; public: /// API: convo_info_volatile/ConvoInfoVolatile::erase_1to1 diff --git a/include/session/config/encrypt.hpp b/include/session/config/encrypt.hpp index b4b46ef44..709df747f 100644 --- a/include/session/config/encrypt.hpp +++ b/include/session/config/encrypt.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -34,10 +35,10 @@ namespace session::config { /// - `domain` -- short string for the keyed hash /// /// Outputs: -/// - `std::vector` -- Returns the encrypted message bytes -std::vector encrypt( - std::span message, - std::span key_base, +/// - `std::vector` -- Returns the encrypted message bytes +std::vector encrypt( + std::span message, + std::span key_base, std::string_view domain); /// API: encrypt/encrypt_inplace @@ -50,10 +51,24 @@ std::vector encrypt( /// - `key_base` -- Fixed key that all clients, must be 32 bytes. /// - `domain` -- short string for the keyed hash void encrypt_inplace( - std::vector& message, - std::span key_base, + std::vector& message, + std::span key_base, std::string_view domain); +/// API: encrypt/encrypt_prealloced +/// +/// Encrypts a pre-allocated buffer in place. `message` must have exactly ENCRYPT_DATA_OVERHEAD +/// bytes of trailing space already allocated beyond the plaintext (i.e. message.size() must equal +/// plaintext_size + ENCRYPT_DATA_OVERHEAD). The plaintext in the leading bytes is encrypted in +/// place, and the auth tag and nonce are written into the trailing ENCRYPT_DATA_OVERHEAD bytes. +/// +/// Inputs: +/// - `message` -- buffer containing plaintext followed by ENCRYPT_DATA_OVERHEAD reserved bytes +/// - `key_base` -- Fixed key that all clients, must be 32 bytes. +/// - `domain` -- short string for the keyed hash +void encrypt_prealloced( + std::span message, std::span key_base, std::string_view domain); + /// API: encrypt/ENCRYPT_DATA_OVERHEAD /// /// Member variable @@ -82,10 +97,10 @@ struct decrypt_error : std::runtime_error { /// - `domain` -- short string for the keyed hash /// /// Outputs: -/// - `std::vector` -- Returns the decrypt message bytes -std::vector decrypt( - std::span ciphertext, - std::span key_base, +/// - `std::vector` -- Returns the decrypted message bytes +std::vector decrypt( + std::span ciphertext, + std::span key_base, std::string_view domain); /// API: encrypt/decrypt_inplace @@ -98,8 +113,8 @@ std::vector decrypt( /// - `key_base` -- Fixed key that all clients, must be 32 bytes. /// - `domain` -- short string for the keyed hash void decrypt_inplace( - std::vector& ciphertext, - std::span key_base, + std::vector& ciphertext, + std::span key_base, std::string_view domain); /// Returns the target size of the message with padding, assuming an additional `overhead` bytes of @@ -126,6 +141,6 @@ inline constexpr size_t padded_size(size_t s, size_t overhead = ENCRYPT_DATA_OVE /// - `data` -- the data; this is modified in place /// - `overhead` -- encryption overhead to account for to reach the desired padded size. The /// default, if omitted, is the space used by the `encrypt()` function defined above. -void pad_message(std::vector& data, size_t overhead = ENCRYPT_DATA_OVERHEAD); +void pad_message(std::vector& data, size_t overhead = ENCRYPT_DATA_OVERHEAD); } // namespace session::config diff --git a/include/session/config/groups/info.hpp b/include/session/config/groups/info.hpp index 66c0149ca..33a43c69a 100644 --- a/include/session/config/groups/info.hpp +++ b/include/session/config/groups/info.hpp @@ -55,9 +55,9 @@ class Info : public ConfigBase { /// push config changes. /// - `dumped` -- either `std::nullopt` to construct a new, empty object; or binary state data /// that was previously dumped from an instance of this class by calling `dump()`. - Info(std::span ed25519_pubkey, - std::optional> ed25519_secretkey, - std::optional> dumped); + Info(std::span ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: groups/Info::storage_namespace /// @@ -174,7 +174,7 @@ class Info : public ConfigBase { /// /// Declaration: /// ```cpp - /// void set_profile_pic(std::string_view url, std::span key); + /// void set_profile_pic(std::string_view url, std::span key); /// void set_profile_pic(profile_pic pic); /// ``` /// @@ -184,7 +184,7 @@ class Info : public ConfigBase { /// - `key` -- Decryption key /// - Second function: /// - `pic` -- Profile pic object - void set_profile_pic(std::string_view url, std::span key); + void set_profile_pic(std::string_view url, std::span key); void set_profile_pic(profile_pic pic); /// API: groups/Info::set_expiry_timer diff --git a/include/session/config/groups/keys.hpp b/include/session/config/groups/keys.hpp index 5eaa27bb4..433ae5ed1 100644 --- a/include/session/config/groups/keys.hpp +++ b/include/session/config/groups/keys.hpp @@ -88,7 +88,7 @@ class Keys : public ConfigSig { Ed25519Secret user_ed25519_sk; struct key_info { - std::array key; + std::array key; std::chrono::system_clock::time_point timestamp; // millisecond precision int64_t generation; @@ -108,8 +108,8 @@ class Keys : public ConfigSig { /// Hashes of messages we have successfully parsed; used for deciding what needs to be renewed. std::map> active_msgs_; - sodium_cleared> pending_key_; - sodium_vector pending_key_config_; + cleared_b32 pending_key_; + sodium_vector pending_key_config_; int64_t pending_gen_ = -1; bool needs_dump_ = false; @@ -120,21 +120,20 @@ class Keys : public ConfigSig { void set_verifier(ConfigMessage::verify_callable v) override { verifier_ = std::move(v); } void set_signer(ConfigMessage::sign_callable s) override { signer_ = std::move(s); } - std::vector sign(std::span data) const; + std::vector sign(std::span data) const; // Checks for and drops expired keys. void remove_expired(); // Loads existing state from a previous dump of keys data - void load_dump(std::span dump); + void load_dump(std::span dump); // Inserts a key into the correct place in `keys_`. void insert_key(std::string_view message_hash, key_info&& key); // Returned the blinding factor for a given session X25519 pubkey. This depends on the group's // seed and thus is only obtainable by an admin account. - std::array subaccount_blind_factor( - const std::array& session_xpk) const; + b32 subaccount_blind_factor(std::span session_xpk) const; public: /// The multiple of members keys we include in the message; we add junk entries to the key list @@ -192,10 +191,10 @@ class Keys : public ConfigSig { /// - `dumped` -- either `std::nullopt` to construct a new, empty object; or binary state data /// that was previously dumped from an instance of this class by calling `dump()`. /// - `info` and `members` -- will be loaded with the group keys, if present in the dump. - Keys(std::span user_ed25519_secretkey, - std::span group_ed25519_pubkey, - std::optional> group_ed25519_secretkey, - std::optional> dumped, + Keys(const ed25519::PrivKeySpan& user_ed25519_secretkey, + std::span group_ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& group_ed25519_secretkey, + std::optional> dumped, Info& info, Members& members); @@ -232,8 +231,8 @@ class Keys : public ConfigSig { /// Inputs: none. /// /// Outputs: - /// - `std::vector>` - vector of encryption keys. - std::vector> group_keys() const; + /// - `std::vector>` - vector of encryption keys. + std::vector> group_keys() const; /// API: groups/Keys::size /// @@ -258,8 +257,8 @@ class Keys : public ConfigSig { /// Inputs: none. /// /// Outputs: - /// - `std::span` of the most current group encryption key. - std::span group_enc_key() const; + /// - `std::span` of the most current group encryption key. + std::span group_enc_key() const; /// API: groups/Keys::is_admin /// @@ -269,7 +268,7 @@ class Keys : public ConfigSig { /// /// Outputs: /// - `true` if this object knows the group's master key - bool admin() const { return _sign_sk && _sign_pk; } + bool admin() const { return !_sign_sk.empty() && _sign_pk; } /// API: groups/Keys::load_admin_key /// @@ -292,7 +291,7 @@ class Keys : public ConfigSig { /// /// Outputs: nothing. After a successful call, `admin()` will return true. Throws if the given /// secret key does not match the group's pubkey. - void load_admin_key(std::span secret, Info& info, Members& members); + void load_admin_key(const ed25519::PrivKeySpan& secret, Info& info, Members& members); /// API: groups/Keys::rekey /// @@ -325,12 +324,12 @@ class Keys : public ConfigSig { /// config will be dirtied after the rekey and will require a push. /// /// Outputs: - /// - `std::span` containing the data that needs to be pushed to the config + /// - `std::span` containing the data that needs to be pushed to the config /// keys namespace /// for the group. (This can be re-obtained from `pending_config()` if needed until it has /// been confirmed or superceded). This data must be consumed or copied from the returned /// string_view immediately: it will not be valid past other calls on the Keys config object. - std::span rekey(Info& info, Members& members); + std::span rekey(Info& info, Members& members); /// API: groups/Keys::key_supplement /// @@ -352,11 +351,11 @@ class Keys : public ConfigSig { /// Session IDs are specified in hex. /// /// Outputs: - /// - `std::vector` containing the message that should be pushed to the swarm + /// - `std::vector` containing the message that should be pushed to the swarm /// containing encrypted /// keys for the given user(s). - std::vector key_supplement(const std::vector& sids) const; - std::vector key_supplement(std::string sid) const { + std::vector key_supplement(const std::vector& sids) const; + std::vector key_supplement(std::string sid) const { return key_supplement(std::vector{{std::move(sid)}}); } @@ -386,7 +385,7 @@ class Keys : public ConfigSig { /// delete messages without having the full admin group keys. /// /// Outputs: - /// - `std::vector` -- contains a subaccount swarm signing value; this can be + /// - `std::vector` -- contains a subaccount swarm signing value; this can be /// passed (by the user) /// into `swarm_subaccount_sign` to sign a value suitable for swarm authentication. /// (Internally this packs the flags, blinding factor, and group admin signature together and @@ -398,7 +397,7 @@ class Keys : public ConfigSig { /// /// The signing value produced will be the same (for a given `session_id`/`write`/`del` /// values) when constructed by any admin of the group. - std::vector swarm_make_subaccount( + std::vector swarm_make_subaccount( std::string_view session_id, bool write = true, bool del = false) const; /// API: groups/Keys::swarm_verify_subaccount @@ -432,14 +431,12 @@ class Keys : public ConfigSig { /// not validate or does not meet the requirements. static bool swarm_verify_subaccount( std::string group_id, - std::span session_ed25519_secretkey, - std::span signing_value, + const ed25519::PrivKeySpan& session_ed25519_secretkey, + std::span signing_value, bool write = false, bool del = false); bool swarm_verify_subaccount( - std::span signing_value, - bool write = false, - bool del = false) const; + std::span signing_value, bool write = false, bool del = false) const; /// API: groups/Keys::swarm_auth /// @@ -487,8 +484,8 @@ class Keys : public ConfigSig { /// - struct containing three binary values enabling swarm authentication (see description /// above). swarm_auth swarm_subaccount_sign( - std::span msg, - std::span signing_value, + std::span msg, + std::span signing_value, bool binary = false) const; /// API: groups/Keys::swarm_subaccount_token @@ -510,23 +507,23 @@ class Keys : public ConfigSig { /// /// Outputs: /// - 36 byte token that can be used for swarm token revocation. - std::vector swarm_subaccount_token( + std::vector swarm_subaccount_token( std::string_view session_id, bool write = true, bool del = false) const; /// API: groups/Keys::pending_config /// /// If a rekey has been performed but not yet confirmed then this will contain the config /// message to be pushed to the swarm. If there is no push current pending then this returns - /// nullopt. The value should be used immediately (i.e. the std::span may + /// nullopt. The value should be used immediately (i.e. the std::span may /// not remain valid if other calls to the config object are made). /// /// Inputs: None /// /// Outputs: - /// - `std::optional>` -- returns a populated config message that + /// - `std::optional>` -- returns a populated config message that /// should be pushed, /// if not yet confirmed, otherwise when no pending update is present this returns nullopt. - std::optional> pending_config() const; + std::optional> pending_config() const; /// API: groups/Keys::pending_key /// @@ -540,11 +537,11 @@ class Keys : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::optional>` the encryption key generated by the last + /// - `std::optional>` the encryption key generated by the last /// `rekey()` call. /// This is set to a new key when `rekey()` is called, and is cleared when any config message /// is successfully loaded by `load_key`. - std::optional> pending_key() const; + std::optional> pending_key() const; /// API: groups/Keys::load_key /// @@ -579,7 +576,7 @@ class Keys : public ConfigSig { /// it could mean we decrypted one for us, but already had it. bool load_key_message( std::string_view hash, - std::span data, + std::span data, int64_t timestamp_ms, Info& info, Members& members); @@ -648,7 +645,7 @@ class Keys : public ConfigSig { /// Outputs: /// - opaque binary data containing the group keys and other Keys config data that can be passed /// to the `Keys` constructor to reinitialize a Keys object with the current state. - std::vector dump(); + std::vector dump(); /// API: groups/Keys::make_dump /// @@ -659,8 +656,8 @@ class Keys : public ConfigSig { /// Inputs: None /// /// Outputs: - /// - `std::vector` -- Returns binary data of the state dump - std::vector make_dump() const; + /// - `std::vector` -- Returns binary data of the state dump + std::vector make_dump() const; /// API: groups/Keys::encrypt_message /// @@ -687,10 +684,8 @@ class Keys : public ConfigSig { /// /// Outputs: /// - `ciphertext` -- the encrypted, etc. value to send to the swarm - std::vector encrypt_message( - std::span plaintext, - bool compress = true, - size_t padding = 256) const; + std::vector encrypt_message( + std::span plaintext, bool compress = true, size_t padding = 256) const; /// API: groups/Keys::decrypt_message /// @@ -707,7 +702,7 @@ class Keys : public ConfigSig { /// by `encrypt_message()`. /// /// Outputs: - /// - `std::pair>` -- the session ID (in hex) and the + /// - `std::pair>` -- the session ID (in hex) and the /// plaintext binary /// data that was encrypted. /// @@ -715,8 +710,8 @@ class Keys : public ConfigSig { /// some diagnostic info on what part failed. Typically a production session client would catch /// (and possibly log) but otherwise ignore such exceptions and just not process the message if /// it throws. - std::pair> decrypt_message( - std::span ciphertext) const; + std::pair> decrypt_message( + std::span ciphertext) const; }; } // namespace session::config::groups diff --git a/include/session/config/groups/members.hpp b/include/session/config/groups/members.hpp index 0ea32b001..1d0a8c742 100644 --- a/include/session/config/groups/members.hpp +++ b/include/session/config/groups/members.hpp @@ -310,9 +310,9 @@ class Members : public ConfigBase { /// push config changes. /// - `dumped` -- either `std::nullopt` to construct a new, empty object; or binary state data /// that was previously dumped from an instance of this class by calling `dump()`. - Members(std::span ed25519_pubkey, - std::optional> ed25519_secretkey, - std::optional> dumped); + Members(std::span ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: groups/Members::storage_namespace /// diff --git a/include/session/config/local.hpp b/include/session/config/local.hpp index 7c8e11598..febd1d6cc 100644 --- a/include/session/config/local.hpp +++ b/include/session/config/local.hpp @@ -44,8 +44,8 @@ class Local : public ConfigBase { /// /// Outputs: /// - `Local` - Constructor - Local(std::span ed25519_secretkey, - std::optional> dumped); + Local(const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: local/Local::storage_namespace /// @@ -86,12 +86,12 @@ class Local : public ConfigBase { /// Inputs: None /// /// Outputs: - /// - `std::tuple, std::vector>` - Returns a + /// - `std::tuple, std::vector>` - Returns a /// tuple containing /// - `seqno_t` -- sequence number of 0 - /// - `std::vector` -- empty data vector + /// - `std::vector` -- empty data vector /// - `std::vector` -- empty list of message hashes - std::tuple>, std::vector> push() + std::tuple>, std::vector> push() override { return {0, {}, {}}; }; diff --git a/include/session/config/namespaces.h b/include/session/config/namespaces.h index b96703233..f0e2b4903 100644 --- a/include/session/config/namespaces.h +++ b/include/session/config/namespaces.h @@ -24,6 +24,10 @@ typedef enum NAMESPACE { NAMESPACE_GROUP_INFO = 13, NAMESPACE_GROUP_MEMBERS = 14, + // Device group namespaces: + NAMESPACE_DEVICES = 21, + NAMESPACE_ACCOUNT_PUBKEYS = -21, + // The local config should never be pushed but this gives us a nice identifier for each config // type NAMESPACE_LOCAL = 9999, diff --git a/include/session/config/namespaces.hpp b/include/session/config/namespaces.hpp index 6945f70ab..89de2eec2 100644 --- a/include/session/config/namespaces.hpp +++ b/include/session/config/namespaces.hpp @@ -24,6 +24,10 @@ enum class Namespace : std::int16_t { GroupInfo = NAMESPACE_GROUP_INFO, GroupMembers = NAMESPACE_GROUP_MEMBERS, + // Device group namespaces: + Devices = NAMESPACE_DEVICES, + AccountPubkeys = NAMESPACE_ACCOUNT_PUBKEYS, + // The local config should never be pushed but this gives us a nice identifier for each config // type Local = NAMESPACE_LOCAL, diff --git a/include/session/config/pro.h b/include/session/config/pro.h index 05420f57e..4928f0729 100644 --- a/include/session/config/pro.h +++ b/include/session/config/pro.h @@ -13,7 +13,7 @@ extern "C" { typedef struct pro_pro_config pro_pro_config; struct pro_pro_config { - bytes64 rotating_privkey; + cbytes64 rotating_privkey; session_protocol_pro_proof proof; }; diff --git a/include/session/config/pro.hpp b/include/session/config/pro.hpp index ac37647aa..c629f97cb 100644 --- a/include/session/config/pro.hpp +++ b/include/session/config/pro.hpp @@ -21,7 +21,7 @@ class ProConfig { public: /// Rotating private key for the public key specified in the proof. On the wire we store the /// seed. At runtime we derive the full key for convenience. - cleared_uc64 rotating_privkey; + cleared_b64 rotating_privkey; /// A cryptographic proof for entitling an Ed25519 key to Session Pro ProProof proof; diff --git a/include/session/config/profile_pic.hpp b/include/session/config/profile_pic.hpp index 7c82b4573..2f73f51b6 100644 --- a/include/session/config/profile_pic.hpp +++ b/include/session/config/profile_pic.hpp @@ -11,9 +11,9 @@ struct profile_pic { static constexpr size_t MAX_URL_LENGTH = 223; std::string url; - std::vector key; + std::vector key; - static void check_key(std::span key) { + static void check_key(std::span key) { if (!(key.empty() || key.size() == 32)) throw std::invalid_argument{"Invalid profile pic key: 32 bytes required"}; } @@ -22,13 +22,13 @@ struct profile_pic { profile_pic() = default; // Constructs from a URL and key. Key must be empty or 32 bytes. - profile_pic(std::string_view url, std::span key) : - url{url}, key{to_vector(key)} { + profile_pic(std::string_view url, std::span key) : + url{url}, key{to_vector(key)} { check_key(this->key); } - // Constructs from a string/std::vector pair moved into the constructor - profile_pic(std::string&& url, std::vector&& key) : + // Constructs from a string/std::vector pair moved into the constructor + profile_pic(std::string&& url, std::vector&& key) : url{std::move(url)}, key{std::move(key)} { check_key(this->key); } @@ -66,7 +66,7 @@ struct profile_pic { /// /// Inputs: /// - `new_key` -- binary data of a new key to be set. Must be 32 bytes - void set_key(std::vector new_key) { + void set_key(std::vector new_key) { check_key(new_key); key = std::move(new_key); } diff --git a/include/session/config/protos.hpp b/include/session/config/protos.hpp index 2880aed38..e8f59c1ae 100644 --- a/include/session/config/protos.hpp +++ b/include/session/config/protos.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include "namespaces.hpp" @@ -21,9 +22,9 @@ namespace session::config::protos { /// Outputs: /// Returns the wrapped config. Will throw on serious errors (e.g. `ed25519_sk` or `ns` are /// invalid). -std::vector wrap_config( - std::span ed25519_sk, - std::span data, +std::vector wrap_config( + const ed25519::PrivKeySpan& ed25519_sk, + std::span data, int64_t seqno, config::Namespace ns); @@ -44,9 +45,9 @@ std::vector wrap_config( /// Throws a std::invalid_argument if the given ed25519_sk is invalid. (It is recommended that only /// the std::runtime_error is caught for detecting non-wrapped input as the invalid secret key is /// more serious). -std::vector unwrap_config( - std::span ed25519_sk, - std::span data, +std::vector unwrap_config( + const ed25519::PrivKeySpan& ed25519_sk, + std::span data, config::Namespace ns); } // namespace session::config::protos diff --git a/include/session/config/user_groups.hpp b/include/session/config/user_groups.hpp index 8a60913bb..183d02076 100644 --- a/include/session/config/user_groups.hpp +++ b/include/session/config/user_groups.hpp @@ -100,8 +100,8 @@ struct base_group_info { /// Struct containing legacy group info (aka "groups"). struct legacy_group_info : base_group_info { std::string session_id; // The legacy group "session id" (33 bytes). - std::vector enc_pubkey; // bytes (32 or empty) - std::vector enc_seckey; // bytes (32 or empty) + std::vector enc_pubkey; // bytes (32 or empty) + std::vector enc_seckey; // bytes (32 or empty) std::chrono::seconds disappearing_timer{0}; // 0 == disabled. /// Constructs a new legacy group info from an id (which must look like a session_id). Throws @@ -191,7 +191,7 @@ struct group_info : base_group_info { // (to distinguish it from a 05 x25519 pubkey session id). /// Group secret key (64 bytes); this is only possessed by admins. - std::vector secretkey; + std::vector secretkey; /// Group authentication signing value (100 bytes); this is used by non-admins to authenticate /// (using the swarm key generation functions in config::groups::Keys). This value will be @@ -199,7 +199,7 @@ struct group_info : base_group_info { /// is an admin), and so does not need to be explicitly cleared when being promoted to admin. /// /// Producing and using this value is done with the groups::Keys `swarm` methods. - std::vector auth_data; + std::vector auth_data; /// Tracks why we were removed from the group. Values are: /// - NOT_REMOVED: that we haven't been removed, @@ -282,8 +282,8 @@ class UserGroups : public ConfigBase { /// Outputs: /// - `UserGroups` - Constructor UserGroups( - std::span ed25519_secretkey, - std::optional> dumped); + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: user_groups/UserGroups::storage_namespace /// @@ -368,7 +368,7 @@ class UserGroups : public ConfigBase { /// std::string_view room, /// std::string_view pubkey_encoded) const; /// community_info get_or_construct_community( - /// std::string_view base_url, std::string_view room, std::span + /// std::string_view base_url, std::string_view room, std::span /// pubkey) const; /// ``` /// @@ -394,7 +394,7 @@ class UserGroups : public ConfigBase { community_info get_or_construct_community( std::string_view base_url, std::string_view room, - std::span pubkey) const; + std::span pubkey) const; /// API: user_groups/UserGroups::get_or_construct_community(string_view) /// @@ -470,7 +470,7 @@ class UserGroups : public ConfigBase { protected: // Drills into the nested dicts to access open group details DictFieldProxy community_field( - const community_info& og, std::span* get_pubkey = nullptr) const; + const community_info& og, std::span* get_pubkey = nullptr) const; void set_base(const base_group_info& bg, DictFieldProxy& info) const; diff --git a/include/session/config/user_profile.h b/include/session/config/user_profile.h index 8177665e7..a95761063 100644 --- a/include/session/config/user_profile.h +++ b/include/session/config/user_profile.h @@ -226,6 +226,52 @@ LIBSESSION_EXPORT int user_profile_get_nts_expiry(const config_object* conf); /// - `expiry` -- [in] Integer of the expiry timer in seconds LIBSESSION_EXPORT void user_profile_set_nts_expiry(config_object* conf, int expiry); +/// API: user_profile/user_profile_get_nts_delete_before +/// +/// Gets the "delete before" unix timestamp (seconds) for the "Note to Self" conversation: messages +/// in it older than this are to be deleted, and arriving ones older than it dropped. Returns 0 if +/// no such instruction is set. +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// +/// Outputs: +/// - `int64_t` -- the timestamp, or 0 if unset +LIBSESSION_EXPORT int64_t user_profile_get_nts_delete_before(const config_object* conf); + +/// API: user_profile/user_profile_set_nts_delete_before +/// +/// Sets the "delete before" unix timestamp (seconds) for the "Note to Self" conversation. Pass 0 +/// (or a negative value) to clear it. +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// - `before` -- [in] unix timestamp (seconds) before which messages are to be deleted +LIBSESSION_EXPORT void user_profile_set_nts_delete_before(config_object* conf, int64_t before); + +/// API: user_profile/user_profile_get_nts_delete_attach_before +/// +/// As `user_profile_get_nts_delete_before`, but covering the attachments alone: the messages +/// themselves stay. Returns 0 if no such instruction is set. +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// +/// Outputs: +/// - `int64_t` -- the timestamp, or 0 if unset +LIBSESSION_EXPORT int64_t user_profile_get_nts_delete_attach_before(const config_object* conf); + +/// API: user_profile/user_profile_set_nts_delete_attach_before +/// +/// Sets the "delete attachments before" unix timestamp (seconds) for the "Note to Self" +/// conversation. Pass 0 (or a negative value) to clear it. +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// - `before` -- [in] unix timestamp (seconds) before which attachments are to be deleted +LIBSESSION_EXPORT void user_profile_set_nts_delete_attach_before( + config_object* conf, int64_t before); + /// API: user_profile/user_profile_get_blinded_msgreqs /// /// Returns true if blinded message requests should be retrieved (from SOGS servers), false if they @@ -267,6 +313,46 @@ LIBSESSION_EXPORT int user_profile_get_blinded_msgreqs(const config_object* conf /// - `void` -- Returns Nothing LIBSESSION_EXPORT void user_profile_set_blinded_msgreqs(config_object* conf, int enabled); +/// API: user_profile/user_profile_get_notify_media_saved +/// +/// Returns true if we tell somebody when we save a file they sent us. True is the default, and +/// what an account that has never set this returns: Session's clients report it, so it is what a +/// sender expects. +/// +/// Declaration: +/// ```cpp +/// BOOL user_profile_get_notify_media_saved( +/// [in] const config_object* conf +/// ); +/// ``` +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// +/// Outputs: +/// - `bool` -- true to tell the sender +LIBSESSION_EXPORT bool user_profile_get_notify_media_saved(const config_object* conf); + +/// API: user_profile/user_profile_set_notify_media_saved +/// +/// Sets the above. +/// +/// Declaration: +/// ```cpp +/// VOID user_profile_set_notify_media_saved( +/// [in] config_object* conf, +/// [in] bool notify +/// ); +/// ``` +/// +/// Inputs: +/// - `conf` -- [in] Pointer to the config object +/// - `notify` -- [in] false to stop telling senders that we saved their files +/// +/// Outputs: +/// - `void` -- Returns Nothing +LIBSESSION_EXPORT void user_profile_set_notify_media_saved(config_object* conf, bool notify); + /// API: user_profile/user_profile_get_profile_updated /// /// Returns the timestamp that the user last updated their profile information; or `0` if it's @@ -348,9 +434,9 @@ LIBSESSION_EXPORT bool user_profile_remove_pro_config(config_object* conf); /// - `conf` -- [in] Pointer to the config object /// /// Outputs: -/// - `session_protocol_pro_profile_bitset` - bitset indicating which profile features are enabled. -LIBSESSION_EXPORT session_protocol_pro_profile_bitset -user_profile_get_pro_features(const config_object* conf); +/// - `uint64_t` - bitset (mask of SESSION_PROTOCOL_PRO_PROFILE_FEATURE_* bits) indicating which +/// profile features are enabled. +LIBSESSION_EXPORT uint64_t user_profile_get_pro_features(const config_object* conf); /// API: user_profile/user_profile_set_pro_badge /// diff --git a/include/session/config/user_profile.hpp b/include/session/config/user_profile.hpp index e2f5f96ae..abfb78b7d 100644 --- a/include/session/config/user_profile.hpp +++ b/include/session/config/user_profile.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include "base.hpp" @@ -55,6 +56,16 @@ using namespace std::literals; /// when `T > t`). /// T - The unix timestamp (seconds) that the user last re-uploaded their profile information /// (automatically updates when calling `set_reupload_profile_pic`). +/// d - "delete before" unix timestamp (seconds) for the "Note to Self" pseudo-conversation: +/// messages in it older than this are to be deleted, and arriving ones older than it dropped. +/// Omitted when 0. Named to match the equivalent field in the contacts and group info configs; +/// note to self needs its own because it has no contacts entry to carry one. +/// x - set to 1 to suppress the notification that tells someone we saved a file they sent. Omitted +/// when we do send them, which is the default and what nearly every account will carry -- hence +/// a key that is absent rather than a value that is false, so the common case costs nothing in +/// every push. Note this is the *negative*: present means do not tell them. +/// D - "delete attachments before" unix timestamp (seconds) for "Note to Self": as above but for +/// attachments alone, leaving the messages themselves. Omitted when 0. class UserProfile : public ConfigBase { public: friend class UserProfileTester; @@ -79,8 +90,8 @@ class UserProfile : public ConfigBase { /// Outputs: /// - `UserProfile` - Constructor UserProfile( - std::span ed25519_secretkey, - std::optional> dumped); + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped); /// API: user_profile/UserProfile::storage_namespace /// @@ -148,7 +159,7 @@ class UserProfile : public ConfigBase { /// /// Declaration: /// ```cpp - /// void set_profile_pic(std::string_view url, std::span key); + /// void set_profile_pic(std::string_view url, std::span key); /// void set_profile_pic(profile_pic pic); /// ``` /// @@ -158,7 +169,7 @@ class UserProfile : public ConfigBase { /// - `key` -- Decryption key /// - Second function: /// - `pic` -- Profile pic object - void set_profile_pic(std::string_view url, std::span key); + void set_profile_pic(std::string_view url, std::span key); void set_profile_pic(profile_pic pic); /// API: user_profile/UserProfile::set_reupload_profile_pic @@ -167,7 +178,7 @@ class UserProfile : public ConfigBase { /// /// Declaration: /// ```cpp - /// void set_reupload_profile_pic(std::string_view url, std::span key); + /// void set_reupload_profile_pic(std::string_view url, std::span key); /// void set_reupload_profile_pic(profile_pic pic); /// ``` /// @@ -177,7 +188,7 @@ class UserProfile : public ConfigBase { /// - `key` -- Decryption key /// - Second function: /// - `pic` -- Profile pic object - void set_reupload_profile_pic(std::string_view url, std::span key); + void set_reupload_profile_pic(std::string_view url, std::span key); void set_reupload_profile_pic(profile_pic pic); /// API: user_profile/UserProfile::get_nts_priority @@ -221,6 +232,54 @@ class UserProfile : public ConfigBase { /// - `timer` -- Default to 0 seconds, will set the expiry timer void set_nts_expiry(std::chrono::seconds timer = 0s); + /// API: user_profile/UserProfile::get_nts_delete_before + /// + /// Returns the "delete before" timestamp for the Note-to-self conversation: messages in it + /// older than this are to be deleted, and arriving ones older than it dropped. This is what + /// makes clearing that conversation mean the same thing on every device -- the instruction is + /// recorded rather than inferred from when some config happened to be written. + /// + /// Note to self needs its own because it has no contacts entry to carry one. + /// + /// Inputs: None + /// + /// Outputs: + /// - `std::chrono::sys_seconds` -- the timestamp, or the epoch if no such instruction is set + std::chrono::sys_seconds get_nts_delete_before() const; + + /// API: user_profile/UserProfile::set_nts_delete_before + /// + /// Sets the Note-to-self "delete before" timestamp. Pass the epoch (or a non-positive time) to + /// clear it. + /// + /// Inputs: + /// - `before` -- messages older than this are to be deleted + void set_nts_delete_before(std::chrono::sys_seconds before); + + /// API: user_profile/UserProfile::get_nts_delete_attach_before + /// + /// As `get_nts_delete_before`, but covering the attachments alone: the messages themselves + /// stay. + /// + /// Only ever holds a value that says something `get_nts_delete_before` does not: deleting a + /// message takes its attachments with it, so setting either of the pair clears this one when + /// the message instruction already covers it. + /// + /// Inputs: None + /// + /// Outputs: + /// - `std::chrono::sys_seconds` -- the timestamp, or the epoch if no such instruction is set + std::chrono::sys_seconds get_nts_delete_attach_before() const; + + /// API: user_profile/UserProfile::set_nts_delete_attach_before + /// + /// Sets the Note-to-self "delete attachments before" timestamp. Pass the epoch (or a + /// non-positive time) to clear it. + /// + /// Inputs: + /// - `before` -- attachments older than this are to be deleted + void set_nts_delete_attach_before(std::chrono::sys_seconds before); + /// API: user_profile/UserProfile::get_blinded_msgreqs /// /// Accesses whether or not blinded message requests are enabled for the client. Can have three @@ -250,6 +309,36 @@ class UserProfile : public ConfigBase { /// default). void set_blinded_msgreqs(std::optional enabled); + /// API: user_profile/UserProfile::get_notify_media_saved + /// + /// Whether to tell somebody that we saved a file they sent us. True by default, and for an + /// account that has never set it either way: Session's clients report it, so it is what a + /// sender expects. + /// + /// Whether that notification should be sent is a privacy decision rather than a technical one, + /// and it belongs to the person rather than to the device they happen to be holding — which is + /// why it lives here, where it follows the account, and not in a device-local config. + /// + /// A plain bool rather than the tri-state `get_blinded_msgreqs` uses: "never asked" and + /// "explicitly wants the default" are the same instruction, and keeping them apart would only + /// matter if the default were ever to flip — which would be a decision taken across every + /// client at once, where having accounts that never expressed a preference move with it is + /// exactly what you would want. + /// + /// Inputs: None + /// + /// Outputs: + /// - `bool` -- true to tell the sender, which is the default. + bool get_notify_media_saved() const; + + /// API: user_profile/UserProfile::set_notify_media_saved + /// + /// Sets the above. + /// + /// Inputs: + /// - `notify` -- false to stop telling senders that we saved their files. + void set_notify_media_saved(bool notify); + /// API: user_profile/UserProfile::get_profile_updated /// /// Returns the timestamp that the user last updated their profile information; or `0` if it's @@ -294,18 +383,18 @@ class UserProfile : public ConfigBase { /// - `bool` - Flag indicating whether the config had Session Pro config removed or not. bool remove_pro_config(); - /// API: user_profile/UserProfile::get_pro_features + /// API: user_profile/UserProfile::get_profile_flags /// - /// Retrieves the bitset indicating which pro features the user currently has enabled. + /// Retrieves the flags indicating which pro features the user currently has enabled. /// /// Inputs: None /// /// Outputs: - /// - Bitset with individual bits set on it corresponding to - /// SESSION_PROTOCOL_PRO_PROFILE_FEATURES_BITSET. It is possible to receive bits set that don't - /// have a corresponding enum value if you are receiving a bitset from a newer client with newer - /// features enabled. These flags should be ignored by clients that do not recognise them. - ProProfileBitset get_profile_bitset() const; + /// - `ProProfileFlags` with the individual `ProProfileFlags::*` bits set that the user has + /// enabled. It is possible to receive bits set that don't have a corresponding enumerator if + /// you are receiving flags from a newer client with newer features enabled; unrecognised bits + /// should be ignored. + ProProfileFlags get_profile_flags() const; /// API: user_profile/UserProfile::set_pro_badge /// @@ -336,9 +425,9 @@ class UserProfile : public ConfigBase { /// Inputs: None /// /// Outputs: - /// - `std::optional` - The unix timestamp in + /// - `std::optional` - The unix timestamp in /// seconds that the users pro access will expire, or nullopt if unset. - std::optional get_pro_access_expiry() const; + std::optional get_pro_access_expiry() const; /// API: user_profile/UserProfile::set_pro_access_expiry /// @@ -347,7 +436,7 @@ class UserProfile : public ConfigBase { /// Inputs: /// - `access_expiry_ts` -- The timestamp (unix epoch seconds) that the users Session Pro access /// will expire, or nullopt to remove the value. - void set_pro_access_expiry(std::optional access_expiry_ts); + void set_pro_access_expiry(std::optional access_expiry_ts); /// API: user_profile/UserProfile::get_pro_auto_renewing /// @@ -412,9 +501,9 @@ class UserProfile : public ConfigBase { /// Inputs: None /// /// Outputs: - /// - `std::optional` - the unix timestamp (seconds) at which a refund was - /// requested, or nullopt if no refund has been requested (or the stored value is stale). - std::optional get_refund_requested() const; + /// - `std::optional` - the unix timestamp (seconds) at which a refund + /// was requested, or nullopt if no refund has been requested (or the stored value is stale). + std::optional get_refund_requested() const; /// API: user_profile/UserProfile::set_refund_requested /// @@ -426,7 +515,7 @@ class UserProfile : public ConfigBase { /// Inputs: /// - `when` -- the timestamp (unix epoch seconds) at which the refund was requested, or nullopt /// to clear the refund-requested state. - void set_refund_requested(std::optional when); + void set_refund_requested(std::optional when); /// API: user_profile/UserProfile::get_pro_prepaid /// @@ -440,7 +529,7 @@ class UserProfile : public ConfigBase { /// Outputs: /// - `std::optional` - the unix timestamp (seconds) at which a purchase was /// initiated, or nullopt if none is pending (or the stored value is stale). - std::optional get_pro_prepaid() const; + std::optional get_pro_prepaid() const; /// API: user_profile/UserProfile::set_pro_prepaid /// @@ -453,7 +542,7 @@ class UserProfile : public ConfigBase { /// Inputs: /// - `when` -- the timestamp (unix epoch seconds) at which the purchase was initiated, or /// nullopt to clear the marker. - void set_pro_prepaid(std::optional when); + void set_pro_prepaid(std::optional when); /// API: user_profile/UserProfile::pro_renewal_target /// @@ -476,7 +565,13 @@ class UserProfile : public ConfigBase { /// /// Outputs: /// - `std::optional` - when to renew, or nullopt for "no renewal needed". - std::optional pro_renewal_target(sys_seconds now) const; + std::optional pro_renewal_target(std::chrono::sys_seconds now) const; + + private: + // Enables/disables a single profile feature flag in the synced "f" set. The set stores feature + // *bit positions*, so `flag` must be a single-bit ProProfileFlags value (deflated here to its + // position via countr_zero). + void set_profile_feature(ProProfileFlags flag, bool enabled); }; } // namespace session::config diff --git a/include/session/core.hpp b/include/session/core.hpp new file mode 100644 index 000000000..d1ee78625 --- /dev/null +++ b/include/session/core.hpp @@ -0,0 +1,642 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/callbacks.hpp" +#include "core/configs.hpp" +#include "core/devices.hpp" +#include "core/globals.hpp" +#include "core/pro.hpp" +#include "core/schema/schema_registry.hpp" +#include "session/network/key_types.hpp" +#include "session/network/service_node.hpp" + +/// The "Core" class holds a Session account's own state, in an encrypted sqlite database: its keys, +/// its device group, its configs, and the bookkeeping needed to talk to the network on its behalf. +/// +/// Core was once meant to be the whole data model -- conversations, contacts and messages included +/// -- and the split into Core and `session::client::Client` divided that job rather than shrinking +/// it. The two together are what Core alone was originally envisioned to be, so a reader looking +/// for conversations or message history should look up, not deeper. +/// +/// Core is the primitive layer, and deliberately stops short of interpretation. It holds the +/// account's keys, talks to swarms, signs and authenticates requests, encrypts and decrypts, and +/// stores what it must to do those things. What it does *not* do is decide what any of it means: a +/// message that arrives is handed up as an authenticated sender plus a span of decrypted bytes, and +/// nothing in Core parses those bytes, knows what a conversation is, or can tell an outgoing +/// message from an incoming one. +/// +/// That is `session::client::Client`'s work. The dividing line is worth stating because it is easy +/// to erode one convenience at a time: if a question can be answered without knowing what a message +/// *says*, it belongs here; the moment answering it requires reading the payload, it belongs above. +/// +/// The apparent exception proves it — Core does parse its own namespaces, the device group and the +/// account keys, because that state *is* Core's rather than a conversation's. +/// +/// Core can drive its own network once one is attached with set_network(), after which it polls the +/// account's swarms on a timer. Without one it can still be fed inbound messages through +/// receive_messages(), but it cannot send: a send with no network attached reports no_network. +/// +/// The typical intended flow for using the Core is to construct it early in the application and +/// store it for the application duration: +/// +/// session::core::Core core{ +/// std::filesystem::path{"/path/to/libsession.db"}, +/// session::sqlite::argon2id_password{"user-supplied password"} +/// }; +/// +/// (Or keep it in a unique or shared ptr if you have more complex ownership needs). +/// +/// The above examples shows a usage where the database is: +/// - encrypted using AEGIS-256 +/// - encrypted using a password resulting from argon2id password hashing +/// - is fully encrypted, with the first 16 bytes of the file containing the password salt. +/// +/// The above can be modified to pass any of the options supported by session::sqlite::Database, but +/// some of the most common ones are depicted here. +/// +/// If you have secure storage of a 32 byte secure random value (for example, 32 bytes generated by +/// libsodium's randombytes_buf) then instead of the argon2id_password argument you can pass a +/// raw_key: +/// +/// session::core::Core core{ +/// std::filesystem::path{"/path/to/libsession.db"}, +/// session::sqlite::raw_key{key} +/// }; +/// +/// If you have such a raw key that you can keep secure, this will open the database substantially +/// faster than needing to perform an argon2id hash. DO NOT use this approach with a user-supplied +/// password. +/// +/// (Note that the above makes a secure copy when opening the database. If you cannot avoid making +/// a copy yourself, remember to copy into at least a `session::cleared_b32` or use similar secure +/// clearing after use to ensure the key bytes does not remain in unused process memory). +/// +/// If you want password protection instead of secure raw key protection then replace +/// +/// session::sqlite::raw_key{key} +/// +/// with +/// +/// session::sqlite::argon2id_password{user_pass} +/// +/// You can optionally crank up the argon2 settings for a more secure (but slower to open) database +/// by adding additional arguments to the argon2id_password constructor; see the session-sqlite +/// documentation for details. +/// +/// For iOS, where full encryption causes the OS to kill the process but opens up a magic special +/// snowflake exception if it thinks your file is SQLite, you can pass an extra argument value: +/// +/// session::sqlite::plaintext_header +/// +/// which will cause the initial 24 bytes of the file to be unencrypted, allowing the file to pass +/// iOS's sniff test to get magic permissions. Note that if you combine this with argon2id_password +/// you must also create and store a session::sqlite::salt value and pass that every time the +/// database is opened. The salt should be random bytes generated by a cryptographically secure +/// RNG, but can be stored without encryption once generated (i.e. it is not a sensitive value). +/// +/// Example construction: +/// +/// ```C++ +/// #include +/// #include +/// +/// int main() { +/// +/// session::core::Core core{ +/// std::filesystem::path{"/path/to/libsession.db"}, +/// session::sqlite::argon2id_password{"correct horse battery staple"} +/// }; +/// +/// run_my_app(core); +/// } +/// +/// +/// // Later on somewhere deep inside `run_my_app` when an updated revocation list is received: +/// try { +/// core.pro.update_revocations(...); +/// } catch (const std::exception& e) { +/// log::warning(cat, "Failed to update Pro revocation list: {}", e.what()); +/// } +/// +/// // When checking to verify a Pro account proof: +/// if (!core.pro.proof_is_revoked(...)) { +/// // bro is Pro! +/// } +/// +/// ``` + +namespace oxen::quic { +struct Ticker; +} // namespace oxen::quic + +namespace session::pro_backend { +struct ProRevocationItem; +}; // namespace session::pro_backend + +namespace session::network { +class Network; +} + +namespace session { +class TestHelper; +} + +// Forward declared rather than including SessionProtos.pb.h so that consumers of this header do not +// inherit a protobuf include dependency; only the send_dm() Content overload needs the full type. +namespace SessionProtos { +class Content; +} + +namespace session::core { + +using namespace std::literals; +namespace quic = oxen::quic; + +namespace detail { + class CoreComponent; + + /// Extracts an option of type T from a pack; returns the first match wrapped in optional, or + /// nullopt if not present. Mirrors the same helper in session::sqlite::Database. + /// + /// Public so that a layer wrapping the Core constructor (such as session::client::Client, which + /// must find and chain the caller's `callbacks` before forwarding the rest) can pick options + /// out of the pack without duplicating this. + template + constexpr auto maybe_instance(Opts&&... opts) { + using Ret = std::optional; + if constexpr (sizeof...(Opts) == 0) + return Ret{std::nullopt}; + else { + auto finder = []( + auto&& self, Opt&& o, More&&... more) -> Ret { + if constexpr (std::same_as, T>) + return std::make_optional(std::forward(o)); + else if constexpr (sizeof...(More) > 0) + return self(self, std::forward(more)...); + else + return std::nullopt; + }; + return finder(finder, std::forward(opts)...); + } + } + + /// Collects every option of type T from a pack, in the order given. Unlike maybe_instance this + /// is for options that may meaningfully be repeated. + template + std::vector all_instances(Opts&&... opts) { + std::vector found; + ( + [&](Opt&& o) { + if constexpr (std::same_as, T>) + found.push_back(std::forward(o)); + }(std::forward(opts)), + ...); + return found; + } +} // namespace detail + +/// Wraps a predefined 32-byte account seed to pass to the Core constructor, overriding any seed +/// already stored in the database. Used when restoring an existing account from a seed. +struct predefined_seed { + cleared_b32 bytes; + explicit predefined_seed(std::span s) { + std::ranges::copy(s, bytes.begin()); + } + explicit predefined_seed(std::span s) { + std::ranges::copy(std::as_bytes(s), bytes.begin()); + } + + /// Constructs a predefined_seed from a mnemonic word list. + /// + /// Accepts 12 or 13 words (128-bit seed; the upper 16 bytes are set to zero), or 24 or 25 + /// words (256-bit seed). 13- and 25-word inputs include a checksum word which is validated. + /// + /// @throws std::invalid_argument if the word count is not 12, 13, 24, or 25. + /// @throws mnemonics::unknown_word_error if a word is not found in the language dictionary. + /// @throws mnemonics::checksum_error if the checksum word (if present) does not match. + explicit predefined_seed( + std::span words, const mnemonics::Mnemonics& lang); + explicit predefined_seed( + std::span words, std::string_view lang_name = "English"); +}; + +/// Opens the account without inventing an identity for it. +/// +/// Ordinarily a Core with no stored seed and no predefined_seed generates one, which gives an +/// application no way to *ask* whether a database already holds an account: finding out has already +/// answered it. With this option that database opens with no account, globals.have_account() +/// reports which case it is, and the application resolves it with globals.create_account() or +/// globals.restore_account(). +/// +/// Between construction and that call the account has no identity, so anything needing one -- +/// session_id(), account_seed(), send_dm(), attaching a network -- throws no_account. Nothing +/// else in Core needs it that early: components initialise fine without it and polling does not +/// start until a network is attached. +struct defer_account {}; + +/// Additional database migrations to apply during Core construction, on behalf of a layer built on +/// top of Core (such as a Client holding conversations and message history in the same database). +/// +/// May be passed more than once, and each set is applied in the order given, after all of Core's +/// own migrations. Two consequences of that ordering: +/// +/// - Extension migrations may depend on Core's tables; Core's may never depend on an extension's. +/// - A Core migration added later runs *after* an extension's on an already-populated database, +/// but before it on a fresh one. Only the rule above makes both orders safe. +/// +/// Extension migrations run from apply_migrations(), i.e. before any Core component's init() has +/// run: the Core they are handed is constructed but not yet initialised, so (for example) the +/// account seed is not loaded and globals::session_id() is unavailable. The layer that supplied +/// them does not exist at all yet. +struct schema_extension { + /// Distinguishes these migrations from Core's and from other extensions' in the + /// migrations_applied table. Must be non-empty, must not contain ':', and must never change + /// once released: it forms part of the recorded identity of an applied migration, so changing + /// it re-runs the entire set. + std::string_view owner; + + /// The migrations themselves. Not copied, but only referenced while the Core constructor + /// runs, so this need only outlive construction. + std::span migrations; + + /// Optional: the schema with every one of the above applied, as generated into the registry's + /// FULL_SCHEMA. When a database has none of this owner's migrations applied it is built from + /// this in one step and they are all recorded without running, which is both faster than + /// replaying the chain and the reason a readable current schema can exist at all. + /// + /// Anything a migration does beyond DDL — seeding rows, say — must therefore also be in here, + /// since on a fresh database that migration never runs. + std::string_view full_schema; +}; + +/// Concept satisfied by any type usable as a Core constructor option: a sqlite database option +/// (encryption, behaviour), or a Core-specific option tag (predefined_seed, callbacks, +/// schema_extension). All options can be passed in any order after the db_path positional +/// argument. +template +concept CoreOption = sqlite::DatabaseOption> || + std::same_as, predefined_seed> || + std::same_as, callbacks> || + std::same_as, defer_account> || + std::same_as, schema_extension>; + +class Core { + friend class session::TestHelper; // for unit tests + + // Declared first, so it is constructed first and destroyed last: it must outlive every + // component that uses it, and the poll ticker below. + quic::Loop _loop; + + // Singly owned: a Network must not be kept alive by anything else, least of all by a callback + // it hands out, so that its destructor never runs on its own loop thread. + // + // Custom deleter allows network::Network to remain an incomplete type in this header. + struct NetworkDeleter { + void operator()(network::Network*) const; + }; + std::unique_ptr _network; + + sqlite::Database db; + friend class detail::CoreComponent; + + core::callbacks callbacks; + + // Called during the constructor: the database is opened and all members are constructed, but + // `init()` hasn't called called yet. + void apply_migrations(); + + // Extra migration sets supplied via schema_extension options, in the order given. Cleared once + // applied, so that the spans they hold are not retained beyond construction. + std::vector _schema_extensions; + + std::list _comp_init; + void register_comp_init(detail::CoreComponent* c); + + // Performs the non-templated part of Core construction: this executes any needed database + // migrations, and then calls init() on each sub-component. + void init(); + + // Polling-related members and methods + std::chrono::milliseconds _poll_interval = 20s; + std::shared_ptr _poll_ticker; + void _update_polling(); + void _poll(); + + // Sends one round of retrieves to `node` for `namespaces`. A retrieve is capped by the storage + // server, so one round may not exhaust a namespace; `round` counts continuations and bounds + // them. Every round goes to the same node: the retrieve cursor is stored per (namespace, + // node), so continuing against a different swarm member would resume from that member's + // position. + void _send_poll( + network::Network* net, + network::service_node node, + std::vector namespaces, + int round); + void _handle_poll_response( + network::service_node node, + std::vector namespaces, + std::string body, + int round); + + // Decrypts and dispatches one-to-one messages from Namespace::Default. + void _handle_direct_messages(std::span messages); + + // Handles a PFS fetch response + void _handle_pfs_response(std::span sid, std::string body); + + // Stores PFS keys for a remote session_id in the pfs_key_cache. Returns true if the keys + // differ from the previously cached entry (or no entry existed). + bool _store_pfs_keys( + std::span session_id, + std::span x25519_pub, + std::span mlkem768_pub); + + // Stores a NAK (no keys found) for a remote session_id in the pfs_key_cache. + void _store_pfs_nak(std::span session_id); + + // Monotonic message ID counter for send_dm(). + int64_t _next_message_id{1}; + + // Queued sends waiting for a PFS key fetch to complete. + struct PendingSend { + int64_t id; + std::array recipient; + std::vector content; + sys_ms sent_timestamp; + std::optional pro_privkey; + std::chrono::milliseconds ttl; + bool force_v2; + }; + std::vector _pending_sends; + + // Drains pending sends whose PFS key fetch has completed. + void _flush_pending_sends(std::span session_id); + + // Called on every terminal outcome of a PFS key fetch: notifies the application and then + // releases any sends that were queued waiting on those keys. All fetch completion paths must + // go through here rather than firing the callback directly, or queued sends for that recipient + // are never dispatched. + void _pfs_fetch_done(std::span session_id, PfsKeyFetch result); + + // Encrypts, envelopes, and dispatches a single DM. Called from send_dm() and from + // _flush_pending_sends() when a queued send is ready. Fires the message_send_status + // callback on completion. + void _do_send_dm( + int64_t message_id, + std::span recipient, + std::span content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey, + std::chrono::milliseconds ttl, + bool force_v2); + + public: + /// Deletes messages from *our own* swarm by hash, and forgets the cursors that named them. + /// + /// Only our own: a delete is signed by the account that owns the swarm, so this can reach the + /// copy of a message we sent that lives in our swarm for our other devices, and the copy of a + /// message we received that was delivered to us. It cannot touch the copy sitting in someone + /// else's swarm — asking them to remove that is what an unsend request is for. + /// + /// Best effort, and `on_complete` says only whether the request was accepted. A storage server + /// that no longer holds a hash reports that rather than failing, which is the same outcome we + /// wanted; treating it as an error would make a second delete look broken. + /// + /// The hashes are removed from the cursor history as part of this, which is the whole reason + /// that history exists: the next retrieve from a node then measures from the newest hash we + /// still hold rather than from one the node has just been told to forget. + /// + /// Does nothing and reports success for an empty list. Throws std::logic_error if no network + /// is attached. + void delete_from_swarm( + std::vector hashes, std::function on_complete); + + private: + // Dispatches a fully-encoded payload to a swarm for storage via the attached network object, + // firing on_complete with the outcome and, where the storage server reported one, the hash it + // assigned the message. Throws std::logic_error if no network is attached. + void _send_to_swarm( + std::span dest_pubkey, + config::Namespace ns, + std::vector payload, + std::chrono::milliseconds ttl, + std::function swarm_hash)> + on_complete); + + // Constructs a sqlite::Database from the subset of opts that satisfy sqlite::DatabaseOption. + template + static sqlite::Database _make_db(std::filesystem::path path, Opts&&... opts) { + return std::apply( + [&](DBOpts&&... db_opts) { + return sqlite::Database{std::move(path), std::forward(db_opts)...}; + }, + std::tuple_cat([](T&& o) { + if constexpr (sqlite::DatabaseOption>) + return std::tuple>{std::forward(o)}; + else + return std::tuple<>{}; + }(std::forward(opts))...)); + } + + public: + // Constructor taking a db path and any mix of Core and database options in any order. + // - callbacks (optional, defaults to empty): event callbacks for the application + // - predefined_seed (optional): overrides any seed stored in the database + // - database options: see sqlite::DatabaseOption (encryption, behaviour flags, etc.) + template + Core(std::filesystem::path db_path, Opts&&... opts) : + callbacks{detail::maybe_instance(std::forward(opts)...) + .value_or(core::callbacks{})}, + db{_make_db(std::move(db_path), std::forward(opts)...)} { + _schema_extensions = detail::all_instances(std::forward(opts)...); + if (auto s = detail::maybe_instance(std::forward(opts)...)) + globals._predefined_seed = std::move(s->bytes); + globals._defer_account = + detail::maybe_instance(std::forward(opts)...).has_value(); + init(); + } + + /// Set an optional network interface that can be used to make network requests to swarm + /// members. Ownership is taken: nothing else may hold on to the Network. + void set_network(std::unique_ptr network); + + /// Constructs the network in place and attaches it, forwarding the arguments to its + /// constructor. Returns a reference to it, valid until it is replaced or this Core is + /// destroyed. + /// + /// core.make_network(session::network::config::Config{}); + /// + /// The usual way to call `set_network`: the object exists only to be owned here, so building it + /// and handing it over in one step spares the caller a `make_unique` — and spares it the chance + /// to keep a copy of what it has just given away, which the ownership rule above forbids. + /// + /// `N` names a subclass when there is one, which in practice means a test double: + /// + /// auto& net = core.make_network(); + /// + /// This is a template, so `N` has to be complete where it is called — which it is, since the + /// caller is naming its constructor. `session::network::Network` is only forward declared here. + template N = network::Network, typename... Args> + N& make_network(Args&&... args) { + auto net = std::make_unique(std::forward(args)...); + auto& ref = *net; + set_network(std::move(net)); + return ref; + } + + /// How long a cached PFS key is considered fresh (no re-fetch needed). + static constexpr auto PFS_KEY_FRESH_DURATION = 24h; + /// How long a cached PFS key is usable as a fallback before it expires entirely. + static constexpr auto PFS_KEY_EXPIRY_DURATION = 48h; + /// How long a NAK (successful fetch that returned no keys) suppresses re-fetching. + static constexpr auto PFS_KEY_NAK_DURATION = 1h; + + /// Checks and/or initiates a background fetch of the X25519 and ML-KEM-768 account public keys + /// for the given remote session_id (33-byte 0x05-prefixed X25519 pubkey), caching the result + /// in the pfs_key_cache table. + /// + /// Returns a PfsKeyStatus value describing the current cache state: + /// - fresh -- cached key is less than PFS_KEY_FRESH_DURATION (24h) old; no fetch initiated. + /// - stale -- cached key is 24–48h old; a background re-fetch has been initiated and the + /// pfs_keys_fetched callback will fire when it completes. + /// - fetching -- no usable key is cached; a background fetch has been initiated and the + /// pfs_keys_fetched callback will fire when it completes. + /// - nak -- a recent fetch returned no keys; re-fetching is suppressed for + /// PFS_KEY_NAK_DURATION (1h). + PfsKeyStatus prefetch_pfs_keys(std::span session_id); + + /// Sets the polling interval used when a network object is attached. The default is 20s. + /// + /// Takes effect by replacing the active ticker, if there is one, and so restarts the interval: + /// the next poll is a full `interval` away rather than at the point the old ticker would have + /// fired. Attaching a network polls immediately and then every `interval`, so setting this + /// before `set_network()` costs no initial delay. + /// + /// Callable from any thread and at any time: the work is marshalled onto the event loop, so it + /// may not have taken effect by the time this returns. + /// + /// The interval is deliberately the application's choice rather than something Core adapts, + /// because what it trades away is not Core's to spend -- a desktop or terminal client wants + /// new messages promptly, and a mobile one wants its battery. Note that nothing serialises + /// polls against each other: an interval short enough that a poll can still be outstanding + /// when the next one fires gets overlapping requests, which is wasteful but not incorrect + /// (the swarm cursor only advances once a batch has been handled, and message handlers are + /// already required to tolerate seeing a message twice). + void set_poll_interval(std::chrono::milliseconds interval); + + /// Encrypt and send a direct message to the given recipient. + /// + /// Returns a unique message_id that will later be reported via the message_send_status + /// callback. + /// + /// Version selection: + /// - If fresh/stale PFS+PQ account keys are cached for the recipient, a v2 PFS message is + /// sent. + /// - If no keys are cached (NAK) and force_v2 is false, falls back to a v1 message. + /// - If no keys are cached and force_v2 is true, sends a v2 non-PFS message. + /// - If a key fetch is in progress, the send is queued and dispatched when the fetch + /// completes, using the above rules. + /// + /// Parameters: + /// - recipient_session_id -- 33-byte 0x05-prefixed session ID of the recipient + /// - content -- serialised SessionProtos::Content protobuf (the inner plaintext) + /// - sent_timestamp -- the message timestamp as a time point (should match sigTimestamp + /// inside the Content protobuf); used in the v1 Envelope when falling back to v1 + /// - pro_privkey -- optional Session Pro rotating Ed25519 private key (32-byte seed or + /// 64-byte libsodium key). When provided, a Pro signature is attached to the message. + /// - ttl -- time-to-live for the stored message; defaults to 14 days + /// - force_v2 -- when true, never fall back to v1; use v2 non-PFS if no PFS keys are + /// available + int64_t send_dm( + std::span recipient_session_id, + std::span content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey = std::nullopt, + std::chrono::milliseconds ttl = 14 * 24h, + bool force_v2 = false); + + /// Overload of send_dm() taking an unserialised Content protobuf, which is serialised and + /// forwarded to the span version above. Callers using this overload need to include + /// and link libsession::protos themselves. + /// + /// This overload additionally maintains the invariant that the span version only documents: + /// the Content's `sigTimestamp` and `sent_timestamp` must agree, because the v1 fallback path + /// puts `sent_timestamp` in the (unauthenticated) Envelope while the signature covers the + /// `sigTimestamp` inside the Content. If `sigTimestamp` is unset it is filled in from + /// `sent_timestamp`; if it is set to a different value that is a caller bug and throws. + /// + /// @throws std::invalid_argument if content.sigTimestamp() is set and disagrees with + /// sent_timestamp. + int64_t send_dm( + std::span recipient_session_id, + const SessionProtos::Content& content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey = std::nullopt, + std::chrono::milliseconds ttl = 14 * 24h, + bool force_v2 = false); + + /// Returns the optional network interface, or nullptr if none is set. Non-owning: a caller + /// must not keep this beyond the point where the network could be replaced or dropped. + network::Network* network() const { return _network.get(); } + + /// The event loop this account's work runs on. + /// + /// Everything Core does off the caller's thread — polling, send completion, and therefore every + /// callback it fires — happens here. A layer above Core dispatches its own database work onto + /// it with `loop().call(...)` so that all access is serialised onto one thread, rather than + /// relying on the database being safe to touch from several. + /// + /// `call()` runs the job inline when the caller is already on this thread, so a single-threaded + /// application pays nothing for the indirection. + quic::Loop& loop(); + + /// The account database, for a layer built on top of Core that keeps its own tables alongside + /// Core's — the same layer that supplies a schema_extension to create them. + /// + /// A layer above Core may create and use its own tables here. It must not write Core's: + /// `namespace_sync`, `devices`, `globals` and the rest are Core's to maintain, and nothing + /// defends them against a well-meaning update from outside. + /// + /// `database().conn()` hands back the connection the calling thread already holds, if any, so a + /// write made from inside a Core callback joins the transaction Core has open rather than + /// deadlocking against it. That sharing is the whole reason this is exposed: opening a second + /// Database on the same file would not have that property. + /// + /// This grants no access to Core's own tables' invariants. Reading Core's tables is fine; + /// writing them behind Core's back is not, and Core does not defend against it. + sqlite::Database& database() { return db; } + + // Global value storage. This are used by some components, but can also be used by the + // application to persist settings. + Globals globals{*this}; + + // Session Pro-related capabilities + Pro pro{*this}; + + // Device groups for handling shared encryption key among account devices + Devices devices{*this}; + + // The account's synced configuration, shared with its other devices + Configs configs{*this}; + + // Passes a batch of messages retrieved from the swarm to the appropriate handler based on the + // namespace they were retrieved from. `is_final` should be true if this batch represents the + // complete current contents of the namespace (i.e. there are no more messages pending + // retrieval), and false if more messages may follow. Note that if there turn out to be no more + // messages after a non-final call, the caller should still call this with an empty span and + // is_final=true to flush any actions that are deferred until the end of a fetch. + void receive_messages( + std::span messages, config::Namespace ns, bool is_final); +}; + +} // namespace session::core diff --git a/include/session/core/callbacks.hpp b/include/session/core/callbacks.hpp new file mode 100644 index 000000000..e3fc45cb7 --- /dev/null +++ b/include/session/core/callbacks.hpp @@ -0,0 +1,219 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::core { + +class Core; + +/// Return value of prefetch_pfs_keys() describing the current cache state at the time of the call. +enum class PfsKeyStatus { + fresh, ///< A fresh cached key exists; no fetch was initiated + stale, ///< A usable but stale key exists; a background re-fetch was initiated. + ///< The pfs_keys_fetched callback will fire when the fetch completes. + fetching, ///< No usable key is cached; a background fetch was initiated. + ///< The pfs_keys_fetched callback will fire when the fetch completes. + nak, ///< An unexpired NAK suppresses fetching; no usable key exists +}; + +/// Result passed to the pfs_keys_fetched callback when a background fetch completes. +enum class PfsKeyFetch { + new_key, ///< A key was retrieved and stored (new or changed from the previous cache entry) + unchanged, ///< Keys were retrieved but match what was already cached + not_found, ///< The fetch succeeded but the remote account pubkey namespace held no valid keys + failed, ///< The network request failed (swarm lookup or send_request) +}; + +/// Reason code passed to the message_decrypt_failed callback. +enum class MessageDecryptFailure { + no_pfs_key, ///< Version 2 message: no PFS account key matched the key indicator AND the + ///< non-PFS fallback decryption also failed; the message cannot be read. + decrypt_failed, ///< Decryption failed (either version); key was found but did not work + bad_format, ///< Message is structurally malformed (e.g. invalid bencode, truncated fields) + unknown_version, ///< Message starts with 0x00 but carries an unrecognised version byte; + ///< likely a future protocol version this build does not understand +}; + +/// A successfully decrypted one-to-one message from Namespace::Default. +struct ReceivedMessage { + std::string hash; ///< Swarm-assigned message hash + sys_ms timestamp; ///< Server-reported upload timestamp + sys_ms expiry; ///< Server-reported expiry timestamp + b33 sender_session_id; ///< 0x05-prefixed sender session ID + int version; ///< Protocol version: 1 or 2 + std::vector content; ///< Decrypted protobuf-encoded payload + std::optional pro_signature; ///< Session Pro signature, if present + bool pfs_encrypted = false; ///< True iff decrypted via PFS+PQ (X-Wing) key derivation; + ///< false for v1 messages and v2 non-PFS fallback messages. +}; + +/// Status of a send operation initiated by Core::send_dm(). +enum class MessageSendStatus { + awaiting_keys, ///< Waiting for a PFS+PQ pubkey fetch to complete before encrypting. + sending, ///< Encryption complete; the store request has been dispatched. + retrying, ///< A previous send attempt failed; retrying. (Not yet implemented: + ///< currently a failed send goes directly to network_error. TODO: + ///< implement automatic retry with a maximum retry count.) + success, ///< The store request was accepted by a swarm node. + network_error, ///< The swarm lookup or store request failed (terminal). + no_network, ///< No network object is attached. + encrypt_failed, ///< Encryption failed (should not normally happen). +}; + +/// Struct holding application callbacks to fire when libsession Core events happen to allow the +/// Core object to fire into the application front-end. +/// +/// The signatures say what a handler may keep. A parameter taken by rvalue reference was read for +/// this delivery and nothing else holds it, so a handler may move from it; one taken by `const&` or +/// as a span is borrowed and valid only for the duration of the call. A handler may declare an +/// rvalue parameter as `T&&`, `const T&` or `T`, whichever suits it: only the last constructs +/// anything, and a handler that just reads pays nothing. +struct callbacks { + + /// Callback that is invoked when a device linking request is received for entry into the device + /// group. This is expected to notify the user of the linking request, and ask them to confirm + /// it. Generally this should be followed (after user interaction) by a call to one of the + /// core.devices methods: ignore_request(), accept_request(), delete_request() with the reqid + /// value. + /// + /// This may fire multiple times: it generally fires when the request first comes in, but + /// will also fire during startup if there is a still-active request that has not been + /// accepted, ignored, or deleted. (This is so that Session a shutdown or crash does not + /// lose a device request). + /// + /// It may also not fire at all if the request has been superceded (such as being accepted + /// by a third device). + /// + /// This request is not fired for the devices own linking request, i.e. when this device is the + /// one requesting entry into a device group. + /// + /// If this callback is not set then new device link requests are ignored by this device. + /// + /// Parameters: + /// - reqid -- a unique identifier for this request that persists across Core restarts and can + /// be used to correlate this request with a subsequent device_added callback. + /// - new_device -- the new device metadata included in the link request. + /// - sas -- a span of 21 string_views representing the short authentication string for this + /// request. The first 7 are the standard display; all 21 are available for the extended + /// view. Formatting and joining is left to the caller. + std::function sas)> + device_link_request; + + /// Callback that is invoked when a new device has been linked to the account. If a batch + /// of messages being processed includes both a device link request *and* an acceptance + /// (such as could happen if third device accepts the request) then only this, not the + /// request, will be fired. + /// + /// This callback is not fired if *this* is the device that has been added: see + /// device_self_added instead for that case. + /// + /// Note that this is fired once the new device is confirmed via stored swarm message, i.e. + /// it does not fire instantly upon calling `accept_request()`. + /// + /// Paramters: + /// - reqid -- if `on_device_link_request` had previously been called for this device, this + /// value will be the same value, allowing the application to correlate linking requests and + /// acceptance. If there was no previous link request (such as when catching up on device + /// updates performed by other account devices) then the value will be 0. + /// - new_device -- the metadata about the new device. + std::function device_added; + + /// Callback invoked when *this* device has been confirmed linked to the account by another + /// device. + std::function device_self_added; + + /// Callback that is invoked if we determine that a device has been kicked out of the device + /// group, either initiated by this device or another device. This does not, however, fire if + /// the *current* device gets kicked out; see device_self_removed for that. + /// + /// Parameters: + /// - removed_device -- the most recent info we have (locally) for the removed device. + std::function device_removed; + + /// Callback invoked when *this* device has been confirmed removed from the account (typically + /// from another device) from an incoming device group update. + std::function device_self_removed; + + /// Callback invoked when a background PFS key fetch initiated by prefetch_pfs_keys() completes. + /// Not invoked for cache hits or NAK suppressions (i.e. only fires when prefetch_pfs_keys() + /// returns stale or fetching). + /// + /// Parameters: + /// - session_id -- 33-byte session ID (0x05 prefix + X25519 pubkey) of the remote user + /// - result -- the outcome of the fetch: new_key, unchanged, not_found, or failed + std::function session_id, PfsKeyFetch result)> + pfs_keys_fetched; + + /// Callback invoked when a one-to-one message from Namespace::Default is successfully + /// decrypted. The message is passed as an rvalue reference: the callback may move from it + /// (e.g. to take ownership of the content vector) or simply read it in place. + /// + /// Parameters: + /// - msg -- the decrypted message data + std::function message_received; + + /// Callback invoked when a one-to-one message from Namespace::Default could not be decrypted + /// or parsed. The raw swarm message and a reason code are provided so the caller can decide + /// how to handle it (e.g. log, queue for retry, surface to the user). + /// + /// When receive_messages() is called directly by the application, `msg` is a reference to one + /// of the SwarmMessage elements passed in, which the caller can identify exactly by comparing + /// pointers. When triggered by internal polling, `msg` refers to an internally-owned object. + /// + /// Parameters: + /// - msg -- the raw swarm message that could not be decrypted + /// - reason -- why decryption failed + std::function + message_decrypt_failed; + + /// Callback fired as a send operation initiated by Core::send_dm() progresses. This is + /// typically invoked multiple times for a single message — once or more for intermediate + /// states (awaiting_keys, sending, retrying) followed by a terminal state (success, + /// network_error, no_network, or encrypt_failed). + /// + /// Parameters: + /// - message_id -- the value returned by the originating send_dm() call + /// - status -- the current state of the send + /// - swarm_hash -- the hash the swarm assigned the stored message, on `success` and when the + /// storage server reported one. Unset for every other status. + std::function swarm_hash)> + message_send_status; + + /// Callback fired when merging config messages from the swarm changed one or more of the + /// account's configs, so that the layer holding a queryable copy of that state knows to go and + /// reconcile it. + /// + /// Fires once per batch of merges rather than once per config, with every namespace that + /// changed: one poll can carry all four, and reacting to each in turn would show the + /// application a half-applied state. It fires *after* the changed configs have been dumped, + /// so what a handler reads is already on disk. + /// + /// Only merges are reported. A config the application changed itself is not news to it, and + /// Local never appears at all, since it merges nothing. + /// + /// This says *that* something changed, not what. There is deliberately no diff: a config diff + /// describes a transition between config states, while the reconciling layer's own currency is + /// not a config state and is not tracked -- a merge can jump several updates at once, and a + /// crash between merging and reconciling leaves it behind by an unrecorded amount. Comparing + /// against its own stored state is what makes reconciliation self-correcting, and a diff would + /// silently skip anything those cases had left behind. + /// + /// Parameters: + /// - changed -- the namespaces whose configs the merge altered. Valid only for the duration of + /// the call. + std::function changed)> configs_changed; +}; + +} // namespace session::core diff --git a/include/session/core/component.hpp b/include/session/core/component.hpp new file mode 100644 index 000000000..450c8a95a --- /dev/null +++ b/include/session/core/component.hpp @@ -0,0 +1,46 @@ +#pragma once + +namespace session::sqlite { +class Connection; +} +namespace oxen::quic { +class Loop; +} // namespace oxen::quic +namespace session::core { + +namespace quic = oxen::quic; + +class Core; +struct callbacks; + +namespace detail { + // Internal base class bridge between Core and the various components of core. This bridge + // can be used to allow components to access selected private parts of core, such as the + // database, without needing components to be direct friends of Core. + class CoreComponent { + protected: + friend class core::Core; + Core& core; + + // Gets a thread-unique database connection from the Core's Database's connection pool. This + // is unique to the calling thread and must not be used across threads. + sqlite::Connection conn(); + + // Returns the application callbacks registered with Core. + core::callbacks& cb(); + + // Returns the event loop for scheduling async work. + quic::Loop& loop(); + + explicit CoreComponent(Core& core); + + // Default component `init()` does nothing; classes can override this if they want to be + // called after database migrations are complete, but still during the parent Core + // construction. This will be called on each CoreComponent-derived member of Core, in the + // same order that those members were constructed (i.e. class declaration order). + virtual void init() {} + }; + +} // namespace detail + +} // namespace session::core diff --git a/include/session/core/configs.hpp b/include/session/core/configs.hpp new file mode 100644 index 000000000..32f5d2995 --- /dev/null +++ b/include/session/core/configs.hpp @@ -0,0 +1,199 @@ +#pragma once + +#include +#include +#include +#include + +#include "component.hpp" +#include "swarm_message.hpp" + +namespace session::config { +class ConfigBase; +class Contacts; +class ConvoInfoVolatile; +class Local; +class UserGroups; +class UserProfile; +} // namespace session::config + +namespace session { +class TestHelper; +} // namespace session + +namespace session::core { + +/// This account's synced configuration: what it knows about itself, its contacts and its +/// conversations, in the form its other devices share. +/// +/// Core owns these because everything mechanical about them is Core's work -- retrieving them from +/// their namespaces, merging, keeping the dumps, pushing what changed -- and none of that requires +/// knowing what a contact or a conversation actually is. Deciding what the contents *mean*, and +/// reconciling them into something queryable, belongs to the layer above, which reads these objects +/// and is told when they change. +/// +/// The configs are built on first use rather than during init(), because they are encrypted to the +/// account key and a Core opened with `defer_account` does not have one yet. Nothing can reach +/// them before an account exists in any case: a network cannot be attached without one, so neither +/// polling nor pushing can run, and a direct caller gets the same `no_account` any other +/// account-dependent call would throw. +class Configs : public detail::CoreComponent { + friend class session::TestHelper; + + std::unique_ptr _user_profile; + std::unique_ptr _contacts; + std::unique_ptr _convo_info_volatile; + std::unique_ptr _user_groups; + std::unique_ptr _local; + + bool _loaded = false; + + // Depth of nested Batch guards. + int _batch_depth = 0; + + // Namespaces a merge has altered since the last time the application was told, deduplicated. + // Held rather than reported immediately so that one poll carrying several config namespaces + // produces one notification of a settled state rather than several of partial ones. + std::vector _changed; + + // Writes the dumps of everything that changed and schedules a push if one is owed -- unless a + // Batch is holding it back, in which case releasing that Batch is what runs it. + void _flush(); + + // Debounce state: when the current run of changes began, and when the last one arrived. + std::chrono::steady_clock::time_point _burst_started{}; + std::chrono::steady_clock::time_point _last_change{}; + bool _push_scheduled = false; + bool _push_in_flight = false; + + // Deferred work is handed to the event loop, which outlives this component and has no way to + // cancel a call already scheduled. Callbacks capture a weak reference to this and do nothing + // if it has expired, which is what stops a pending push firing into a destroyed Core. + std::shared_ptr _alive = std::make_shared(0); + + void _schedule_push(); + void _arm_push_timer(std::chrono::milliseconds delay); + void _push_if_due(); + void _send_push(); + + // Constructs the configs from their stored dumps, or empty if there are none. Requires an + // account; throws globals::no_account if there is not one yet. + void _load(); + + // Writes `conf`'s dump if it has changed since the last one was written. + void _store(config::ConfigBase& conf); + + // The configs that have somewhere to be pushed to, which is every one except Local. + // + // Local already reports needs_push() as false unconditionally, so this is not what stops it + // being pushed -- it is what stops that from being the only thing that does. The push path + // asks "which configs go to a swarm", and answering that by trusting each config to decline + // would make a config with no destination indistinguishable from one with nothing to say. + std::vector _pushable(); + + public: + // Both defined where the config types are complete, so that this header can forward-declare + // them rather than making every consumer of core.hpp parse all five. The constructor needs it + // as much as the destructor does: its cleanup path has to be able to destroy the members. + explicit Configs(Core& core); + ~Configs(); + + config::UserProfile& user_profile(); + config::Contacts& contacts(); + config::ConvoInfoVolatile& convo_info_volatile(); + config::UserGroups& user_groups(); + + /// Device-local settings. Shares the config machinery -- and so gets dumped and reloaded like + /// the rest -- but is never pushed anywhere and never merges anything. + config::Local& local(); + + /// Every config, for the operations that do not care which is which. + std::vector all(); + + /// The config kept in the given namespace, or nullptr if that namespace holds none. + /// + /// A fixed mapping rather than a search over `storage_namespace()`, deliberately: Local reports + /// UserProfile's namespace as a stand-in for the one it does not have, so a search would find + /// two configs for namespace 2 and the answer would depend on the order it looked. + config::ConfigBase* for_namespace(config::Namespace ns); + + /// Merges config messages retrieved from `ns` into the config kept there, and dumps the result + /// if the merge changed it. + /// + /// A merge can leave the config needing a push even when nothing local changed, since resolving + /// a conflict between two other devices produces a new state that only this device holds. + void merge(config::Namespace ns, std::span messages); + + /// Holds back dumping until a run of changes is finished. + /// + /// A config is dumped when it changes, which is right when nobody knows any better -- but a + /// caller working through a batch of messages does know better, and dumping between them writes + /// intermediate states nobody will ever read. Holding one of these says "there is more + /// coming"; releasing it says "now". + /// + /// This is what keeps the timer honest. Debouncing is a guess at where a batch ended, and a + /// guess is only needed where nothing knows: with a batch held across the work, dumping happens + /// once at a boundary that is *known*, and the timer is left to cover only the changes that + /// arrive with no such boundary to see. + /// + /// Nests: `merge()` holds one itself, so a caller already holding one across several merges + /// still gets a single flush at the end. Neither copyable nor movable, so it cannot outlive + /// the scope it was declared in. + class Batch { + Configs& _configs; + + public: + explicit Batch(Configs& configs); + ~Batch(); + Batch(const Batch&) = delete; + Batch& operator=(const Batch&) = delete; + }; + + [[nodiscard]] Batch batch() { return Batch{*this}; } + + /// Writes the dump of every config that has changed since it was last written. Immediate: a + /// held Batch does not defer this, since asking for it explicitly is the point. + void store_dumps(); + + /// Whether any config holds changes that have not reached the swarm. + bool needs_push(); + + /// How long a push waits for changes to stop arriving before going out, and the longest it will + /// wait once they started. A run of changes coalesces into one request rather than one per + /// change; the cap is what stops a steady trickle deferring the push indefinitely. + /// + /// Settable mostly so a test does not have to spend real seconds proving it. + std::chrono::milliseconds push_debounce = std::chrono::seconds{2}; + std::chrono::milliseconds push_max_delay = std::chrono::seconds{10}; + + /// Writes the defaults a brand-new account starts with. + /// + /// Only for an account created here. Restoring one deliberately does not come through here: + /// the real values are about to arrive from the swarm, and writing local ones first would put + /// them in competition with what the account already says. + void initialise_new_account(); + + /// Set false to stop config changes from ever leaving this device. + /// + /// For working against a real account without risking it: everything else carries on as normal + /// -- changes are held, dumped, and reported as owed by needs_push() -- but nothing is sent, so + /// nothing this device does can reach the account's other devices or overwrite what they hold. + /// + /// Deliberately not a pretence that the push happened: `needs_push()` keeps saying yes, so the + /// state is visibly unpublished rather than looking settled. Turning it back on lets the next + /// push carry everything accumulated since. + bool push_enabled = true; + + /// Pushes whatever is owed immediately, rather than waiting out the debounce. + /// + /// Everything dirty goes in one `sequence` request: a store per config message, then a single + /// delete naming every hash those stores obsolete. Only what this account owns goes in it -- + /// a group's configs live under a different pubkey and are pushed separately even if their + /// swarm turns out to be the same one, because a request that carried both would tell that + /// swarm the two belong to the same person. + /// + /// Does nothing if a push is already in flight; the one in flight re-checks when it completes. + void push_now(); +}; + +} // namespace session::core diff --git a/include/session/core/devices.hpp b/include/session/core/devices.hpp new file mode 100644 index 000000000..15558cd3d --- /dev/null +++ b/include/session/core/devices.hpp @@ -0,0 +1,383 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "component.hpp" +#include "swarm_message.hpp" + +namespace session { +class TestHelper; +} // namespace session + +namespace session::core { + +using namespace std::literals; + +class Core; + +namespace device { + + enum class Type { + Unknown, + Session_iOS, + Session_Desktop, + Session_Android, + Session_CLI, + }; + + // The Type a stored or encoded type string denotes; Unknown for anything else, which a caller + // keeps verbatim in `Info::other_device`. The inverse of `Info::encoded_type()`. + inline Type type_from_encoded(std::string_view t) { + if (t == "i") + return Type::Session_iOS; + if (t == "a") + return Type::Session_Android; + if (t == "d") + return Type::Session_Desktop; + if (t == "c") + return Type::Session_CLI; + return Type::Unknown; + } + + /// A device's membership of the account's device group. + /// + /// **The numbers are a rank, and merging depends on it.** State never goes on the wire — it is + /// inferred from which message a record arrived in — so a state change moves no field that the + /// record's seqno versions, and a merge guarded on the seqno alone would discard every one of + /// them. Records are therefore compared as `(state, seqno)` lexicographically, which is why + /// these are ordered from least to most authoritative and why the stored integer *is* the rank. + /// + /// Rank only ever increases and the order is total, so a merged result is the maximum over + /// everything received, regardless of the order it arrived in. + enum class State { + Unregistered = 0, ///< Not in the group, and never was: local device info that cannot be + ///< pushed because this device has not joined one yet. Says nothing + ///< about any other device, which is why a removal is `Kicked`. + Pending = 1, ///< A device with a pending link request. This is used for two cases: + ///< - This device has sent a request to join the account's device group and + ///< is awaiting acceptance by an existing device. + ///< - Another device has sent a link request that has been received but not + ///< yet accepted or ignored by this device. + Registered = 2, ///< Device is in the account's registered device set. Outranks Pending so + ///< that an acceptance propagates even against a newer link request, which + ///< is what lets registration complete at all. + Kicked = 3, ///< Removed from the group, and permanently: a kicked device id can never + ///< rejoin, only be replaced by a fresh one. Outranks everything, so the + ///< tombstone that carries a removal cannot be undone by a stale record + ///< replaying an earlier state. Always accompanied by `kicked`, which the + ///< schema enforces. + }; + + // Value returned to indicate the push status of a device info or account keys update. + enum class PushStatus { + Synced = 0, // We have pushed and confirmed (i.e. fetched the update) + Pushed = 1, // We have pushed, but not yet confirmed + Pending = 2, // We need to push, but haven't yet done so + NotInGroup = 3, // We are not in the device group and so can't push + }; + + struct Info { + // Unique device id, in raw bytes. Typically randomized during device initial setup. + std::array id; + + // Device seqno. Incremented on device key rotation and/or info updates. + int64_t seqno; + + // Timestamp of the most recent update. + std::chrono::sys_seconds timestamp; + + // The device type; one of the above enum values, or DevType::Unknown if the device type is + // not one of the standard Session clients. + Type type = device::Type::Unknown; + + // When device type is not one of the standard session clients, this will be set to a + // free-form string indicating the device type. Will be empty if no device type is provided + // in the device info at all. When DevType has a non-Unknown value, this will be + // empty/ignored. + std::string other_device; + + // Device-provided description of itself. This could contain the OS type or version, + // possible a device nickname, but is generally free-form data. + std::string description; + + // Indicates whether the device is registered, pending registration, or not registered. + State state; + + // For state == State::Unregistered, this timestamp (if set) indicates that the device was + // removed from the device group at that timestamp. It will be nullopt for a device that + // was never in the device group. + std::optional kicked; + + // Application version triplet as reported by the device. The 2nd and 3rd values will + // always be in [0, 999]. (If setting device info, they will be clamped if outside this + // range). + std::array version; + + // The current device-specific X25519 pubkey + std::array pk_x25519; + + // The current device-specific MLKEM-768 pubkey + std::array pk_mlkem768; + + // Fields from a device running a newer libsession than ours, kept so that we republish them + // rather than silently dropping what we do not understand. + // + // Space for future versions of libsession, not for client data: everything here is carried + // by every other device on the account, and the payload is padded in buckets sized on the + // assumption that a record stays within its budget. Anything added must fit it. + oxenc::bt_dict extra; + + // Returns the encoded device type string: "i", "a", or "d" for the standard Session + // client types, `other_device` for unknown types, or "" if unknown with no other_device. + std::string_view encoded_type() const { + switch (type) { + case Type::Session_iOS: return "i"; + case Type::Session_Android: return "a"; + case Type::Session_Desktop: return "d"; + case Type::Session_CLI: return "c"; + default: return other_device; + } + } + + // Returns true if the user-settable fields (those controlled by update_info()) are equal to + // the corresponding fields in `other`. Does NOT compare id, seqno, timestamp, state, pk_*, + // or kicked. The unknown `extra` fields are included in the comparison. + bool same_user_fields(const Info& other) const; + }; + + using map = std::map, Info>; + + struct decryption_failed : std::runtime_error { + using std::runtime_error::runtime_error; + }; +}; // namespace device + +class Devices final : detail::CoreComponent { + public: + private: + friend class Core; + friend class Globals; + friend class session::TestHelper; + explicit Devices(Core& c) : detail::CoreComponent{c} {} + + void init() override; + + // Records that this account owes a device group, for `establish_group()` to act on. Called by + // Globals when it generates an account, which is before this component has initialised -- hence + // a stored flag rather than doing the work there. + void _mark_group_owed(); + + std::array self_id; + + // Encrypts the inner device data for all the members of the device group. + std::vector encrypt_device_data(const device::map& devices); + + // Processes a single incoming device group ("D") or link request ("L") message. `data` is the + // full raw message bytes including the outer bt-dict wrapper with the "" type key. + void receive_device_group_message(std::span data); + void receive_link_request(std::span data); + + // Handlers for incoming swarm messages by namespace, called from Core::receive_messages. + void parse_device_messages(std::span messages, bool is_final); + void parse_account_pubkeys(std::span messages, bool is_final); + + // Decrypts an incoming encrypted device group ("G") message, returning the bt-encoded group + // payload plaintext (a bt-dict containing at minimum a "D" devices subdict and optionally a + // "K" account keys list). Throws if parsing or decryption fails. Throws + // `device::decryption_failed` if we could not find a key that successfully decrypts the data + // (i.e. we are not in the device group, or all our keys have rotated past this message). + std::vector decrypt_device_data(std::span data); + + public: + // Returns the current device's random identifier, in hex. + std::string device_id() const; + + // Returns info for all registered and/or pending devices and/or unregistered devices for this + // account. If `only_device` is non-empty it must be a 32-byte device id that is used to + // filter the results to just that one device. + device::map devices( + bool include_registered = true, + bool include_pending = false, + bool include_unregistered = false, + std::span only_device = {}); + + // Returns *this* device's info and whether it is registered in the device group. + std::pair device_info(); + + struct LinkRequestResult { + std::vector message; // encrypted bytes to push to Namespace::Devices + std::array sas; // emoji SAS sequence for user display + }; + + // Builds an outgoing link request message to upload to Namespace::Devices. This should + // only be called when this device is not currently registered in the device group; throws + // std::logic_error if it is already registered. The returned message is to be pushed to + // Namespace::Devices with a 10-minute TTL. The sas field contains the short authentication + // string that should be displayed to the user for verification against the accepting device. + LinkRequestResult build_link_request(); + + // Updates this device's info locally to match the given info; if the current device is + // registered then this dirties the device config data, requiring a push. + // + // The state and pk_* fields of the input value are ignored. + void update_info(const device::Info& info); + + // Creates the account's device group with this device as its only member, if one is owed. + // + // Owed means the account was *generated* here rather than restored: a brand new account has no + // group and nothing else will ever make one, whereas a restored account may already have a + // group belonging to devices that are merely offline, and inventing a second one would orphan + // them. `Globals` records which happened; this acts on that record and clears it, so it runs + // exactly once per account and survives a crash between creating the account and getting here. + // + // Does nothing if this device is already registered, so it is safe to call at any time. + // + // Registering ourselves is what breaks the deadlock the rest of this class sits behind: + // `needs_push()` only reports a device group push for a registered device, and the only other + // thing that registers one is receiving a group message we can decrypt -- which cannot happen + // until some device has pushed one. + // + // Also mints the account's first shared key seed, since the group payload carries it. That is + // not the same as *publishing* PFS keys: nothing goes to Namespace::AccountPubkeys here, and + // until it does no other account treats this one as supporting v2 encryption. + void establish_group(); + + // Stores the X25519 + MLKEM768 keys that make up an "X-Wing" key + struct XWingKeys { + cleared_b32 x25519_sec; + std::array x25519_pub; + cleared_array mlkem768_sec; + std::array mlkem768_pub; + }; + + struct DeviceKeys : XWingKeys { + std::chrono::sys_seconds created; + std::optional rotated; + }; + + // Rotates the device keys used for encrypting device group data. This also implicitly updates + // the current device's public keys. If the current device is registered, calling this will + // dirty the config data and require another push. + // + // This returns the newly created keys. (It can be safely discarded as it will already be + // stored in the database). + DeviceKeys rotate_device_keys(); + + // Returns current and recent local device private keys. This will be sorted with most recent + // key first. If there is no current key at all, this generates one. + std::vector active_device_keys(); + + struct AccountKeys : XWingKeys { + std::chrono::sys_seconds created; + std::optional rotated; + }; + + // How long after rotation to keep an old account key. 14 days is the maximum 1-to-1 message + // TTL, plus 24h for sender key update lag, plus 24h safety margin. + static constexpr auto ACCOUNT_KEY_RETENTION = 16 * 24h; + + // Base rotation period and jitter window for account key rotation. The formula is designed so + // that the minimum rotation time across all N devices in the group is Unif[PERIOD-WINDOW/2, + // PERIOD+WINDOW/2], regardless of N, masking the number of devices in the group. + static constexpr auto ACCOUNT_KEY_ROTATION_PERIOD = 12h; + static constexpr auto ACCOUNT_KEY_ROTATION_WINDOW = 2h; + + // How long to keep a pending link request before pruning it as stale. + static constexpr auto LINK_REQUEST_MAX_AGE = 10min; + + // Rotates the shared account keys used for PFS+PQ message encryption. Generates a new random + // seed, stores it in the database, marks the previous active key as rotated, and prunes keys + // older than ACCOUNT_KEY_RETENTION. Should be called when account_rotation_due() is true and + // when a device first joins the device group with no existing account keys. + void rotate_account_keys(); + + // Returns the current active account keys after pruning obsolete ones: that is, the current key + // plus all keys that were rotated away fewer than ACCOUNT_KEY_RETENTION ago. Keys are returned + // sorted from newest to oldest. If there are no keys at all, generates an initial one. + // Returns account keys, ordered with the active (unrotated) key first then + // most-recently-rotated first. Expired rotated keys are pruned before querying. If + // key_indicator is given, only keys whose ML-KEM-768 pubkey begins with those two bytes are + // returned (using the indexed key_indicator virtual column); otherwise all retained keys are + // returned and a new key is auto-generated if none is currently active. + std::vector active_account_keys( + std::optional> key_indicator = std::nullopt); + + // Returns the time when this device's unique device key is due to be rotated. Returns nullopt + // if this device is not currently part of the device group. + std::optional next_device_rotation(); + + bool device_rotation_due() { + auto t = next_device_rotation(); + return t && *t <= clock_now(); + } + + // Return true if the account key is due to be rotated by this device. Returns nullopt if this + // device is not currently part of the device group. + std::optional next_account_rotation(); + + bool account_rotation_due() { + auto t = next_account_rotation(); + return t && *t <= clock_now(); + } + + struct DeviceGroupPush { + std::vector message; // encrypted bytes to push to Namespace::Devices + int64_t seqno; // this device's seqno at the moment the message was built + }; + + // Builds the account's device group ("G") message for upload to Namespace::Devices. + // + // Throws std::logic_error if this device is not registered: a device outside the group has + // nothing to say about it, and pushing anyway would announce a group of one that every other + // device would merge in as authoritative. + // + // `seqno` comes back rather than being read again afterwards because the message is built from + // a snapshot: pass it to mark_device_group_pushed() once the swarm confirms the store, so that + // a change made while the push was in flight stays dirty. + DeviceGroupPush build_device_group_message(); + + // Builds the signed account public key message for upload to namespace -21. The message is a + // bt-encoded dict containing the current active ML-KEM-768 pubkey ("M"), X25519 pubkey ("X"), + // and a "positive alternative" Ed25519 signature ("~") over the preceding fields, allowing + // recipients who only know the account's Session ID (X25519) to verify the keys. + // Throws if there are no active account keys. + std::vector build_account_pubkey_message(); + + // Flags indicating which messages need to be pushed to the swarm. + struct NeedsPush { + bool device_group; ///< True if an updated device group message needs to be pushed + bool account_pubkey; ///< True if an updated account pubkey message needs to be pushed + }; + + // Returns whether a push is currently needed. Should be called after processing a final swarm + // message batch (or at startup) to determine whether outgoing pushes are required. + // + // device_group is true when this device is registered AND any of the following hold: + // - our own device info has changed since the last confirmed device group push + // - any device has a state transition (registered/removed) that needs broadcasting + // - any account key seed has not yet been distributed via a confirmed push + // + // account_pubkey is true when the current active account key has not yet been seen confirmed + // on the swarm (i.e. neither we nor another device has pushed it and we've received it back). + NeedsPush needs_push(); + + // Marks the device group message as successfully pushed with the given own-device seqno (which + // the caller reads from device_info() before building the push message). Updates pushed_seqno, + // clears broadcast_needed on all device rows, and marks all account key seeds as distributed. + void mark_device_group_pushed(int64_t seqno); +}; + +} // namespace session::core diff --git a/include/session/core/globals.hpp b/include/session/core/globals.hpp new file mode 100644 index 000000000..b0cb6a8e9 --- /dev/null +++ b/include/session/core/globals.hpp @@ -0,0 +1,205 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "component.hpp" + +namespace session::core { + +class Core; +// Defined in core.hpp, which includes this header; only referenced here as a parameter type. +struct predefined_seed; + +// Core component contains one-off global values that don't make sense storing in a table, typically +// because the value is highly special purpose or is only used in one single place. If you ever +// find yourself wanting to put multiple values in here under the same key, that is a sign that you +// should not be using this class and should instead refactor to use proper table relations. +// +// A note on keys: to avoid conflicts, external users of these globals should use prefix names that +// are unlikely to conflict with other uses. For example, "session_ios_dark_mode" is a decent name, +// but "pubkey" is a terrible one. All internal libsession keys in this table begin with an +// underscore, and should never be accesses outside libsession itself. +// +/// Thrown when something requiring the account's identity is used before there is one. Only +/// reachable when the Core was constructed with defer_account: without it, construction either +/// finds a seed, is given one, or generates one. +struct no_account : std::logic_error { + no_account() : + std::logic_error{ + "This Session account has no identity yet; call create_account() or " + "restore_account() first"} {} +}; + +class Globals final : detail::CoreComponent { + + private: + friend class Core; + explicit Globals(Core& c) : detail::CoreComponent{c} {} + + void _require_account() const { + if (!_have_account) + throw no_account{}; + } + + // Holds the account seed; loaded during initialization (created if it doesn't exist). A new + // account seed is generated during initialization if the database doesn't contain one (e.g. if + // brand new). + // + // Read-only access is available via the account_seed() method. + session::secure_buffer _account_seed; + network::ed25519_pubkey _pubkey_ed25519; + network::x25519_pubkey _pubkey_x25519; + std::array _session_id; // AKA pubkey_x25519 with a 0x05 byte prefix + std::string _session_id_hex; // hex encoding of _session_id, computed once in init() + + void init() override; + + // If set by the Core constructor before init(), used as the initial account seed when the + // database does not yet contain one. Cleared after use in init(). + std::optional _predefined_seed; + + // Set by the Core constructor from the defer_account option: suppresses generating an account + // during init() when the database has none. + bool _defer_account = false; + + // False between construction and the account being resolved, which only happens when + // defer_account was given and the database held no seed. + bool _have_account = false; + + // Derives and caches the key material for `seed`, and stores the seed if `persist`. + void _adopt_seed(const cleared_b32& seed, bool persist); + + // Records that a freshly *generated* account owes a device group. Not called for a restored + // account: that one may already have a group belonging to devices we have not met. + void _mark_new_account(); + + public: + /// Whether this account has an identity yet. + /// + /// Only ever false when the Core was constructed with defer_account and the database held no + /// seed. Until it is true, everything needing the account -- session_id(), account_seed(), + /// send_dm(), attaching a network -- throws no_account. + bool have_account() const { return _have_account; } + + /// Generates a fresh account and stores it. + /// + /// @throws std::logic_error if this account already has an identity: adopting a second one + /// would orphan every message and key already stored against the first. + void create_account(); + + /// Adopts an existing account seed, as typed from a recovery phrase or transferred from + /// another device, and stores it. + /// + /// This is also the first half of linking a new device to an existing account: a link request + /// is encrypted to the account root key, so the seed must be adopted before + /// devices.build_link_request() can be called. + /// + /// @throws std::logic_error if this account already has an identity. + void restore_account(const predefined_seed& seed); + + public: + // Retrieval methods. These query for the given key and, if the type matches, return the given + // value. You get back nullopt if the database key does not exist, or if it contains + std::optional get_integer(std::string_view key); + std::optional get_real(std::string_view key); + std::optional get_text(std::string_view key); + std::optional> get_blob(std::string_view key); + // Same as get_blob, but allocates a libsodium secure buffer to old the value. + // + // Do not use this to access the "seed" value: that value is cached in the Core object and + // accessible via CoreComponent::access_seed(). + std::optional get_blob_secure(std::string_view key); + // Reads a fixed size blob into `to`. If the database does not contain a BLOB value of byte + // length `to.size()`, returns false; other writes the blob value to `to` and returns true. + bool get_blob_to(std::string_view key, std::span to); + + // Retrieves the value of whatever type it currently contains. Returns a std::monostate if the + // database key is not set at all, otherwise of of the other variant values. + std::variant> get( + std::string_view key); + std::variant get_secure( + std::string_view key); + + // Assignment. If the database key already exists, this overwrites it. + void set(std::string_view key, int64_t integer); + void set(std::string_view key, double real); + void set(std::string_view key, std::string_view text); + void set(std::string_view key, std::span blob); + + /// Removes a key from the globals table. + /// + /// Returns true if the key existed and was removed, false if it was not set in the first + /// place. Erasing a key that was never set is not an error. + bool erase(std::string_view key); + + /// RAII accessor returned by account_seed(). Holds the underlying secure buffer open for + /// reading while alive; the buffer becomes unreadable again when the last copy is destroyed. + struct AccountSeedAccess { + private: + friend class Globals; + explicit AccountSeedAccess(const session::secure_buffer::r_accessor& acc) : _acc{acc} {} + session::secure_buffer::r_accessor _acc; + + auto buf() const { return _acc.buf.first<96>(); } + + public: + /// The raw 32-byte account seed (identical to ed25519_secret().first<32>()). + std::span seed() const& { return buf().first<32>(); } + std::span seed() const&& = delete; + /// The 64-byte Ed25519 secret key in libsodium format (seed || pubkey). + std::span ed25519_secret() const& { return buf().first<64>(); } + std::span ed25519_secret() const&& = delete; + /// The 32-byte X25519 secret key derived from the account seed. This is also the clamped + /// private scalar of the Ed25519 key, usable for advanced scalar-multiplication operations. + std::span x25519_key() const& { return buf().last<32>(); } + std::span x25519_key() const&& = delete; + }; + + // Each of these needs an identity, so each throws no_account when there is not one yet. That + // is only reachable via defer_account; without it an account always exists by the time the + // Core constructor returns. + AccountSeedAccess account_seed() { + _require_account(); + auto acc = _account_seed.access(); + return AccountSeedAccess{acc}; + } + // These are computed from the account_seed during construction: + std::span session_id() { + _require_account(); + return _session_id; + } + const std::string& session_id_hex() const { + _require_account(); + return _session_id_hex; + } + const network::ed25519_pubkey& pubkey_ed25519() const { + _require_account(); + return _pubkey_ed25519; + } + const network::x25519_pubkey& pubkey_x25519() const { + _require_account(); + return _pubkey_x25519; + } + + /// Returns the account seed as a mnemonic word list with checksum, stored in secure memory. + /// + /// If `force_24` is false (the default), returns 13 words when the upper 16 bytes of the + /// seed are all zero (128-bit entropy), or 25 words otherwise. If `force_24` is true, + /// always returns 25 words. + mnemonics::secure_mnemonic seed_mnemonic( + const mnemonics::Mnemonics& lang, bool force_24 = false); + mnemonics::secure_mnemonic seed_mnemonic( + std::string_view lang_name = "English", bool force_24 = false); +}; + +} // namespace session::core diff --git a/include/session/core/link_sas.hpp b/include/session/core/link_sas.hpp new file mode 100644 index 000000000..e065ea774 --- /dev/null +++ b/include/session/core/link_sas.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + +namespace session { + +using namespace std::literals; + +/// The 64 emoji used for short authentication strings, from the Matrix client-server API +/// specification v1.17, section 10.12.2.2.6. Selected for reasonable distinctiveness and +/// cross-platform compatibility. +inline constexpr std::array SAS_EMOJI = { + // clang-format off + "🐶"sv, "🐱"sv, "🦁"sv, "🐎"sv, "🦄"sv, "🐷"sv, "🐘"sv, "🐰"sv, + "🐼"sv, "🐓"sv, "🐧"sv, "🐢"sv, "🐟"sv, "🐙"sv, "🦋"sv, "🌷"sv, + "🌳"sv, "🌵"sv, "🍄"sv, "🌏"sv, "🌙"sv, "☁️"sv, "🔥"sv, "🍌"sv, + "🍎"sv, "🍓"sv, "🌽"sv, "🍕"sv, "🎂"sv, "❤️"sv, "😀"sv, "🤖"sv, + "🎩"sv, "👓"sv, "🔧"sv, "🎅"sv, "👍"sv, "☂️"sv, "⌛"sv, "⏰"sv, + "🎁"sv, "💡"sv, "📕"sv, "✏️"sv, "📎"sv, "✂️"sv, "🔒"sv, "🔑"sv, + "🔨"sv, "☎️"sv, "🏁"sv, "🚂"sv, "🚲"sv, "✈️"sv, "🚀"sv, "🏆"sv, + "⚽"sv, "🎸"sv, "🎺"sv, "🔔"sv, "⚓"sv, "🎧"sv, "📁"sv, "📌"sv, + // clang-format on +}; +static_assert(SAS_EMOJI.size() == 64); + +} // namespace session + +namespace session::core { + +/// Computes the Argon2id seed underlying the SAS for a device link request. The derivation is: +/// 1. salt = BLAKE2b-16(M, pers="SessionLinkEmoji") +/// 2. seed = Argon2id(M, salt, size=16, ops=2, mem=16MiB) +/// +/// The seed can be stored to avoid re-running the expensive Argon2id computation; pass it to +/// sas_from_seed() to recover the emoji sequence at any time. +std::array derive_sas_seed(std::span plaintext); + +/// Extracts the 21 SAS emoji from a pre-computed seed (as returned by derive_sas_seed). This is +/// a cheap bit-extraction operation with no cryptographic cost. +std::array sas_from_seed(std::span seed); + +/// Convenience wrapper: derives the seed and immediately returns the emoji sequence. +/// +/// Returns 21 string_view values (into the SAS_EMOJI table) for the full SAS sequence. The first +/// 7 are the standard short display; all 21 are available for the extended view. Formatting and +/// joining is left to the caller; the recommended layout is the first 7 joined with spaces for the +/// standard view, and 3 lines of 7 (joined with spaces within lines, newlines between) for the +/// extended view. +std::array link_request_sas(std::span plaintext); + +} // namespace session::core diff --git a/include/session/core/pro.hpp b/include/session/core/pro.hpp new file mode 100644 index 000000000..57b35e613 --- /dev/null +++ b/include/session/core/pro.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +#include "component.hpp" + +namespace session::pro_backend { +struct ProRevocationItem; +} +namespace session::core { + +class Core; + +class Pro final : detail::CoreComponent { + public: + /// API: core/Pro::pro_proof_is_revoked + /// + /// Check if the proof identified by its `revocation_tag` is revoked as of the given + /// timestamp from the list of proofs stored in the database. + /// + /// Outputs: + /// - `bool` -- True if the proof was revoked, false otherwise. + bool proof_is_revoked( + std::span revocation_tag, std::chrono::sys_seconds unix_ts); + + /// API: core/Pro::pro_update_revocations + /// + /// Update the list of pro revocations. If the `revocations_ticket` matches the current ticket, + /// this is a no-op. + /// + /// Inputs: + /// - `revocations_ticket` -- Ticket that describes the version of the revocations. This value + /// comes alongside the revocation list when queried. This ticket changes whenever the + /// revocation list is updated and is used to identify when an actual update is needed. + /// - `revocations` -- New list of Session Pro revocations. + /// - `retain_for` -- How long to keep each entry after it was last seen in a list, for + /// memory-only aging (from the revocation response's `retain_for`). + void update_revocations( + uint32_t ticket, + std::span revocations, + std::chrono::seconds retain_for); + + private: + friend class Core; + + explicit Pro(Core& core) : detail::CoreComponent{core} {} + + // Stores the version of the revocation list that we last updated. Used as an optimization to + // short-circuit updates that are the same as the previous update. + std::optional revocations_ticket_; +}; + +} // namespace session::core diff --git a/include/session/core/schema/schema_registry.hpp b/include/session/core/schema/schema_registry.hpp new file mode 100644 index 000000000..c087dce1c --- /dev/null +++ b/include/session/core/schema/schema_registry.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +namespace session::sqlite { +class Connection; +} +namespace session::core { +class Core; +} + +namespace session::core::schema { + +struct Migration { + std::string name; + void (*apply)(session::sqlite::Connection&, Core& core); +}; + +extern const std::span MIGRATIONS; + +/// The schema as it stands with every migration in MIGRATIONS applied, generated from the +/// directory's full_schema.sql; empty when that file does not exist. +/// +/// A database with none of this set's migrations applied is built from this and has them all +/// recorded without being run, so the file is both the fresh-install path and the one place to read +/// the current schema — rather than having to replay the migration chain in your head. +extern const std::string_view FULL_SCHEMA; + +} // namespace session::core::schema diff --git a/include/session/core/swarm_message.hpp b/include/session/core/swarm_message.hpp new file mode 100644 index 000000000..371ecbe60 --- /dev/null +++ b/include/session/core/swarm_message.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include +#include + +namespace session::core { + +/// A single message retrieved from the swarm, as returned by a retrieve request. The data, +/// hash, timestamp, and expiry fields are exactly the four values the server returns per +/// message; data is owned externally and must remain valid for the lifetime of this struct. +struct SwarmMessage { + std::span data; + std::string hash; + sys_ms timestamp; + sys_ms expiry; +}; + +} // namespace session::core diff --git a/include/session/crypto/ed25519.hpp b/include/session/crypto/ed25519.hpp new file mode 100644 index 000000000..baef7f27b --- /dev/null +++ b/include/session/crypto/ed25519.hpp @@ -0,0 +1,311 @@ +#pragma once + +#include +#include +#include +#include + +#include "session/sodium_array.hpp" +#include "session/util.hpp" + +namespace session::ed25519 { + +/// A span-like type representing a fully-expanded Ed25519 private key (always 64 bytes). +/// Implicitly constructible from any fixed-extent 32- or 64-byte byte/unsigned-char spannable: +/// - 32-byte input (seed): the 64-byte key is computed via libsodium and stored internally. +/// - 64-byte input: holds a non-owning span into the caller's data — no copy or allocation. +/// +/// Non-copyable and non-moveable to avoid dangling references to internal storage. + +/// Concept for types that implicitly convert to a 32- or 64-byte byte/unsigned-char span. +/// Used by PrivKeySpan and OptionalPrivKeySpan to accept Ed25519 seeds and full keys. +template +concept Ed25519KeySpannable = std::convertible_to> || + std::convertible_to> || + std::convertible_to> || + std::convertible_to>; + +struct PrivKeySpan { + template + PrivKeySpan(const T& src) { + if constexpr (std::convertible_to>) + data_ = std::span{src}.data(); + else if constexpr (std::convertible_to>) + data_ = reinterpret_cast( + std::span{src}.data()); + else if constexpr (std::convertible_to>) { + expand_seed(std::span{src}); + data_ = storage_->data(); + } else { + expand_seed(std::span{src}); + data_ = storage_->data(); + } + } + + // Constructor for runtime-known sizes (e.g. at C API boundaries). + // Throws std::invalid_argument if size is not 32 or 64. + PrivKeySpan(const std::byte* data, size_t size); + PrivKeySpan(const unsigned char* data, size_t size) : + PrivKeySpan{reinterpret_cast(data), size} {} + + // Named factory for dynamic-span input (runtime size check, throws if not 32 or 64). + static PrivKeySpan from(std::span key) { return {key.data(), key.size()}; } + static PrivKeySpan from(std::span key) { return {key.data(), key.size()}; } + + PrivKeySpan(const PrivKeySpan&) = delete; + PrivKeySpan& operator=(const PrivKeySpan&) = delete; + PrivKeySpan(PrivKeySpan&&) = delete; + PrivKeySpan& operator=(PrivKeySpan&&) = delete; + + std::span span() const { + return std::span(data_, 64); + } + operator std::span() const { return span(); } + operator std::span() const { return span(); } + const std::byte* data() const { return data_; } + auto begin() const { return data_; } + auto end() const { return data_ + 64; } + static constexpr size_t size() { return 64; } + // Returns the 32-byte seed (first half of the libsodium key). + std::span seed() const { return span().first<32>(); } + // Returns the 32-byte Ed25519 public key (second half of the libsodium key). + std::span pubkey() const { return span().last<32>(); } + + private: + void expand_seed(std::span seed); + void expand_seed(std::span seed); + + const std::byte* data_ = nullptr; + std::optional storage_; +}; + +/// Like PrivKeySpan but with an optional (nullable) state. Use this when a private key parameter +/// is optional; PrivKeySpan retains its always-has-value guarantee. +/// +/// Implicitly constructible from the same 32- or 64-byte sources as PrivKeySpan, plus from +/// default/nullopt for the empty state. Non-copyable and non-moveable for the same reason as +/// PrivKeySpan. +struct OptionalPrivKeySpan { + /// Constructs a null (no-key) state. + OptionalPrivKeySpan() = default; + OptionalPrivKeySpan(std::nullopt_t) {} + + template + OptionalPrivKeySpan(const T& src) : key_{std::in_place, src} {} + + // Constructor for runtime-known sizes (e.g. at C API boundaries). + // size == 0 produces the null state; size == 32 or 64 constructs the key. + // Throws std::invalid_argument if size is not 0, 32, or 64. + OptionalPrivKeySpan(const unsigned char* data, size_t size) { + if (size) + key_.emplace(data, size); + } + + OptionalPrivKeySpan(const OptionalPrivKeySpan&) = delete; + OptionalPrivKeySpan& operator=(const OptionalPrivKeySpan&) = delete; + OptionalPrivKeySpan(OptionalPrivKeySpan&&) = delete; + OptionalPrivKeySpan& operator=(OptionalPrivKeySpan&&) = delete; + + bool has_value() const { return key_.has_value(); } + explicit operator bool() const { return has_value(); } + + const PrivKeySpan& value() const { return key_.value(); } + const PrivKeySpan& operator*() const { return *key_; } + const PrivKeySpan* operator->() const { return &*key_; } + + private: + std::optional key_; +}; + +/// Generates a random Ed25519 key pair. +/// Write-to-output form. +void keypair(std::span pk, std::span sk); +/// Return-value form: returns {pubkey, seckey} where seckey uses cleared memory. +std::pair keypair(); + +/// Generates a deterministic Ed25519 key pair from a 32-byte seed. +/// Write-to-output form. +void seed_keypair( + std::span pk, + std::span sk, + std::span seed); +/// Return-value form: returns {pubkey, seckey} where seckey uses cleared memory. +std::pair keypair(std::span ed25519_seed); + +/// Returns the seed portion of an Ed25519 key as a non-owning span into the caller's data. +/// The overload accepting a 64-byte (libsodium-style) key returns the first 32 bytes (the seed). +/// The overload accepting a 32-byte value returns that span unchanged (it is already a seed). +inline std::span extract_seed( + std::span ed25519_privkey) noexcept { + return ed25519_privkey.first<32>(); +} +inline std::span extract_seed( + std::span ed25519_seed) noexcept { + return ed25519_seed; +} + +/// Generates a signature for the message using the libsodium-style ed25519 secret key, 64 bytes. +/// +/// Inputs: +/// - `ed25519_privkey` -- the Ed25519 private key; accepts a 32-byte seed or 64-byte libsodium key. +/// - `msg` -- the data to generate a signature for. +/// +/// Outputs: +/// - The 64-byte ed25519 signature +/// +/// Write-to-output form. +void sign( + std::span sig, + const PrivKeySpan& ed25519_privkey, + std::span msg); +/// Return-value form. +b64 sign(const PrivKeySpan& ed25519_privkey, std::span msg); + +/// Produces a random but validly-*encoded* Ed25519 signature: a uniformly random group element (the +/// `R` half) followed by a uniformly random scalar mod L (the `s` half). It is not a signature of +/// any message and verifies against nothing -- it exists only as a decoy, so a payload carrying a +/// real signature is indistinguishable on the wire from one carrying none. Far cheaper than signing +/// throwaway data, and independent of message size. +b64 decoy_signature(); + +/// Verify a message and signature for a given pubkey. +/// +/// Inputs: +/// - `sig` -- the signature to verify, 64 bytes. +/// - `pubkey` -- the pubkey for the secret key that was used to generate the signature, 32 bytes. +/// - `msg` -- the data to verify the signature for. +/// +/// Outputs: +/// - A flag indicating whether the signature is valid +bool verify( + std::span sig, + std::span pubkey, + std::span msg); + +/// Derives a deterministic Ed25519 keypair from a seed and a domain string. +/// +/// The derived seed is: Blake2b32(data=ed25519_seed, hash_key=domain) +/// +/// This is a general subkey derivation primitive; use a distinct domain string per use case +/// to produce independent derived keys from the same root seed. +/// +/// Returns the (pubkey, seckey) pair; the secret key uses cleared memory. +std::pair derive_subkey( + std::span ed25519_seed, std::span domain); + +/// Extracts the 32-byte public key from a 64-byte libsodium Ed25519 secret key. +/// Write-to-output form. +void sk_to_pk(std::span pk, const PrivKeySpan& sk); +/// Return-value form. +b32 sk_to_pk(const PrivKeySpan& sk); + +/// Returns true if `pk` is a usable Ed25519 public key: a canonical encoding of a point that is on +/// the curve and in its prime-order subgroup. +/// +/// Use this to validate a key that arrived from outside (a peer, a url, a server response) before +/// doing anything with it. Only a small fraction of 32-byte values satisfy this, so it rejects +/// almost all garbage -- but note that a well-formed key belonging to someone else passes just as +/// readily, so this answers "could this be a public key" and never "is this the right key". +bool is_valid_pubkey(std::span pk); + +/// Converts an Ed25519 public key to an X25519 public key. +/// Throws std::runtime_error if the key is invalid. +/// Write-to-output form: result written into `out`. +void pk_to_x25519(std::span out, std::span pk); +/// Return-value form. +b32 pk_to_x25519(std::span pk); + +/// Converts an Ed25519 public key to a 33-byte 0x05-prefixed Session ID by converting the +/// Ed25519 pubkey to X25519 and prefixing 0x05. +/// Write-to-output form: result written into `out`. +void pk_to_session_id(std::span out, std::span pk); +/// Return-value form. +b33 pk_to_session_id(std::span pk); + +/// Converts an Ed25519 secret key to an X25519 secret key. +/// Write-to-output form. +void sk_to_x25519(std::span out, std::span seed); +/// Return-value form (using cleared memory). +inline cleared_b32 sk_to_x25519(std::span seed) { + cleared_b32 xsk; + sk_to_x25519(xsk, seed); + return xsk; +} +/// Overload for a full 64-byte Ed25519 secret key (seed || pubkey); only the seed (first 32 +/// bytes) is used. +inline cleared_b32 sk_to_x25519(std::span full_key) { + return sk_to_x25519(full_key.first<32>()); +} +/// Overload for PrivKeySpan (deduced exactly, suppressing implicit conversions). +template T> +inline cleared_b32 sk_to_x25519(const T& sk) { + return sk_to_x25519(sk.seed()); +} + +/// Derives the X25519 {secret, public} key pair from an Ed25519 private key. +/// Equivalent to `{sk_to_x25519(sk), pk_to_x25519(sk.pubkey())}` but as a single call. +std::pair x25519_keypair(const PrivKeySpan& sk); + +/// Returns the private Ed25519 scalar `a` from a seed or PrivKeySpan (using cleared memory). +/// +/// Ed25519 and X25519 share the same private scalar: the Ed25519-to-X25519 conversion is +/// defined by using that same scalar on X25519's base point instead of Ed25519's. Use this +/// alias wherever the goal is to obtain the private scalar `a` rather than an X25519 key. +template + requires requires(Args&&... args) { sk_to_x25519(std::forward(args)...); } +inline decltype(auto) sk_to_private(Args&&... args) { + return sk_to_x25519(std::forward(args)...); +} + +/// Computes the Ed25519 group element from a scalar (clamped). +/// Write-to-output form: result written into `out`. +void scalarmult_base(std::span out, std::span scalar); +/// Return-value form. +b32 scalarmult_base(std::span scalar); + +/// Computes the Ed25519 group element from a scalar (no clamping). +/// Write-to-output form: result written into `out`. +void scalarmult_base_noclamp(std::span out, std::span scalar); +/// Return-value form. +b32 scalarmult_base_noclamp(std::span scalar); + +/// Multiplies an Ed25519 point by a scalar (no clamping). +/// Write-to-output form: result written into `out`. +void scalarmult_noclamp( + std::span out, + std::span scalar, + std::span point); +/// Return-value form. +b32 scalarmult_noclamp(std::span scalar, std::span point); + +/// Reduces a 64-byte scalar modulo the Ed25519 group order to 32 bytes. +/// Write-to-output form: result written into `out`. +void scalar_reduce(std::span out, std::span in); +/// Return-value form. +b32 scalar_reduce(std::span in); + +/// Negates a 32-byte Ed25519 scalar. +/// Write-to-output form: result written into `out` (safe to alias `in`). +void scalar_negate(std::span out, std::span in); +/// Return-value form. +b32 scalar_negate(std::span in); + +/// Multiplies two 32-byte Ed25519 scalars. +/// Write-to-output form: result written into `out` (safe to alias `x` or `y`). +void scalar_mul( + std::span out, + std::span x, + std::span y); +/// Return-value form. +b32 scalar_mul(std::span x, std::span y); + +/// Adds two 32-byte Ed25519 scalars. +/// Write-to-output form: result written into `out` (safe to alias `x` or `y`). +void scalar_add( + std::span out, + std::span x, + std::span y); +/// Return-value form. +b32 scalar_add(std::span x, std::span y); + +} // namespace session::ed25519 diff --git a/include/session/crypto/mlkem768.hpp b/include/session/crypto/mlkem768.hpp new file mode 100644 index 000000000..967812f3a --- /dev/null +++ b/include/session/crypto/mlkem768.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +#include "session/util.hpp" + +namespace session::mlkem768 { + +inline constexpr size_t PUBLICKEYBYTES = 1184; +inline constexpr size_t SECRETKEYBYTES = 2400; +inline constexpr size_t CIPHERTEXTBYTES = 1088; +inline constexpr size_t SHAREDSECRETBYTES = 32; +inline constexpr size_t SEEDBYTES = 64; // 2 * MLKEM_SYMBYTES + +/// Generates a keypair deterministically from a 64-byte seed. Throws on failure. +void keygen( + std::span pk, + std::span sk, + std::span seed); + +/// Encapsulates a shared secret to `pk` using a 32-byte random seed, writing the ciphertext and +/// shared secret into the provided spans. Throws on failure. +void encapsulate( + std::span ciphertext, + std::span shared_secret, + std::span pk, + std::span seed); + +/// Decapsulates a shared secret from `ciphertext` using `sk`. Returns false on failure. +bool decapsulate( + std::span shared_secret, + std::span ciphertext, + std::span sk); + +} // namespace session::mlkem768 diff --git a/include/session/crypto/x25519.hpp b/include/session/crypto/x25519.hpp new file mode 100644 index 000000000..748cf0a6e --- /dev/null +++ b/include/session/crypto/x25519.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include + +#include "session/sodium_array.hpp" +#include "session/util.hpp" + +namespace session::x25519 { + +/// Generates a random X25519 keypair. +/// Write-to-output form. +void keypair(std::span pk, std::span sk); +/// Return-value form: returns {pubkey, seckey}. +std::pair keypair(); + +/// Generates a deterministic X25519 keypair from a 32-byte seed. +/// Write-to-output form. +void seed_keypair( + std::span pk, + std::span sk, + std::span seed); +/// Return-value form: returns {pubkey, seckey}. +std::pair seed_keypair(std::span seed); + +/// Computes the X25519 public key corresponding to `sk`: out = sk * G. +/// Write-to-output form. +void scalarmult_base(std::span out, std::span scalar); +/// Return-value form. +b32 scalarmult_base(std::span scalar); + +/// Computes X25519 scalar multiplication: out = scalar * point. +/// Returns false if the result is the all-zeros point (degenerate case). +/// Write-to-output form. +bool scalarmult( + std::span out, + std::span scalar, + std::span point); +/// Return-value form. Throws on degenerate case. +b32 scalarmult(std::span scalar, std::span point); + +} // namespace session::x25519 diff --git a/include/session/curve25519.hpp b/include/session/curve25519.hpp deleted file mode 100644 index b476f5ac1..000000000 --- a/include/session/curve25519.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "types.hpp" - -namespace session::curve25519 { - -/// Generates a random curve25519 key pair -std::pair, std::array> curve25519_key_pair(); - -/// API: curve25519/to_curve25519_pubkey -/// -/// Generates a curve25519 public key for an ed25519 public key. -/// -/// Inputs: -/// - `ed25519_pubkey` -- the ed25519 public key. -/// -/// Outputs: -/// - The curve25519 public key -std::array to_curve25519_pubkey(std::span ed25519_pubkey); - -/// API: curve25519/to_curve25519_seckey -/// -/// Generates a curve25519 secret key given given a libsodium-style secret key, 64 -/// bytes. -/// -/// Inputs: -/// - `ed25519_seckey` -- the libsodium-style secret key, 64 bytes. -/// -/// Outputs: -/// - The curve25519 secret key -std::array to_curve25519_seckey(std::span ed25519_seckey); - -} // namespace session::curve25519 diff --git a/include/session/ed25519.hpp b/include/session/ed25519.hpp deleted file mode 100644 index 8a6de9213..000000000 --- a/include/session/ed25519.hpp +++ /dev/null @@ -1,73 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace session::ed25519 { - -/// Generates a random Ed25519 key pair -std::pair, std::array> ed25519_key_pair(); - -/// Given an Ed25519 seed this returns the associated Ed25519 key pair -std::pair, std::array> ed25519_key_pair( - std::span ed25519_seed); - -/// API: ed25519/seed_for_ed_privkey -/// -/// Returns the seed for an ed25519 key pair given either the libsodium-style secret key, 64 -/// bytes. If a 32-byte value is provided it is assumed to be the seed and the value will just -/// be returned directly. -/// -/// Inputs: -/// - `ed25519_privkey` -- the libsodium-style secret key of the sender, 64 bytes. Can also be -/// passed as a 32-byte seed. -/// -/// Outputs: -/// - The ed25519 seed -std::array seed_for_ed_privkey(std::span ed25519_privkey); - -/// API: ed25519/sign -/// -/// Generates a signature for the message using the libsodium-style ed25519 secret key, 64 bytes. -/// -/// Inputs: -/// - `ed25519_privkey` -- the libsodium-style secret key, 64 bytes. -/// - `msg` -- the data to generate a signature for. -/// -/// Outputs: -/// - The ed25519 signature -std::vector sign( - std::span ed25519_privkey, std::span msg); - -/// API: ed25519/verify -/// -/// Verify a message and signature for a given pubkey. -/// -/// Inputs: -/// - `sig` -- the signature to verify, 64 bytes. -/// - `pubkey` -- the pubkey for the secret key that was used to generate the signature, 32 bytes. -/// - `msg` -- the data to verify the signature for. -/// -/// Outputs: -/// - A flag indicating whether the signature is valid -bool verify( - std::span sig, - std::span pubkey, - std::span msg); - -/// API: ed25519/ed25519_pro_privkey_for_ed25519_seed -/// -/// Generate the deterministic Master Session Pro key for signing requests to interact with the -/// Session Pro features of the protocol. -/// -/// Inputs: -/// - `ed25519_seed` -- the seed to the long-term key for the Session account to derive the -/// deterministic key from. -/// -/// Outputs: -/// - The libsodium-style Master Session Pro Ed25519 secret key, 64 bytes. -std::array ed25519_pro_privkey_for_ed25519_seed( - std::span ed25519_seed); - -} // namespace session::ed25519 diff --git a/include/session/encrypt.hpp b/include/session/encrypt.hpp new file mode 100644 index 000000000..d396be799 --- /dev/null +++ b/include/session/encrypt.hpp @@ -0,0 +1,196 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "util.hpp" + +namespace session::encryption { + +// ─── Constants ─────────────────────────────────────────────────────────────── + +inline constexpr size_t XCHACHA20_KEYBYTES = 32; +inline constexpr size_t XCHACHA20_NONCEBYTES = 24; +inline constexpr size_t XCHACHA20_ABYTES = 16; // authentication tag size + +inline constexpr size_t BOX_PUBLICKEYBYTES = 32; +inline constexpr size_t BOX_SECRETKEYBYTES = 32; +inline constexpr size_t BOX_MACBYTES = 16; +inline constexpr size_t BOX_NONCEBYTES = 24; +inline constexpr size_t BOX_SEALBYTES = BOX_PUBLICKEYBYTES + BOX_MACBYTES; // 48 + +inline constexpr size_t SECRETBOX_KEYBYTES = 32; +inline constexpr size_t SECRETBOX_NONCEBYTES = 24; +inline constexpr size_t SECRETBOX_MACBYTES = 16; + +// ─── XChaCha20-Poly1305 AEAD ───────────────────────────────────────────────── + +/// Encrypts `msg` with `key` and `nonce`, writing ciphertext (msg.size() + ABYTES bytes) into +/// `out`. +inline void xchacha20poly1305_encrypt( + std::span out, + std::span msg, + std::span nonce, + std::span key) { + crypto_aead_xchacha20poly1305_ietf_encrypt( + ucdata(out), + nullptr, + ucdata(msg), + msg.size(), + nullptr, + 0, + nullptr, + ucdata(nonce), + ucdata(key)); +} + +/// Decrypts `ciphertext` with `key` and `nonce`, writing plaintext (ciphertext.size() - ABYTES +/// bytes) into `out`. Returns false if authentication fails. +inline bool xchacha20poly1305_decrypt( + std::span out, + std::span ciphertext, + std::span nonce, + std::span key) { + return 0 == crypto_aead_xchacha20poly1305_ietf_decrypt( + ucdata(out), + nullptr, + nullptr, + ucdata(ciphertext), + ciphertext.size(), + nullptr, + 0, + ucdata(nonce), + ucdata(key)); +} + +// ─── XChaCha20 stream ──────────────────────────────────────────────────────── + +/// XOR-encrypts/decrypts `in` with the XChaCha20 keystream derived from `nonce` and `key`, +/// writing the result into `out`. `out` and `in` must be the same size and may alias. +inline void xchacha20_xor( + std::span out, + std::span in, + std::span nonce, + std::span key) { + crypto_stream_xchacha20_xor(ucdata(out), ucdata(in), in.size(), ucdata(nonce), ucdata(key)); +} + +// ─── HChaCha20 ─────────────────────────────────────────────────────────────── + +/// Derives a 32-byte subkey from a 32-byte key and a 16-byte nonce prefix using HChaCha20. +/// This is the subkey-derivation step used internally by XChaCha20. +inline void hchacha20( + std::span out, + std::span nonce_prefix, + std::span key) { + crypto_core_hchacha20(ucdata(out), ucdata(nonce_prefix), ucdata(key), nullptr); +} + +// ─── Secretstream (streaming XChaCha20-Poly1305) ───────────────────────────── + +/// Initialises a secretstream pull (decryption) state from a header and key. +inline void secretstream_init_pull( + crypto_secretstream_xchacha20poly1305_state& st, + std::span header, + std::span key) { + crypto_secretstream_xchacha20poly1305_init_pull(&st, ucdata(header), ucdata(key)); +} + +/// Encrypts one chunk and appends it to the stream. `out` must be at least +/// `in.size() + crypto_secretstream_xchacha20poly1305_ABYTES` bytes. `ad` may be empty. +/// Returns the number of bytes written into `out`. +inline size_t secretstream_push( + crypto_secretstream_xchacha20poly1305_state& st, + std::span out, + std::span in, + std::span ad, + unsigned char tag) { + unsigned long long out_len; + crypto_secretstream_xchacha20poly1305_push( + &st, ucdata(out), &out_len, ucdata(in), in.size(), ucdata(ad), ad.size(), tag); + return static_cast(out_len); +} + +/// Decrypts one chunk from the stream. `out` must be at least +/// `in.size() - crypto_secretstream_xchacha20poly1305_ABYTES` bytes. `ad` may be empty. +/// Returns the number of bytes written and sets `tag_out` on success, or returns std::nullopt if +/// authentication fails. +inline std::optional secretstream_pull( + crypto_secretstream_xchacha20poly1305_state& st, + std::span out, + unsigned char& tag_out, + std::span in, + std::span ad = {}) { + unsigned long long out_len; + if (0 != + crypto_secretstream_xchacha20poly1305_pull( + &st, ucdata(out), &out_len, &tag_out, ucdata(in), in.size(), ucdata(ad), ad.size())) + return std::nullopt; + return static_cast(out_len); +} + +// ─── Box (X25519 + XSalsa20-Poly1305) ──────────────────────────────────────── + +/// Encrypts `msg` for `recipient_pk` from `sender_sk`, writing ciphertext into `out`. +/// `out` must be `msg.size() + BOX_MACBYTES` bytes. +inline void box_easy( + std::span out, + std::span msg, + std::span nonce, + std::span recipient_pk, + std::span sender_sk) { + if (0 != crypto_box_easy( + ucdata(out), + ucdata(msg), + msg.size(), + ucdata(nonce), + ucdata(recipient_pk), + ucdata(sender_sk))) + throw std::runtime_error{"crypto_box_easy failed (invalid public key?)"}; +} + +/// Seals `msg` for `pk` (anonymous sender), writing ciphertext into `out`. +/// `out` must be `msg.size() + BOX_SEALBYTES` bytes. +inline void box_seal( + std::span out, + std::span msg, + std::span pk) { + if (0 != crypto_box_seal(ucdata(out), ucdata(msg), msg.size(), ucdata(pk))) + throw std::runtime_error{"crypto_box_seal failed (invalid public key?)"}; +} + +/// Decrypts a sealed box. `out` must be `ciphertext.size() - BOX_SEALBYTES` bytes. +/// Returns false if authentication fails. +inline bool box_seal_open( + std::span out, + std::span ciphertext, + std::span pk, + std::span sk) { + return 0 == crypto_box_seal_open( + ucdata(out), ucdata(ciphertext), ciphertext.size(), ucdata(pk), ucdata(sk)); +} + +// ─── Secretbox (XSalsa20-Poly1305 with shared key) ─────────────────────────── + +/// Decrypts a secretbox ciphertext using a shared key. `out` must be +/// `ciphertext.size() - crypto_secretbox_MACBYTES` bytes. Returns false if authentication fails. +inline bool secretbox_open_easy( + std::span out, + std::span ciphertext, + std::span nonce, + std::span key) { + return 0 == + crypto_secretbox_open_easy( + ucdata(out), ucdata(ciphertext), ciphertext.size(), ucdata(nonce), ucdata(key)); +} + +} // namespace session::encryption diff --git a/include/session/fields.hpp b/include/session/fields.hpp index b70980d11..e70c50272 100644 --- a/include/session/fields.hpp +++ b/include/session/fields.hpp @@ -6,6 +6,8 @@ #include #include +#include "util.hpp" + namespace session { using namespace std::literals; @@ -28,17 +30,4 @@ struct Disappearing { std::chrono::seconds timer = 0s; }; -/// A Session ID: an x25519 pubkey, with a 05 identifying prefix. On the wire we send just the -/// 32-byte pubkey value (i.e. not hex, without the prefix). -struct SessionID { - /// The fixed session netid, 0x05 - static constexpr unsigned char netid = 0x05; - - /// The raw x25519 pubkey, as bytes - std::array pubkey; - - /// Returns the full pubkey in hex, including the netid prefix. - std::string hex() const; -}; - } // namespace session diff --git a/include/session/format.hpp b/include/session/format.hpp new file mode 100644 index 000000000..24e73ccbb --- /dev/null +++ b/include/session/format.hpp @@ -0,0 +1,218 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace session { + +/// Concept matching contiguous ranges of std::byte. +template +concept byte_spannable = std::ranges::contiguous_range && + std::same_as>, std::byte>; + +/// User-defined literals for convenient fmt::format usage, re-exported from oxen::log::literals. +/// +/// "_format" works like fmt::format but with the format string as a UDL: +/// +/// "xyz {}"_format(42) // returns std::string "xyz 42" +/// +/// "_format_to" appends in-place to an existing string (more efficient than +=): +/// +/// std::string s = "hello"; +/// "xyz {}"_format_to(s, 42) // s is now "helloxyz 42" +/// +/// Available via `using namespace session::literals;` or `using namespace session;`. +inline namespace literals { + using oxen::log::literals::operator""_format; + using oxen::log::literals::operator""_format_to; +} // namespace literals + +} // namespace session + +namespace fmt { + +// Disable fmt's generic range formatter for byte spans so that our byte_spannable formatter takes +// precedence (avoids ambiguity when fmt/ranges.h is also included). +template +struct range_format_kind + : std::integral_constant {}; + +/// Generic formatter for any byte_spannable type (std::span, std::array, std::vector of std::byte). +/// +/// Format spec: +/// {} or {:x} — full lowercase hex (default) +/// {:z} — hex with leading zero bytes stripped +/// {:a} — base32z encoding +/// {:b} — base64 encoding (padded) +/// {:B} — base64 encoding (unpadded) +/// {:r} — raw bytes +/// +/// Ellipsis truncation: use {:W.T} before any mode letter, where W is the total output width +/// (including the single "…" character) and T is the number of characters shown after the +/// ellipsis. W must be >= 2 and >= T+2. If the encoded value fits within W characters, no +/// truncation occurs. +/// +/// For example, with a 32-byte all-zero value: +/// {:x} → "0000000000000000000000000000000000000000000000000000000000000000" +/// {:z} → "0" +/// {:10.4} → "00000…0000" +/// {:9.4x} → "0000…0000" +template +struct formatter { + private: + enum class mode_t { full_hex, stripped_hex, b32z, b64, b64_unpadded, raw }; + mode_t mode = mode_t::full_hex; + bool do_ellipsis = false; + int ellipsis_width = -1, ellipsis_tail = -1; + + public: + constexpr fmt::format_parse_context::iterator parse(fmt::format_parse_context& ctx) { + auto it = ctx.begin(); + for (; it != ctx.end(); ++it) { + char c = *it; + if (c == '}') + break; + + bool mode_set = false; + switch (c) { + case 'x': + mode = mode_t::full_hex; + mode_set = true; + break; + case 'z': + mode = mode_t::stripped_hex; + mode_set = true; + break; + case 'r': + mode = mode_t::raw; + mode_set = true; + break; + case 'a': + mode = mode_t::b32z; + mode_set = true; + break; + case 'b': + mode = mode_t::b64; + mode_set = true; + break; + case 'B': + mode = mode_t::b64_unpadded; + mode_set = true; + break; + case '0': + // Leading zero before any width digits means zero-fill, which we don't support + if (!do_ellipsis && ellipsis_width == -1) + throw fmt::format_error{ + "invalid format for byte span: 0-fill is not supported"}; + [[fallthrough]]; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + auto& v = do_ellipsis ? ellipsis_tail : ellipsis_width; + v = (v < 0 ? 0 : v) * 10 + (c - '0'); + break; + } + case '.': + if (!do_ellipsis && ellipsis_width >= 2) { + do_ellipsis = true; + break; + } + [[fallthrough]]; + default: throw fmt::format_error{"invalid format spec for byte span"}; + } + + if (mode_set) { + if (++it == ctx.end() || *it != '}') + throw fmt::format_error{ + "invalid format for byte span: trailing characters after mode"}; + break; + } + } + + if (do_ellipsis) { + if (ellipsis_tail < 0) + throw fmt::format_error{ + "invalid ellipsis format for byte span: missing tail length after '.'"}; + if (ellipsis_tail > ellipsis_width - 2) + throw fmt::format_error{ + "invalid ellipsis format for byte span: width must be >= tail+2"}; + } else if (ellipsis_width >= 0) { + throw fmt::format_error{ + "invalid format for byte span: width specified without '.' and tail length"}; + } + + return it; + } + + auto format(const T& v, fmt::format_context& ctx) const { + const auto* data = reinterpret_cast(std::ranges::data(v)); + std::span bytes{data, std::ranges::size(v)}; + + fmt::memory_buffer buf; + auto out = do_ellipsis ? fmt::appender(buf) : ctx.out(); + + switch (mode) { + case mode_t::raw: out = std::copy(bytes.begin(), bytes.end(), out); break; + case mode_t::b64: out = oxenc::to_base64(bytes.begin(), bytes.end(), out); break; + case mode_t::b64_unpadded: + out = oxenc::to_base64(bytes.begin(), bytes.end(), out, false); + break; + case mode_t::b32z: out = oxenc::to_base32z(bytes.begin(), bytes.end(), out); break; + case mode_t::stripped_hex: { + auto it = bytes.begin(); + while (it != bytes.end() && *it == 0) + ++it; + if (it == bytes.end()) { + *out++ = '0'; + break; + } + // If the first remaining byte would produce a leading 0 in hex (e.g. 0x0a → "0a"), + // skip the leading '0' so the output starts with the significant hex digit. + if (*it < 16) { + char pair[2]; + oxenc::to_hex(it, it + 1, pair); + *out++ = pair[1]; + ++it; + } + out = oxenc::to_hex(it, bytes.end(), out); + break; + } + case mode_t::full_hex: + default: out = oxenc::to_hex(bytes.begin(), bytes.end(), out); break; + } + + if (!do_ellipsis) + return out; + + std::string_view full{buf.data(), buf.size()}; + auto final_out = ctx.out(); + if (full.size() <= static_cast(ellipsis_width)) { + final_out = std::copy(full.begin(), full.end(), final_out); + } else { + final_out = std::copy( + full.begin(), full.begin() + (ellipsis_width - 1 - ellipsis_tail), final_out); + constexpr std::string_view ellipsis_char{"…"}; + final_out = std::copy(ellipsis_char.begin(), ellipsis_char.end(), final_out); + final_out = std::copy(full.end() - ellipsis_tail, full.end(), final_out); + } + return final_out; + } +}; + +} // namespace fmt diff --git a/include/session/hash.hpp b/include/session/hash.hpp index 24e1a2382..7bafe983d 100644 --- a/include/session/hash.hpp +++ b/include/session/hash.hpp @@ -1,13 +1,23 @@ #pragma once #include +#include +#include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include #include -#include "types.hpp" +#include "session/util.hpp" namespace session::hash { @@ -22,10 +32,13 @@ namespace session::hash { /// - `msg` -- the message to generate a hash for. /// - `key` -- an optional key to be used when generating the hash. Can be omitted or an empty /// string for an unkeyed hash. Must be less than 64 bytes long. +/// +/// Deprecated: prefer hash::blake2b (unkeyed) or hash::blake2b_key (keyed) instead. +[[deprecated("Use hash::blake2b or hash::blake2b_key instead")]] void hash( - std::span hash, - std::span msg, - std::optional> key = std::nullopt); + std::span hash, + std::span msg, + std::optional> key = std::nullopt); /// API: hash/hash /// @@ -40,10 +53,381 @@ void hash( /// /// Outputs: /// - a `size` byte hash. -std::vector hash( +/// +/// Deprecated: prefer hash::blake2b (unkeyed) or hash::blake2b_key (keyed) instead. +[[deprecated("Use hash::blake2b or hash::blake2b_key instead")]] +std::vector hash( const size_t size, - std::span msg, - std::optional> key = std::nullopt); + std::span msg, + std::optional> key = std::nullopt); + +template +concept ByteContainer = + std::ranges::contiguous_range && oxenc::basic_char>; +template +concept HashInput = + ByteContainer || oxenc::endian_swappable_integer || std::same_as; + +namespace detail { + + template + std::integral_constant extract_extent(const std::array&); + template + std::integral_constant extract_extent(std::span); + template + std::integral_constant extract_extent(const T (&)[N]); + std::integral_constant extract_extent(...); + + template + constexpr size_t container_extent_v = decltype(extract_extent(std::declval()))::value; + + template + auto make_hashable(const U& val) { + if constexpr (ByteContainer) + return std::span{ + reinterpret_cast(std::ranges::data(val)), + std::ranges::size(val)}; + else if constexpr (oxenc::little_endian || sizeof(val) == 1) + return std::span{reinterpret_cast(&val), sizeof(val)}; + else { + std::array swapped; + oxenc::write_host_as_little(val, swapped.data()); + return swapped; + } + } + // Initializes a SHAKE-256 (or SHA3-256) keccak state with the given domain suffix byte and + // absorbs all of `args` into it. The domain byte distinguishes the hash function: + // - 0x1F = SHAKE-256 (crypto_xof_shake256_DOMAIN_STANDARD) + // - 0x06 = SHA3-256 + // See the sha3_256 API doc comment for the explanation of why this works. + template + requires(sizeof...(T) > 0) + void keccak_absorb(crypto_xof_shake256_state& st, unsigned char domain, const T&... args) { + crypto_xof_shake256_init_with_domain(&st, domain); + auto update = [&st](std::span arg) { + crypto_xof_shake256_update(&st, arg.data(), arg.size()); + }; + (update(make_hashable(args)), ...); + } + template + requires(sizeof...(T) > 0) + void update_all(crypto_generichash_blake2b_state& st, const T&... args) { + auto update_hash = [&st](std::span arg) { + crypto_generichash_blake2b_update(&st, arg.data(), arg.size()); + }; + (update_hash(make_hashable(args)), ...); + } + +} // namespace detail + +/// Concept for a fixed-size, writable byte container — the basic requirement for any hash output. +template +concept HashOutputContainer = + std::ranges::contiguous_range && !std::is_const_v> && + oxenc::basic_char> && + detail::container_extent_v != std::dynamic_extent && detail::container_extent_v >= 1; + +template +concept Blake2BOutputContainer = HashOutputContainer && detail::container_extent_v <= 64; + +template +concept Blake2BKey = + std::ranges::contiguous_range && oxenc::basic_char> && + (detail::container_extent_v == std::dynamic_extent || + detail::container_extent_v <= 64); + +/// Helper value to pass a null key to blake2b_key, blake2b_key_pers, or blake2b_hasher (e.g. when +/// only a personalisation string is wanted). +inline constexpr std::span nullkey{}; + +/// API: hash/blake2b_hasher +/// +/// Streaming (piecewise) BLAKE2b hasher with a compile-time fixed output size N (in [1, 64]). +/// Construct with an optional key and/or personalisation string, call update() with data pieces +/// in any order, then call finalize() to produce the result. +/// +/// The output size N is a template parameter and is fixed at construction, so init and finalize +/// always agree on the size. +/// +/// Like shake256, non-copyable and non-moveable; the internal state is zeroed on destruction. +/// +/// Constructors: +/// blake2b_hasher{} — no key, no pers +/// blake2b_hasher{key, nullopt} — key only +/// blake2b_hasher{nullkey, pers} — pers only +/// blake2b_hasher{key, pers} — key + pers +/// +/// The two-argument constructor has no default for pers to force explicit intent: if you want +/// only a key, you must write `std::nullopt`; if you want only a pers, you must write `nullkey`. +/// This prevents accidentally passing a `_b2b_pers` value as a key. +/// +/// Example: +/// +/// hash::blake2b_hasher<32> h{my_key, std::nullopt}; +/// h.update(part1, part2); // update with multiple args at once +/// h.update(part3); // or call update multiple times +/// auto result = h.finalize(); +/// +template + requires(N >= 1 && N <= 64) +struct blake2b_hasher { + crypto_generichash_blake2b_state st; + + /// No-key, no-pers constructor. + blake2b_hasher() { + crypto_generichash_blake2b_init_salt_personal(&st, nullptr, 0, N, nullptr, nullptr); + } + + /// Key + optional personalisation constructor. Pass `nullkey` as key for pers-only; + /// pass `std::nullopt` as pers for key-only. + /// + /// Dynamic-extent keys are silently truncated to 64 bytes (the blake2b key size limit); + /// static-extent keys are guaranteed ≤ 64 at compile time by the Blake2BKey concept. + template + blake2b_hasher(const Key& key, std::optional> pers) { + crypto_generichash_blake2b_init_salt_personal( + &st, + reinterpret_cast(std::ranges::data(key)), + std::min(std::ranges::size(key), 64), + N, + /*salt=*/nullptr, + pers ? reinterpret_cast(pers->data()) : nullptr); + } + + ~blake2b_hasher() { sodium_memzero(&st, sizeof(st)); } + + blake2b_hasher(const blake2b_hasher&) = delete; + blake2b_hasher& operator=(const blake2b_hasher&) = delete; + blake2b_hasher(blake2b_hasher&&) = delete; + blake2b_hasher& operator=(blake2b_hasher&&) = delete; + + /// Feeds one or more contiguous byte containers or integer values into the hash state, in + /// argument order. Integer values are written as raw bytes in little-endian encoding (i.e. + /// they will be byte-swapped on big-endian platforms if necessary). May be called multiple + /// times; each call appends to the state from previous calls. + template + requires(sizeof...(T) > 0) + blake2b_hasher& update(const T&... args) { + detail::update_all(st, args...); + return *this; + } + + /// Write-to-output finalize: writes the N-byte hash into `out`. + template + requires(detail::container_extent_v == N) + void finalize(Out& out) { + crypto_generichash_blake2b_final( + &st, reinterpret_cast(std::ranges::data(out)), N); + } + + /// Return-value finalize: returns a `std::array`. + std::array finalize() { + std::array result; + finalize(result); + return result; + } +}; + +/// API: hash/blake2b_key +/// +/// This version of blake2b() takes a key as the second argument and computes a keyed hash. The key +/// must be between 0 and 64 characters long. (A 0-length key is equivalent to no key). +/// +/// Two overloads are provided: +/// - write-to-output: `blake2b_key(out, key, args...)` writes the hash into `out` +/// - return-value: `blake2b_key(key, args...)` returns a `std::array` +template + requires(sizeof...(T) > 0) +void blake2b_key(Out& out, const Key& key, const T&... args) { + blake2b_hasher>{key, std::nullopt}.update(args...).finalize( + out); +} +template + requires(sizeof...(T) > 0 && N >= 1 && N <= 64) +std::array blake2b_key(const Key& key, const T&... args) { + std::array result; + blake2b_key(result, key, args...); + return result; +} + +/// API: hash/blake2b +/// +/// One-shot hasher that takes an output container and any number of contiguous byte containers or +/// integer values, computes the blake2b hash of the concatentation of the containers (in argument +/// order) and then writes the hash into the output container. Integer values are hashed as their +/// little-endian (fixed size) byte representation. +/// +/// This version uses neither key nor personalisation strings; see blake2b_key, blake2b_pers, and +/// blake2b_key_pers if you want one or both of those. +/// +/// Output must be a fixed extent span or containers (e.g. std::array), and must satisfy the blake2b +/// requirements (output size in [1,64]). +/// +/// It is permitted for overlap between the output and input containers; the output container is not +/// written until all input containers have been consumed. +/// +/// Two overloads are provided: +/// - write-to-output: `blake2b(out, args...)` writes the hash into `out` +/// - return-value: `blake2b(args...)` returns a `std::array` +template + requires(sizeof...(T) > 0) +void blake2b(Out& out, const T&... args) { + return blake2b_key(out, nullkey, args...); +} +template + requires(sizeof...(T) > 0 && N >= 1 && N <= 64) +std::array blake2b(const T&... args) { + std::array result; + blake2b(result, args...); + return result; +} + +/// API: hash/blake2b_key_pers +/// +/// This version of blake2b() takes a both a key and a 16-byte personalisation string as the second +/// and third arguments and computes a keyed hash with a personalisation string. The +/// personalisation string must be exactly 16 bytes, and is typically constructed with +/// "..."_b2b_pers for compile-time validation. The key must be between 0 and 64 bytes long. +/// +/// Two overloads are provided: +/// - write-to-output: `blake2b_key_pers(out, key, pers, args...)` writes the hash into `out` +/// - return-value: `blake2b_key_pers(key, pers, args...)` returns a `std::array` +template + requires(sizeof...(T) > 0) +void blake2b_key_pers( + Out& out, const Key& key, std::span pers, const T&... args) { + blake2b_hasher>{key, pers}.update(args...).finalize(out); +} +template + requires(sizeof...(T) > 0 && N >= 1 && N <= 64) +std::array blake2b_key_pers( + const Key& key, std::span pers, const T&... args) { + std::array result; + blake2b_key_pers(result, key, pers, args...); + return result; +} + +/// API: hash/blake2b_pers +/// +/// This version of blake2b() takes a 16-byte personality string as the second argument and computes +/// a unkeyed hash with a personalisation string. The personalization string must be exact 16 +/// bytes, and is typically constructed with "..."_b2b_pers for compile-time validation. +/// +/// Two overloads are provided: +/// - write-to-output: `blake2b_pers(out, pers, args...)` writes the hash into `out` +/// - return-value: `blake2b_pers(pers, args...)` returns a `std::array` +template + requires(sizeof...(T) > 0) +void blake2b_pers(Out& out, std::span pers, const T&... args) { + return blake2b_key_pers(out, nullkey, pers, args...); +} +template + requires(sizeof...(T) > 0 && N >= 1 && N <= 64) +std::array blake2b_pers(std::span pers, const T&... args) { + std::array result; + blake2b_pers(result, pers, args...); + return result; +} + +/// API: hash/shake256 +/// +/// SHAKE256 XOF hasher/squeezer. Construct with any number of contiguous byte containers or +/// integer values to absorb their concatenation, then call operator() with one or more fixed-size +/// output containers to squeeze output. Multiple operator() calls squeeze sequentially. Integer +/// values are absorbed as their little-endian (fixed-size) byte representation. +/// +/// Unlike blake2b, SHAKE256 has no key or personalisation mechanism; callers achieve domain +/// separation by simply prepending a domain string as the first argument. +/// +/// The internal keccak state is zeroed on destruction. +/// +/// Example: +/// +/// // Squeeze two outputs in one call: +/// hash::shake256("SessionMyKey"_bytes, seed)(out_a, out_b); +/// +/// // Or squeeze incrementally: +/// hash::shake256 sq{"SessionMyKey"_bytes, seed}; +/// sq(out_a); +/// sq(out_b); +/// +struct [[nodiscard]] shake256 { + crypto_xof_shake256_state st; + + template + requires(sizeof...(T) > 0) + explicit shake256(const T&... args) { + detail::keccak_absorb(st, crypto_xof_shake256_DOMAIN_STANDARD, args...); + } + + ~shake256() { sodium_memzero(&st, sizeof(st)); } + + shake256(const shake256&) = delete; + shake256& operator=(const shake256&) = delete; + shake256(shake256&&) = delete; + shake256& operator=(shake256&&) = delete; + + template + requires(sizeof...(Outs) > 0) + shake256& operator()(Outs&&... outs) { + (crypto_xof_shake256_squeeze( + &st, + reinterpret_cast(std::ranges::data(outs)), + std::ranges::size(outs)), + ...); + return *this; + } + + /// Squeezes N bytes from the state and returns them as a `std::array`. + template + requires(N >= 1) + std::array squeeze() { + std::array result; + (*this)(result); + return result; + } +}; + +/// API: hash/sha3_256 +/// +/// One-shot SHA3-256 (NIST FIPS 202) hasher. Takes a fixed-size 32-byte output container and any +/// number of contiguous byte containers or integer values, computes the SHA3-256 hash of their +/// concatenation (in argument order), and writes the result into the output container. Integer +/// values are hashed as their little-endian (fixed-size) byte representation. +/// +/// Implementation note: SHA3-256 and SHAKE-256 share identical Keccak-1600 sponge parameters +/// (state=1600 bits, rate=136 bytes, capacity=512 bits) and differ *only* in the domain suffix +/// byte absorbed into the state during padding before the final squeeze: +/// +/// - SHAKE-256: 0x1F (FIPS 202 §6.2 XOF suffix '11111') +/// - SHA3-256: 0x06 (FIPS 202 §6.1 hash suffix '01', plus the leading '1' of the Keccak +/// multi-rate padding '10*1', making the combined byte '0000 0110') +/// +/// Because the sponge parameters are identical, crypto_xof_shake256_init_with_domain(&st, 0x06) +/// followed by absorbing input and squeezing 32 bytes is exactly SHA3-256. This is the intended +/// use of init_with_domain, not a workaround. +/// +/// The temporary keccak state is zeroed before this function returns. +/// +/// Two overloads are provided: +/// - write-to-output: `sha3_256(out, args...)` writes the hash into `out` +/// - return-value: `sha3_256<32>(args...)` returns a `std::array` +template + requires(detail::container_extent_v == 32 && sizeof...(T) > 0) +void sha3_256(Out& out, const T&... args) { + crypto_xof_shake256_state st; + detail::keccak_absorb(st, 0x06, args...); + crypto_xof_shake256_squeeze(&st, reinterpret_cast(std::ranges::data(out)), 32); + sodium_memzero(&st, sizeof(st)); +} +template + requires(N == 32 && sizeof...(T) > 0) +std::array sha3_256(const T&... args) { + std::array result; + sha3_256(result, args...); + return result; +} // Helper callable usable with unordered_map and similar to hash an array of chars by simply copying // the first sizeof(size_t) bytes, suitable for use with pre-hashed values. @@ -57,4 +441,99 @@ struct identity_hasher { } }; +// ─── SHA-512 ───────────────────────────────────────────────────────────────── + +/// One-shot SHA-512 hasher. Takes a fixed-size 64-byte output container and any number of +/// contiguous byte containers or integer values, computes the SHA-512 hash of their concatenation +/// (in argument order), and writes the result into the output container. +template + requires(detail::container_extent_v == crypto_hash_sha512_BYTES && sizeof...(T) > 0) +void sha512(Out& out, const T&... args) { + crypto_hash_sha512_state st; + crypto_hash_sha512_init(&st); + auto update = [&st](std::span arg) { + crypto_hash_sha512_update(&st, arg.data(), arg.size()); + }; + (update(detail::make_hashable(args)), ...); + crypto_hash_sha512_final(&st, reinterpret_cast(std::ranges::data(out))); + sodium_memzero(&st, sizeof(st)); +} + +// ─── HMAC-SHA-256 ──────────────────────────────────────────────────────────── + +/// One-shot HMAC-SHA-256. Takes a fixed-size 32-byte output container, a key (any byte +/// container), and any number of contiguous byte containers or integer values, computes the +/// HMAC-SHA-256 of their concatenation and writes the result into the output container. +template + requires(detail::container_extent_v == crypto_auth_hmacsha256_BYTES && sizeof...(T) > 0) +void hmac_sha256(Out& out, const Key& key, const T&... args) { + crypto_auth_hmacsha256_state st; + crypto_auth_hmacsha256_init( + &st, + reinterpret_cast(std::ranges::data(key)), + std::ranges::size(key)); + auto update = [&st](std::span arg) { + crypto_auth_hmacsha256_update(&st, arg.data(), arg.size()); + }; + (update(detail::make_hashable(args)), ...); + crypto_auth_hmacsha256_final(&st, reinterpret_cast(std::ranges::data(out))); + sodium_memzero(&st, sizeof(st)); +} + +// ─── Argon2id (password hashing / KDF) ─────────────────────────────────────── + +inline constexpr size_t ARGON2_SALTBYTES = crypto_pwhash_SALTBYTES; +inline constexpr unsigned long long ARGON2_OPSLIMIT_MODERATE = crypto_pwhash_OPSLIMIT_MODERATE; +inline constexpr size_t ARGON2_MEMLIMIT_MODERATE = crypto_pwhash_MEMLIMIT_MODERATE; +inline constexpr int ARGON2ID13 = crypto_pwhash_ALG_ARGON2ID13; + +/// Derives a key from a password using Argon2id (libsodium crypto_pwhash). +/// Throws std::runtime_error if the derivation fails (e.g. out of memory). +/// +/// Inputs: +/// - `out` -- writable byte container to receive the derived key (between 16 and 4294967295 +/// bytes). +/// - `password` -- the password/input data. +/// - `salt` -- the 16-byte Argon2 salt (`crypto_pwhash_SALTBYTES`). +/// - `opslimit` -- CPU cost parameter (e.g. `crypto_pwhash_OPSLIMIT_MODERATE`). +/// - `memlimit` -- memory cost parameter (e.g. `crypto_pwhash_MEMLIMIT_MODERATE`). +/// - `alg` -- algorithm selector (e.g. `crypto_pwhash_ALG_ARGON2ID13`). +template + requires(detail::container_extent_v >= crypto_pwhash_BYTES_MIN) +void argon2( + Out& out, + std::span password, + std::span salt, + unsigned long long opslimit, + size_t memlimit, + int alg) { + if (0 != crypto_pwhash( + reinterpret_cast(std::ranges::data(out)), + std::ranges::size(out), + password.data(), + password.size(), + reinterpret_cast(salt.data()), + opslimit, + memlimit, + alg)) + throw std::runtime_error{"crypto_pwhash failed (out of memory?)"}; +} + } // namespace session::hash + +namespace session { inline namespace literals { + + /// User-defined literal for a 16-byte personalization value for use with BLAKE2b. + /// Enforces the 16-byte length at compile time via the requires clause. Returns a + /// fixed-extent span so it passes directly to blake2b_pers / blake2b_key_pers. Example: + /// + /// using namespace session::literals; // or `using namespace session;` + /// constexpr auto PERS_XYZ = "XYZ-XYZ-XYZ-WXYZ"_b2b_pers; + /// + template + requires(Lit.size == 16) + consteval auto operator""_b2b_pers() { + return operator""_bytes < Lit>(); + } + +}} // namespace session::literals diff --git a/include/session/logging.hpp b/include/session/logging.hpp index 004999019..542190d24 100644 --- a/include/session/logging.hpp +++ b/include/session/logging.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -11,6 +12,10 @@ namespace spdlog::level { enum level_enum : int; } +namespace oxen::log { +class formatted_callback_sink; +} // namespace oxen::log + namespace session { // This is working roughly like an enum class, but with some useful conversions and comparisons @@ -45,6 +50,11 @@ inline const LogLevel LogLevel::warn{LOG_LEVEL_WARN}; inline const LogLevel LogLevel::error{LOG_LEVEL_ERROR}; inline const LogLevel LogLevel::critical{LOG_LEVEL_CRITICAL}; +/// A registered logger, as returned by `add_logger` and accepted by `remove_logger`. The sink is +/// only forward-declared, so spdlog stays out of this header: hold the handle, don't dereference +/// it. +using LoggerHandle = std::shared_ptr; + /// API: add_logger /// /// Adds a logger callback for oxen-logging log messages (such as from the network object). @@ -56,10 +66,26 @@ inline const LogLevel LogLevel::critical{LOG_LEVEL_CRITICAL}; /// callback(std::string_view msg) /// callback(std::string_view msg, std::string_view log_cat, LogLevel level) /// -void add_logger(std::function cb); -void add_logger( +/// Outputs: +/// - a handle naming this logger, for `remove_logger`. Ignoring it is fine if the logger is +/// meant to last as long as the process. +LoggerHandle add_logger(std::function cb); +LoggerHandle add_logger( std::function cb); +/// API: session/remove_logger +/// +/// Removes a logger added by `add_logger`. Removing one that is not registered does nothing. +/// +/// Logging is serialised against this, so once it returns the callback is neither running nor +/// reachable, and whatever it captured can be destroyed. That is the difference from +/// `clear_loggers`, which drops every logger in the process including ones this caller does not +/// own. +/// +/// Inputs: +/// - `logger` -- [in] the handle returned by `add_logger`. +void remove_logger(const LoggerHandle& logger); + /// API: session/logger_reset_level /// /// Resets the log level of all existing category loggers, and sets a new default for any created diff --git a/include/session/mnemonics.hpp b/include/session/mnemonics.hpp new file mode 100644 index 000000000..492ca3a72 --- /dev/null +++ b/include/session/mnemonics.hpp @@ -0,0 +1,196 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::mnemonics { + +/** + * The number of words in each mnemonic language word list. + * + * The encoding uses 3 words per 32-bit chunk, so 24 words encodes 256 bits. 1626 was chosen + * because 1626³ (≈ 4.299 × 10⁹) just barely exceeds 2³² (≈ 4.295 × 10⁹), meaning three words + * can represent any 32-bit value — with a small number of 3-word combinations (~0.09%) that + * exceed 2³²-1 and are therefore invalid. + */ +constexpr size_t NWORDS = 1626; + +/** + * A struct containing information about a mnemonic language. + * + * All string values (english_name, native_name, and words) are encoded in UTF-8. + * + * prefix_len represents the number of unique UTF-8 codepoints (not bytes) required + * to uniquely identify a word in this language. + */ +struct Mnemonics { + std::string_view english_name; + std::string_view native_name; + int prefix_len; + std::array words; +}; + +/// Exception thrown when a word is not found in the mnemonic dictionary. +class unknown_word_error : public std::invalid_argument { + public: + explicit unknown_word_error(std::string word); + + /// The word that was not found in the dictionary. + const std::string& word() const { return word_; } + + private: + std::string word_; +}; + +/// Exception thrown when a checksum word is present but does not match the expected value. +class checksum_error : public std::invalid_argument { + public: + checksum_error(); +}; + +/// Exception thrown when a language name is not found in the language registry. +class unknown_language_error : public std::invalid_argument { + public: + explicit unknown_language_error(std::string name); + + /// The language name that was not found. + const std::string& name() const { return name_; } + + private: + std::string name_; +}; + +/** + * Returns a list of all supported mnemonic languages. + * English is always the first element, followed by other languages sorted by name. + */ +std::span get_languages(); + +/** + * Finds a language by its English or native name. + * + * @param name The name to look for. + * @return A pointer to the Mnemonics struct if found, otherwise nullptr. + */ +const Mnemonics* find_language(std::string_view name); + +/** + * Looks up a language by its English or native name, throwing if not found. + * + * @param name The name to look for. + * @return A reference to the Mnemonics struct. + * @throws unknown_language_error if the language name is not found. + */ +const Mnemonics& get_language(std::string_view name); + +/// Stores mnemonic string_view objects (each pointing into the language word list) in secure +/// (sodium) memory so that the word identities are zeroed on destruction. +/// +/// Call open() to iterate over the words. The returned opened_span holds a read accessor +/// that keeps the buffer readable for its own lifetime, so the following are both safe: +/// +/// for (auto w : m.open()) { ... } +/// auto s = m.open(); for (auto w : s.words) { ... } +/// +/// Do NOT do: `for (auto w : m.open().words)` — the opened_span (and its accessor) would be +/// destroyed before the loop body runs, re-locking the buffer and causing a crash. +struct secure_mnemonic { + session::secure_buffer storage; + + struct opened_span { + session::secure_buffer::r_accessor acc; + std::span words; + + const std::string_view& operator[](size_t i) const { return words[i]; } + const std::string_view* begin() const { return words.data(); } + const std::string_view* end() const { return words.data() + words.size(); } + }; + + opened_span open() { + auto acc = storage.access(); + std::span words{ + reinterpret_cast(acc.buf.data()), + acc.buf.size() / sizeof(std::string_view)}; + return {std::move(acc), words}; + } + + size_t size() const { return storage.size() / sizeof(std::string_view); } +}; + +/** + * Converts a byte span to a mnemonic word list using the specified language, stored in secure + * memory. + * + * @param bytes The input byte span. Its length must be a multiple of 4. + * @param lang The language to use for the mnemonic. + * @param checksum If true (the default), append a checksum word after the encoded words. The + * checksum word repeats one of the seed words, chosen by a CRC-32 over their concatenated + * prefixes (the first `prefix_len` codepoints of each) modulo the word count. This matches + * the Monero/Electrum scheme that Session clients use. + * + * @return A secure_mnemonic containing the words, plus a checksum word if requested. + * @throws std::invalid_argument if the input length is not a multiple of 4. + */ +secure_mnemonic bytes_to_words( + std::span bytes, const Mnemonics& lang, bool checksum = true); + +/// Same as above, but takes a language by name instead of by reference. +/// @throws unknown_language_error if the language name is not found. +secure_mnemonic bytes_to_words( + std::span bytes, std::string_view lang_name, bool checksum = true); + +/** + * Converts a mnemonic word list to bytes using the specified language, stored in secure memory. + * + * Accepts a word count that is either a multiple of 3 (no checksum) or one more than a multiple + * of 3 (with checksum). If a checksum word is present it is validated. + * + * @param words The input word list. + * @param lang The language used for the mnemonic. + * @return A secure_buffer containing the decoded bytes. + * @throws std::invalid_argument if the input length is invalid, or if the word sequence encodes + * an invalid (overflowing) value. + * @throws unknown_word_error if a word is not found in the language dictionary. + * @throws checksum_error if a checksum word is present but does not match. + */ +session::secure_buffer words_to_bytes( + std::span words, const Mnemonics& lang); + +/// Same as above, but takes a language by name instead of by reference. +/// @throws unknown_language_error if the language name is not found. +session::secure_buffer words_to_bytes( + std::span words, std::string_view lang_name); + +/** + * Converts a mnemonic word list to bytes, writing directly into a caller-provided output span. + * + * The size of `out` determines the expected number of seed words: out.size() must be a multiple + * of 4, and words.size() must equal (out.size() / 4 * 3) or (out.size() / 4 * 3) + 1 (the + * latter if a checksum word is appended). + * + * @param words The input word list. + * @param lang The language used for the mnemonic. + * @param out Output span to write decoded bytes into; must be a multiple-of-4 size exactly + * matching the decoded byte count implied by the word count. + * @throws std::invalid_argument if the word count does not match the output size, the word + * sequence encodes an invalid (overflowing) value, or out.size() is not a multiple of 4. + * @throws unknown_word_error if a word is not found in the language dictionary. + * @throws checksum_error if a checksum word is present but does not match. + */ +void words_to_bytes( + std::span words, const Mnemonics& lang, std::span out); + +/// Same as above, but takes a language by name instead of by reference. +/// @throws unknown_language_error if the language name is not found. +void words_to_bytes( + std::span words, + std::string_view lang_name, + std::span out); + +} // namespace session::mnemonics diff --git a/include/session/multi_encrypt.hpp b/include/session/multi_encrypt.hpp index 08c78a4b5..5c93a0152 100644 --- a/include/session/multi_encrypt.hpp +++ b/include/session/multi_encrypt.hpp @@ -8,6 +8,7 @@ #include #include +#include "crypto/ed25519.hpp" #include "sodium_array.hpp" #include "types.hpp" @@ -33,29 +34,29 @@ namespace session { namespace detail { void encrypt_multi_key( - std::array& key_out, - const unsigned char* a, - const unsigned char* A, - const unsigned char* B, + std::span key_out, + std::span a, + std::span A, + std::span B, bool encrypting, std::string_view domain); void encrypt_multi_impl( - std::vector& out, - std::span message, - const unsigned char* key, - const unsigned char* nonce); + std::vector& out, + std::span message, + std::span key, + std::span nonce); bool decrypt_multi_impl( - std::vector& out, - std::span ciphertext, - const unsigned char* key, - const unsigned char* nonce); + std::vector& out, + std::span ciphertext, + std::span key, + std::span nonce); inline void validate_multi_fields( - std::span nonce, - std::span privkey, - std::span pubkey) { + std::span nonce, + std::span privkey, + std::span pubkey) { if (nonce.size() < 24) throw std::logic_error{"nonce must be 24 bytes"}; if (privkey.size() != 32) @@ -76,7 +77,7 @@ extern const size_t encrypt_multiple_message_overhead; /// API: crypto/encrypt_for_multiple /// /// Encrypts a message multiple times for multiple recipients. `callable` is invoked once per -/// encrypted (or junk) value, passed as a `std::span`. +/// encrypted (or junk) value, passed as a `std::span`. /// /// Inputs: /// - `messages` -- a vector of message bodies to encrypt. Must be either size 1, or of the same @@ -97,20 +98,19 @@ extern const size_t encrypt_multiple_message_overhead; /// used to generate individual keys for domain separation, and so should ideally have a different /// value in different contexts (i.e. group keys uses one value, kicked messages use another, /// etc.). *Can* be empty, but should be set to something. -/// - `call` -- this is invoked for each different encrypted value with a std::span; the caller -/// must copy as needed as the std::span doesn't remain valid past the call. +/// - `call` -- this is invoked for each different encrypted value with a std::span; the caller must copy as needed as the span doesn't remain valid past the call. /// - `ignore_invalid_recipient` -- if given and true then any recipients that appear to have /// invalid public keys (i.e. the shared key multiplication fails) will be silently ignored (the /// callback will not be called). If not given (or false) then such a failure for any recipient /// will raise an exception. template void encrypt_for_multiple( - const std::vector> messages, - const std::vector> recipients, - std::span nonce, - std::span privkey, - std::span pubkey, + const std::vector> messages, + const std::vector> recipients, + std::span nonce, + std::span privkey, + std::span pubkey, std::string_view domain, F&& call, bool ignore_invalid_recipient = false) { @@ -129,24 +129,25 @@ void encrypt_for_multiple( if (auto sz = m.size(); sz > max_msg_size) max_msg_size = sz; - std::vector encrypted; + std::vector encrypted; encrypted.reserve(max_msg_size + encrypt_multiple_message_overhead); - sodium_cleared> key; + cleared_b32 key; auto msg_it = messages.begin(); for (const auto& r : recipients) { const auto& m = *msg_it; if (messages.size() > 1) ++msg_it; try { - detail::encrypt_multi_key(key, privkey.data(), pubkey.data(), r.data(), true, domain); + detail::encrypt_multi_key( + key, privkey.first<32>(), pubkey.first<32>(), r.first<32>(), true, domain); } catch (const std::exception&) { if (ignore_invalid_recipient) continue; else throw; } - detail::encrypt_multi_impl(encrypted, m, key.data(), nonce.data()); + detail::encrypt_multi_impl(encrypted, m, key, nonce.first<24>()); call(to_span(encrypted)); } } @@ -154,7 +155,7 @@ void encrypt_for_multiple( /// Wrapper for passing a single message for all recipients; all arguments other than the first are /// identical. template -void encrypt_for_multiple(std::span message, Args&&... args) { +void encrypt_for_multiple(std::span message, Args&&... args) { return encrypt_for_multiple( to_view_vector(&message, &message + 1), std::forward(args)...); } @@ -162,21 +163,17 @@ template void encrypt_for_multiple(std::string_view message, Args&&... args) { return encrypt_for_multiple(to_span(message), std::forward(args)...); } -template -void encrypt_for_multiple(std::span message, Args&&... args) { - return encrypt_for_multiple(to_span(message), std::forward(args)...); -} /// API: crypto/decrypt_for_multiple /// /// Decryption via a lambda: we call the lambda (which must return a std::optional>) repeatedly until we get back a nullopt, and attempt to decrypt each returned +/// std::byte>>) repeatedly until we get back a nullopt, and attempt to decrypt each returned /// value. When decryption succeeds, we return the plaintext to the caller. If none of the fed-in /// values can be decrypt, we return std::nullopt. /// /// Inputs: -/// - `ciphertext` -- callback that returns a std::optional> or -/// std::optional> +/// - `ciphertext` -- callback that returns a std::optional> or +/// std::optional> /// when called, containing the next ciphertext; should return std::nullopt when finished. /// - `nonce` -- the nonce used for encryption/decryption (which must have been provided by the /// sender alongside the encrypted messages, and is the same as the `nonce` value given to @@ -191,34 +188,36 @@ void encrypt_for_multiple(std::span message, Args&&... args) { template < typename NextCiphertext, typename = std::enable_if_t< + std::is_invocable_r_v>, NextCiphertext> || + std::is_invocable_r_v>, NextCiphertext> || std::is_invocable_r_v< std::optional>, - NextCiphertext> || - std::is_invocable_r_v>, NextCiphertext> || + NextCiphertext> || // legacy + std::is_invocable_r_v< + std::optional>, + NextCiphertext> || // legacy std::is_invocable_r_v, NextCiphertext> || - std::is_invocable_r_v, NextCiphertext> || - std::is_invocable_r_v>, NextCiphertext> || - std::is_invocable_r_v>, NextCiphertext>>> -std::optional> decrypt_for_multiple( + std::is_invocable_r_v, NextCiphertext>>> +std::optional> decrypt_for_multiple( NextCiphertext next_ciphertext, - std::span nonce, - std::span privkey, - std::span pubkey, - std::span sender_pubkey, + std::span nonce, + std::span privkey, + std::span pubkey, + std::span sender_pubkey, std::string_view domain) { detail::validate_multi_fields(nonce, privkey, pubkey); if (sender_pubkey.size() != 32) throw std::logic_error{"pubkey requires a 32-byte pubkey"}; - sodium_cleared> key; + cleared_b32 key; detail::encrypt_multi_key( - key, privkey.data(), pubkey.data(), sender_pubkey.data(), false, domain); + key, privkey.first<32>(), pubkey.first<32>(), sender_pubkey.first<32>(), false, domain); - auto decrypted = std::make_optional>(); + auto decrypted = std::make_optional>(); for (auto ciphertext = next_ciphertext(); ciphertext; ciphertext = next_ciphertext()) - if (detail::decrypt_multi_impl(*decrypted, *ciphertext, key.data(), nonce.data())) + if (detail::decrypt_multi_impl(*decrypted, *ciphertext, key, nonce.first<24>())) return decrypted; decrypted.reset(); @@ -243,12 +242,12 @@ std::optional> decrypt_for_multiple( /// - `domain` -- the encryption domain; this is typically a hard-coded string, and must be the same /// as the one used for encryption. /// -std::optional> decrypt_for_multiple( - const std::vector>& ciphertexts, - std::span nonce, - std::span privkey, - std::span pubkey, - std::span sender_pubkey, +std::optional> decrypt_for_multiple( + const std::vector>& ciphertexts, + std::span nonce, + std::span privkey, + std::span pubkey, + std::span sender_pubkey, std::string_view domain); /// API: crypto/encrypt_for_multiple_simple @@ -292,28 +291,28 @@ std::optional> decrypt_for_multiple( /// entries will be somewhat identifiable. /// /// Outputs: -/// std::vector containing bytes that contains the nonce and encoded encrypted -/// messages, suitable for decryption by the recipients with `decrypt_for_multiple_simple`. -std::vector encrypt_for_multiple_simple( - const std::vector>& messages, - const std::vector>& recipients, - std::span privkey, - std::span pubkey, +/// std::vector containing the nonce and encoded encrypted messages, suitable for +/// decryption by the recipients with `decrypt_for_multiple_simple`. +std::vector encrypt_for_multiple_simple( + const std::vector>& messages, + const std::vector>& recipients, + std::span privkey, + std::span pubkey, std::string_view domain, - std::optional> nonce = std::nullopt, + std::optional> nonce = std::nullopt, int pad = 0); /// API: crypto/encrypt_for_multiple_simple /// /// This function is the same as the above, except that instead of taking the sender private and -/// public X25519 keys, it takes the single, 64-byte libsodium Ed25519 secret key (which is then -/// converted into the required X25519 keys). -std::vector encrypt_for_multiple_simple( - const std::vector>& messages, - const std::vector>& recipients, - std::span ed25519_secret_key, +/// public X25519 keys, it takes the Ed25519 private key (32-byte seed or 64-byte libsodium key, +/// which is then converted into the required X25519 keys). +std::vector encrypt_for_multiple_simple( + const std::vector>& messages, + const std::vector>& recipients, + const ed25519::PrivKeySpan& ed25519_secret_key, std::string_view domain, - std::span nonce = {}, + std::optional> nonce = std::nullopt, int pad = 0); /// API: crypto/encrypt_for_multiple_simple @@ -323,19 +322,14 @@ std::vector encrypt_for_multiple_simple( /// the first are identical. /// template -std::vector encrypt_for_multiple_simple( - std::span message, Args&&... args) { +std::vector encrypt_for_multiple_simple( + std::span message, Args&&... args) { return encrypt_for_multiple_simple( to_view_vector(&message, &message + 1), std::forward(args)...); } template -std::vector encrypt_for_multiple_simple(std::string_view message, Args&&... args) { - return encrypt_for_multiple_simple(to_span(message), std::forward(args)...); -} -template -std::vector encrypt_for_multiple_simple( - std::span message, Args&&... args) { - return encrypt_for_multiple_simple(to_span(message), std::forward(args)...); +std::vector encrypt_for_multiple_simple(std::string_view message, Args&&... args) { + return encrypt_for_multiple_simple(to_span(message), std::forward(args)...); } /// API: crypto/decrypt_for_multiple_simple @@ -356,36 +350,36 @@ std::vector encrypt_for_multiple_simple( /// `encrypt_for_multiple_simple`. /// /// Outputs: -/// If decryption succeeds, returns a std::vector containing the decrypted message, -/// in bytes. If parsing or decryption fails, returns std::nullopt. -std::optional> decrypt_for_multiple_simple( - std::span encoded, - std::span privkey, - std::span pubkey, - std::span sender_pubkey, +/// If decryption succeeds, returns a std::vector containing the decrypted message. +/// If parsing or decryption fails, returns std::nullopt. +std::optional> decrypt_for_multiple_simple( + std::span encoded, + std::span privkey, + std::span pubkey, + std::span sender_pubkey, std::string_view domain); /// API: crypto/decrypt_for_multiple_simple /// /// This is the same as the above, except that instead of taking an X25519 private and public key -/// arguments, it takes a single, 64-byte Ed25519 secret key and converts it to X25519 to perform -/// the decryption. +/// arguments, it takes the Ed25519 private key (32-byte seed or 64-byte libsodium key) and +/// converts it to X25519 to perform the decryption. /// /// Note that `sender_pubkey` is still an X25519 pubkey for this version of the function. -std::optional> decrypt_for_multiple_simple( - std::span encoded, - std::span ed25519_secret_key, - std::span sender_pubkey, +std::optional> decrypt_for_multiple_simple( + std::span encoded, + const ed25519::PrivKeySpan& ed25519_secret_key, + std::span sender_pubkey, std::string_view domain); /// API: crypto/decrypt_for_multiple_simple_ed25519 /// /// This is the same as the above, except that it takes both the sender and recipient as Ed25519 /// keys, converting them on the fly to attempt the decryption. -std::optional> decrypt_for_multiple_simple_ed25519( - std::span encoded, - std::span ed25519_secret_key, - std::span sender_ed25519_pubkey, +std::optional> decrypt_for_multiple_simple_ed25519( + std::span encoded, + const ed25519::PrivKeySpan& ed25519_secret_key, + std::span sender_ed25519_pubkey, std::string_view domain); } // namespace session diff --git a/include/session/network/backends/backend_util.hpp b/include/session/network/backends/backend_util.hpp index 5a2a9d2b6..c6dcd2dd4 100644 --- a/include/session/network/backends/backend_util.hpp +++ b/include/session/network/backends/backend_util.hpp @@ -5,8 +5,9 @@ #include namespace session::network::backends { -const std::string_view FRAGMENT_PUBKEY = "p"; -const std::string_view FRAGMENT_STREAM_ENCRYPTION = "d"; +constexpr std::string_view FRAGMENT_PUBKEY = "p"; +constexpr std::string_view FRAGMENT_STREAM_ENCRYPTION = "d"; +constexpr std::string_view FRAGMENT_SROUTER = "sr"; struct MatchedEndpoint { std::string_view base; // everything before the pattern match (e.g. "https://example.com") diff --git a/include/session/network/backends/quic_file_client.hpp b/include/session/network/backends/quic_file_client.hpp new file mode 100644 index 000000000..fee844698 --- /dev/null +++ b/include/session/network/backends/quic_file_client.hpp @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "session/network/key_types.hpp" +#include "session/network/session_network_types.hpp" + +namespace oxen::quic { +class Loop; +class Endpoint; +class Connection; +class BTRequestStream; +class Stream; +class GNUTLSCreds; +class Ticker; +class Address; +struct RemoteAddress; +} // namespace oxen::quic + +namespace session::network { + +/// ALPN used by the QUIC file server protocol. +constexpr auto QUIC_FILES_ALPN = "quic-files"; + +/// QUIC stream error code sent when the client aborts a download (e.g. due to a decryption error +/// in the on_data callback). +constexpr uint64_t QUIC_FILES_CLIENT_ABORT = 499; + +/// A self-contained QUIC client that speaks the "quic-files" protocol for streaming file +/// uploads and downloads to a single file server. Manages its own endpoint, connection lifecycle +/// (with idle timeout), and optional 0-RTT session resumption. +/// +/// The caller is responsible for determining the connection address (which may be a direct address +/// or a session-router tunnel proxy port) and the Ed25519 pubkey of the file server. +class QuicFileClient { + friend void streaming_file_upload( + std::shared_ptr, + attachment::Encryptor, + FileUploadRequest, + std::function); + + public: + using ticket_store_cb = std::function ticket_data, + std::chrono::sys_seconds expiry)>; + using ticket_extract_cb = std::function>( + std::string_view remote_key_hex)>; + + /// Construct a QuicFileClient for the given file server. + /// \param loop The event loop to use. + /// \param ed_pubkey Ed25519 pubkey of the file server (for TLS verification). + /// \param address Host/IP to connect to (e.g. "::1" for session-router proxy, or direct IP). + /// \param port Port to connect to. + QuicFileClient( + std::shared_ptr loop, + ed25519_pubkey ed_pubkey, + std::string address, + uint16_t port, + std::optional max_udp_payload = std::nullopt, + ticket_store_cb ticket_store = nullptr, + ticket_extract_cb ticket_extract = nullptr); + + ~QuicFileClient(); + + /// Update the connection target (e.g. when a session-router tunnel port changes). + /// Closes the current connection if the target changed. + void set_target(ed25519_pubkey ed_pubkey, std::string address, uint16_t port); + + /// Upload pre-accumulated encrypted data to the file server. The on_complete callback + /// receives either file_metadata on success or an int16_t error code on failure. + void upload( + std::vector data, + std::optional ttl, + std::function result)> on_complete); + + /// Download a file by ID from the file server. on_data is called as data chunks arrive + /// with a non-owning view of the data; on_complete signals completion or failure. + void download( + std::string file_id, + std::function data)> on_data, + std::function result)> on_complete); + + /// Close the current connection (if any). + void close(); + + private: + std::shared_ptr _loop; + std::shared_ptr _ep; + std::shared_ptr _conn; + std::shared_ptr _creds; + + // Stream 0: opened as BTRequestStream on each connection and held for the connection + // lifetime. TODO: use this for metadata requests (file info, extend TTL, etc.) + std::shared_ptr _bt_stream; + + ed25519_pubkey _ed_pubkey; + std::string _address; + uint16_t _port; + std::optional _max_udp_payload; + + // 0RTT ticket callbacks (optional; if not provided, 0RTT is not used) + ticket_store_cb _ticket_store; + ticket_extract_cb _ticket_extract; + + // Idle timeout: close the connection after this much inactivity + static constexpr auto IDLE_TIMEOUT = std::chrono::seconds{30}; + static constexpr auto IDLE_CHECK_INTERVAL = std::chrono::seconds{5}; + std::shared_ptr _idle_timer; + std::chrono::steady_clock::time_point _last_activity; + + // Returns the active connection, establishing one if needed. + std::shared_ptr _ensure_connection(); + void _start_idle_timer(); + void _touch(); +}; + +/// Performs a complete streaming file upload from a background thread. This function blocks +/// until the upload completes or fails. It: +/// 1. Reads the file to derive the encryption key (Encryptor phase 1) +/// 2. Opens a QUIC stream on the loop thread and sends the PUT command +/// 3. Pulls encrypted chunks from the Encryptor and pushes them to the stream, +/// using watermarks for backpressure +/// 4. Waits for the server response +/// +/// Must be called from a background thread (not the loop thread). The `on_complete` callback +/// fires on the loop thread when done. +/// +/// `get_client` is called on the loop thread to obtain the QuicFileClient to use; this allows +/// the caller to do any router-specific setup (e.g. tunnel establishment) before the upload. +void streaming_file_upload( + std::shared_ptr loop, + attachment::Encryptor enc, + FileUploadRequest request, + std::function get_client); + +} // namespace session::network diff --git a/include/session/network/backends/session_file_server.hpp b/include/session/network/backends/session_file_server.hpp index 73d9c52f3..ba2a6ff47 100644 --- a/include/session/network/backends/session_file_server.hpp +++ b/include/session/network/backends/session_file_server.hpp @@ -1,24 +1,73 @@ #pragma once +#include + #include "session/network/key_types.hpp" +#include "session/network/network_opt.hpp" #include "session/network/session_network_types.hpp" #include "session/platform.hpp" +namespace session::network::file_server { + +/// Default QUIC file server port (first 5 non-zero Fibonacci digits). +constexpr uint16_t QUIC_DEFAULT_PORT = 11235; + +/// Session-router address of a QUIC file server endpoint, as carried in the `sr=` fragment of a +/// download URL, e.g. `sr=abcdef.sesh:11235`. The port is left out of the fragment when it is the +/// default one. +struct SRouterTarget { + std::string address; // e.g. "abcdef...xyz.sesh" or "name.loki" + uint16_t port = QUIC_DEFAULT_PORT; +}; + +} // namespace session::network::file_server + namespace session::network::config { struct FileServer { + // Where the server's HTTP interface lives; these three form the base of the URLs we generate + // and of the requests we proxy to it, e.g. "http" + "filev2.getsession.org" + 80. This is the + // legacy interface: it is what a request falls back to when the QUIC endpoint below cannot be + // reached, and it is what the URLs handed to other clients point at. std::string scheme; std::string host; uint16_t port; + + // The server's Ed25519 pubkey -- the same key its QUIC endpoint is identified by, NOT the + // X25519 form. Onion requests derive the X25519 form from this, so storing the derived value + // here instead makes every request to the server fail: it is not a valid Ed25519 point. + // + // It also doubles as the server's identity for our purposes: a config whose pubkey differs from + // the built-in one is treated as a custom server, which is what puts a `p=` fragment in + // generated URLs and stops the built-in QUIC endpoint from being assumed. std::string pubkey_hex; + // Largest upload we will attempt. The server enforces its own limit; this one only stops us + // making requests we already know it will reject. uint64_t max_file_size; - bool use_stream_encryption; + + // Session-router endpoint of this server, advertised in the `sr=` fragment of the download URLs + // we generate. The built-in servers do not need it: a recipient resolves their QUIC endpoint + // from the network it is on. A custom server has no such mapping, so without this a recipient + // can only reach it over the legacy HTTP path. + std::optional srouter; }; } // namespace session::network::config namespace session::network::file_server { extern const config::FileServer DEFAULT_CONFIG; +extern const config::FileServer TESTNET_CONFIG; + +/// Ed25519 pubkeys of the QUIC file servers. +using namespace oxenc::literals; +constexpr auto QUIC_FS_ED_PUBKEY_MAINNET = + "b8eef9821445ae16e2e97ef8aa6fe782fd11ad5253cd6723b281341dba22e371"_hex_b; +constexpr auto QUIC_FS_ED_PUBKEY_TESTNET = + "929e33ded05e653fec04b49645117f51851f102a947e04806791be416ed76602"_hex_b; + +/// Session-router .sesh addresses of the QUIC file servers (derived from Ed25519 pubkeys). +extern const std::string QUIC_FS_SESH_ADDRESS_MAINNET; +extern const std::string QUIC_FS_SESH_ADDRESS_TESTNET; struct DownloadInfo { std::string scheme; @@ -27,6 +76,7 @@ struct DownloadInfo { std::string file_id; std::optional custom_pubkey_hex; // If 'p' fragment present bool wants_stream_decryption; // If 'd' fragment present + std::optional srouter_target; // If 'sr' fragment present }; /// API: file_server/parse_download_url @@ -40,6 +90,14 @@ struct DownloadInfo { /// - returns struct containing the information required to download the file. std::optional parse_download_url(std::string_view url); +/// Returns a default session-router target for the QUIC file server, if the given HTTP file +/// server config matches a known default. This provides the fallback mapping when a download +/// URL doesn't contain an explicit `sr=` fragment. +/// +/// Returns nullopt if the HTTP file server is not a known QUIC-capable server. +std::optional default_quic_target( + const config::FileServer& http_config, opt::netid::Target netid); + /// API: file_server/generate_download_url /// /// Generates a download url to the configured file server for a given file id. @@ -47,10 +105,16 @@ std::optional parse_download_url(std::string_view url); /// Inputs: /// - `file_id` -- [in] id for the file generated by uploading to the file server. /// - `config` -- [in] file server configuration to use to generate the download url. +/// - `stream_encrypted` -- [in] how the file that was uploaded is encrypted, which the url carries +/// as a `d` fragment so that a recipient decrypts it the right way. This describes the *file*, +/// not the server, so it belongs to whoever did the encrypting: pass true for anything uploaded +/// through `Network::upload_file`, which always encrypts with the stream scheme. False is for a +/// caller that encrypted the bytes itself with the legacy scheme before uploading them. /// /// Outputs: /// - returns url which can be used to download the file. -std::string generate_download_url(std::string_view file_id, const config::FileServer& config); +std::string generate_download_url( + std::string_view file_id, const config::FileServer& config, bool stream_encrypted); /// API: file_server/parse_http_date /// @@ -123,11 +187,33 @@ file_metadata parse_upload_response(const std::string& body, size_t upload_size) /// Outputs: /// - returns a pair of the parsed `file_metadata` and the raw file data. /// - throws `invalid_url_exception` if the URL cannot be parsed. -std::pair> parse_download_response( +std::pair> parse_download_response( std::string_view download_url, const std::vector>& headers, const std::string& body); +/// API: file_server/extend_ttl +/// +/// Constructs a request to extend the TTL of an existing file on the file server. +/// +/// Inputs: +/// - `file_id` -- [in] the file ID whose TTL should be extended. +/// - `ttl` -- [in] the new TTL duration to request. +/// - `config` -- [in] file server configuration to use for the request. +/// - `request_timeout` -- [in] timeout in milliseconds to use for the request. This won't take any +/// pre-flight operations into account so the request will never timeout if pre-flight operations +/// never complete. +/// - `overall_timeout` -- [in] timeout in milliseconds to use for the request and any pre-flight +/// operations that may need to occur (eg. path building). This value takes presedence over +/// `request_timeout` if provided, the request itself will be given a timeout of this value +/// subtracting however long the pre-flight operations took. +Request extend_ttl( + std::string_view file_id, + std::chrono::seconds ttl, + const config::FileServer& config, + std::chrono::milliseconds request_timeout, + std::optional overall_timeout = std::nullopt); + /// API: file_server/get_client_version /// /// Constructs a request to retrieve the version information for the given platform. diff --git a/include/session/network/ip_country.hpp b/include/session/network/ip_country.hpp new file mode 100644 index 000000000..1d99bad29 --- /dev/null +++ b/include/session/network/ip_country.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +namespace session::ip_country { + +using ipv4 = oxen::quic::ipv4; + +/// API: ip_country/available +/// +/// Whether this build of libsession-util carries a bundled IP-to-country database, i.e. whether it +/// was built with the `WITH_IP_GEOLOCATION` cmake option. When it is false the database is empty +/// and every lookup returns nullopt, so a client compiles and runs identically either way and needs +/// no preprocessor test of its own. +/// +/// Outputs: +/// - `bool` -- true if a database is bundled. +bool available(); + +/// API: ip_country/lookup +/// +/// Looks up the country an IPv4 address is assigned to. +/// +/// Inputs: +/// - `ip` -- the address. `ipv4` (i.e. `oxen::quic::ipv4`) constructs from a string ("1.2.3.4"), +/// from an `in_addr`, or from octets, and is what `service_node::ip` already holds. +/// +/// Outputs: +/// - `std::optional` -- the ISO 3166-1 alpha-2 country code, or nullopt if the +/// address is in unassigned or reserved space, or if no database is bundled. The view points at +/// static storage, so it stays valid forever. +std::optional lookup(ipv4 ip); + +/// API: ip_country/attribution +/// +/// The credit that the bundled database's licence (CC BY 4.0) requires be displayed wherever its +/// results are. Show this, rather than composing your own, so that every Session client credits it +/// identically. +/// +/// Outputs: +/// - `std::string_view` -- the attribution line, or empty when no database is bundled (in which +/// case there is nothing to attribute). +std::string_view attribution(); + +/// API: ip_country/database_version +/// +/// The bundled database's release, e.g. "dbip-country-lite-2026-09". The snapshot is refreshed by +/// hand (see `utils/update-ip-country-db.py`), so this is how a client reports which vintage it +/// resolved an address against. +/// +/// Outputs: +/// - `std::string_view` -- the release identifier, or empty when no database is bundled. +std::string_view database_version(); + +} // namespace session::ip_country diff --git a/include/session/network/key_types.hpp b/include/session/network/key_types.hpp index d3d2b1ef9..98861e439 100644 --- a/include/session/network/key_types.hpp +++ b/include/session/network/key_types.hpp @@ -20,7 +20,7 @@ using namespace std::literals; namespace detail { template - inline constexpr std::array null_bytes = {0}; + inline constexpr std::array null_bytes = {}; void load_from_hex(void* buffer, size_t length, std::string_view hex); void load_from_bytes(void* buffer, size_t length, std::string_view bytes); @@ -28,7 +28,7 @@ namespace detail { } // namespace detail template -struct alignas(size_t) key_base : std::array { +struct alignas(size_t) key_base : std::array { std::string_view view() const { return {reinterpret_cast(this->data()), KeyLength}; } @@ -55,10 +55,7 @@ struct alignas(size_t) key_base : std::array { detail::load_from_bytes(d.data(), d.size(), bytes); return d; } - static Derived from_bytes(std::vector bytes) { - return from_bytes(to_string(bytes)); - } - static Derived from_bytes(std::span bytes) { + static Derived from_bytes(std::span bytes) { return from_bytes(to_string(bytes)); } }; @@ -98,7 +95,7 @@ using x25519_keypair = std::pair; legacy_pubkey parse_legacy_pubkey(std::string_view pubkey_in); ed25519_pubkey parse_ed25519_pubkey(std::string_view pubkey_in); x25519_pubkey parse_x25519_pubkey(std::string_view pubkey_in); -x25519_pubkey compute_x25519_pubkey(std::span ed25519_pk); +x25519_pubkey compute_x25519_pubkey(std::span ed25519_pk); } // namespace session::network diff --git a/include/session/network/network_config.hpp b/include/session/network/network_config.hpp index 50eded586..f17452961 100644 --- a/include/session/network/network_config.hpp +++ b/include/session/network/network_config.hpp @@ -6,7 +6,7 @@ #include #include -#include "network_opt.hpp" +#include "session/network/network_opt.hpp" #include "session/types.hpp" namespace session::network::config { @@ -26,7 +26,13 @@ struct Config { std::optional custom_file_server_port = std::nullopt; std::optional custom_file_server_pubkey_hex = std::nullopt; std::optional custom_file_server_max_file_size = std::nullopt; - bool file_server_use_stream_encryption = false; + std::optional custom_file_server_srouter_address = std::nullopt; + std::optional custom_file_server_srouter_port = std::nullopt; + + // QUIC file server options + std::optional quic_file_server_ed_pubkey; + std::optional quic_file_server_address; + std::optional quic_file_server_port; // General options bool increase_no_file_limit = false; @@ -64,7 +70,7 @@ struct Config { // Quic Transport Options std::chrono::milliseconds quic_handshake_timeout{3s}; std::chrono::seconds quic_keep_alive{10s}; - bool quic_disable_mtu_discovery = false; + std::optional quic_max_udp_payload; template requires(sizeof...(Opt) > 0 && (opt::is_option> && ...)) @@ -90,7 +96,7 @@ struct Config { void handle_config_opt(opt::file_server_port fsp); void handle_config_opt(opt::file_server_pubkey_hex fsph); void handle_config_opt(opt::file_server_max_file_size fsmfs); - void handle_config_opt(opt::file_server_use_stream_encryption fsuse); + void handle_config_opt(opt::file_server_srouter fssr); // General options void handle_config_opt(opt::increase_no_file_limit infl); @@ -112,10 +118,15 @@ struct Config { void handle_config_opt(opt::cache_min_num_refresh_presence_to_include_node mnrp); void handle_config_opt(opt::cache_node_strike_threshold nst); + // QUIC file server options + void handle_config_opt(opt::quic_file_server_ed_pubkey qfep); + void handle_config_opt(opt::quic_file_server_address qfa); + void handle_config_opt(opt::quic_file_server_port qfp); + // Quic transport options void handle_config_opt(opt::quic_handshake_timeout qht); void handle_config_opt(opt::quic_keep_alive qka); - void handle_config_opt(opt::quic_disable_mtu_discovery qdmd); + void handle_config_opt(opt::quic_max_udp_payload qmup); // Onion request router options void handle_config_opt(opt::onionreq_path_strike_threshold pst); diff --git a/include/session/network/network_opt.hpp b/include/session/network/network_opt.hpp index 67c52bfc5..c25920818 100644 --- a/include/session/network/network_opt.hpp +++ b/include/session/network/network_opt.hpp @@ -17,8 +17,8 @@ namespace opt { using namespace std::chrono_literals; namespace { - inline std::vector from_hex(std::string_view s) { - std::vector out; + inline std::vector from_hex(std::string_view s) { + std::vector out; out.reserve(s.size() / 2); oxenc::from_hex(s.begin(), s.end(), std::back_inserter(out)); @@ -48,40 +48,40 @@ namespace opt { static netid mainnet() { auto seed_nodes = { service_node{ - ed25519_pubkey::from_hex("1f000f09a7b07828dcb72af7cd16857050c10c02bd58a" - "fb0e38111fb6cda1fef"), + ed25519_pubkey::from_hex("1f000f09a7b07828dcb72af7cd168570" + "50c10c02bd58afb0e38111fb6cda1fef"), oxen::quic::ipv4{"95.216.33.113"}, uint16_t{22100}, uint16_t{20200}, {2, 11, 0}, swarm::INVALID_SWARM_ID}, service_node{ - ed25519_pubkey::from_hex("1f101f0acee4db6f31aaa8b4df134e85ca8a4878efaef" - "7f971e88ab144c1a7ce"), + ed25519_pubkey::from_hex("1f101f0acee4db6f31aaa8b4df134e85" + "ca8a4878efaef7f971e88ab144c1a7ce"), oxen::quic::ipv4{"37.27.236.229"}, uint16_t{22101}, uint16_t{20201}, {2, 11, 0}, swarm::INVALID_SWARM_ID}, service_node{ - ed25519_pubkey::from_hex("1f202f00f4d2d4acc01e20773999a291cf3e3136c3254" - "74d159814e06199919f"), + ed25519_pubkey::from_hex("1f202f00f4d2d4acc01e20773999a291" + "cf3e3136c325474d159814e06199919f"), oxen::quic::ipv4{"172.96.140.124"}, uint16_t{22102}, uint16_t{20202}, {2, 11, 0}, swarm::INVALID_SWARM_ID}, service_node{ - ed25519_pubkey::from_hex("1f303f1d7523c46fa5398826740d13282d26b5de90fba" - "e5749442f66afb6d78b"), + ed25519_pubkey::from_hex("1f303f1d7523c46fa5398826740d1328" + "2d26b5de90fbae5749442f66afb6d78b"), oxen::quic::ipv4{"208.73.207.54"}, uint16_t{22103}, uint16_t{20203}, {2, 11, 0}, swarm::INVALID_SWARM_ID}, service_node{ - ed25519_pubkey::from_hex("1f604f1c858a121a681d8f9b470ef72e6946ee1b9c5ad" - "15a35e16b50c28db7b0"), + ed25519_pubkey::from_hex("1f604f1c858a121a681d8f9b470ef72e" + "6946ee1b9c5ad15a35e16b50c28db7b0"), oxen::quic::ipv4{"104.194.8.115"}, uint16_t{22104}, uint16_t{20204}, @@ -94,17 +94,9 @@ namespace opt { static netid testnet() { auto seed_nodes = { - // service_node{ - // ed25519_pubkey::from_hex("decaf007f26d3d6f9b845ad031ffdf6d04638c25bb10b8fffbbe99135303c4b9"), - // oxen::quic::ipv4{"144.76.164.202"}, - // uint16_t{35500}, - // uint16_t{35400}, - // {2, 10, 0}, - // swarm::INVALID_SWARM_ID}, // This is the original one - service_node{ - ed25519_pubkey::from_hex("decaf20025ca6389d8225bda6a32d7fc4ee5176d21e3b" - "2e9e08c3505a48a811a"), + ed25519_pubkey::from_hex("decaf20025ca6389d8225bda6a32d7fc" + "4ee5176d21e3b2e9e08c3505a48a811a"), oxen::quic::ipv4{"23.88.6.250"}, uint16_t{35520}, uint16_t{35420}, @@ -211,13 +203,16 @@ namespace opt { file_server_max_file_size(uint16_t max_file_size) : max_file_size{max_file_size} {} }; - /// Can be used to override the default (false) flag indicating whether files uploaded to the - /// file server should use XChaCha20-stream based encryption. - struct file_server_use_stream_encryption { - bool use_stream_encryption; + /// Can be used to tell recipients where a custom file server's QUIC endpoint is, by naming it + /// in the download URLs we generate. The built-in servers need no such option: a recipient + /// resolves their endpoint from the network it is on. The port defaults to the standard QUIC + /// file server port, and is left out of the URL when it is that. + struct file_server_srouter { + std::string address; + std::optional port; - file_server_use_stream_encryption(bool use_stream_encryption) : - use_stream_encryption{use_stream_encryption} {} + file_server_srouter(std::string address, std::optional port = std::nullopt) : + address{std::move(address)}, port{port} {} }; /// Can be used to attempt to increase the NOFILE limit (can cause issues with automated tests). @@ -362,6 +357,26 @@ namespace opt { cache_node_strike_threshold(uint16_t count) : count{count} {} }; + // MARK: QUIC File Server Options + + /// Can be used to override the default QUIC file server Ed25519 pubkey (hex). + struct quic_file_server_ed_pubkey { + std::string pubkey_hex; + quic_file_server_ed_pubkey(std::string pubkey_hex) : pubkey_hex{std::move(pubkey_hex)} {} + }; + + /// Can be used to specify the direct address (IP:PORT) of the QUIC file server for direct mode. + struct quic_file_server_address { + std::string address; + quic_file_server_address(std::string address) : address{std::move(address)} {} + }; + + /// Can be used to override the default (11235) QUIC file server port. + struct quic_file_server_port { + uint16_t port; + quic_file_server_port(uint16_t port) : port{port} {} + }; + // MARK: Quic Transport Options /// Can be used to override the default (10s) handshake timeout duration for Quic connections. @@ -376,8 +391,12 @@ namespace opt { quic_keep_alive(std::chrono::seconds duration) : duration{duration} {} }; - /// Can be used to disable Quic MTU discovery. - struct quic_disable_mtu_discovery {}; + /// Caps the maximum QUIC UDP payload size for path MTU discovery. PMTUD will still + /// probe upward from 1200, but will not exceed this value. Must be at least 1200. + struct quic_max_udp_payload { + size_t size; + explicit quic_max_udp_payload(size_t s) : size{s} {} + }; // MARK: Onion Request Router Options @@ -442,7 +461,7 @@ namespace opt { file_server_port, file_server_pubkey_hex, file_server_max_file_size, - file_server_use_stream_encryption, + file_server_srouter, // General options increase_no_file_limit, @@ -464,17 +483,22 @@ namespace opt { cache_min_num_refresh_presence_to_include_node, cache_node_strike_threshold, + // QUIC file server options + quic_file_server_ed_pubkey, + quic_file_server_address, + quic_file_server_port, + // Quic transport options quic_handshake_timeout, quic_keep_alive, - quic_disable_mtu_discovery, + quic_max_udp_payload, // Onion request router options onionreq_path_strike_threshold, + onionreq_path_build_retry_limit, onionreq_min_path_count, onionreq_single_path_mode, onionreq_disable_pre_build_paths, - onionreq_path_build_retry_limit, onionreq_path_rotation_frequency, onionreq_edge_node_cache_duration>; diff --git a/include/session/network/request_queue.hpp b/include/session/network/request_queue.hpp index 48876309b..558006e0e 100644 --- a/include/session/network/request_queue.hpp +++ b/include/session/network/request_queue.hpp @@ -11,23 +11,30 @@ namespace session::network::detail { -class RequestQueue : public std::enable_shared_from_this { +/// Runs on a loop it does not own: the loop's owner outlives every queue on it, which is what lets +/// the jobs below capture `this` rather than a reference to this object. See _jq. +class RequestQueue { private: friend class TestRequestQueue; - std::shared_ptr _loop; + oxen::quic::Loop& _loop; oxen::quic::event_ptr _timeout; std::deque _queue; std::unordered_map> _requests; std::multimap _req_expiries; - RequestQueue(std::shared_ptr loop) : _loop{std::move(loop)} {}; + /// This queue's own jobs, rather than the loop's shared one, so that ~RequestQueue can take + /// them away from the loop: `stop()` waits out whatever is running and cancels the rest, and + /// only then does anything else get torn down. That is what makes capturing `this` in a job + /// safe -- and it has to be a job of *this* queue for that to hold. + /// + /// Declared last so that it is also the first member destroyed, for the case where something + /// destroys this object without the destructor below having run to completion. + oxen::quic::JobQueue _jq{_loop}; public: - static std::shared_ptr make(std::shared_ptr loop) { - return std::shared_ptr{new RequestQueue{std::move(loop)}}; - } + RequestQueue(oxen::quic::Loop& loop) : _loop{loop} {}; virtual ~RequestQueue(); diff --git a/include/session/network/routing/direct_router.hpp b/include/session/network/routing/direct_router.hpp index 0743e4273..5d5efbe4b 100644 --- a/include/session/network/routing/direct_router.hpp +++ b/include/session/network/routing/direct_router.hpp @@ -9,6 +9,7 @@ #include #include +#include "session/network/backends/quic_file_client.hpp" #include "session/network/backends/session_file_server.hpp" #include "session/network/request_queue.hpp" #include "session/network/routing/network_router.hpp" @@ -19,6 +20,13 @@ namespace session::network { namespace config { struct DirectRouter { FileServer file_server_config; + opt::netid::Target netid; + + // When set, DirectRouter uses the QUIC file server protocol for uploads/downloads + // instead of the legacy HTTP path. All three must be set for the QUIC path to activate. + std::optional quic_file_server_address; + std::optional quic_file_server_ed_pubkey; + uint16_t quic_file_server_port = file_server::QUIC_DEFAULT_PORT; }; } // namespace config @@ -28,6 +36,7 @@ class DirectRouter : public IRouter, public std::enable_shared_from_this _loop; std::weak_ptr _transport; + std::unordered_map> _file_clients; std::unordered_map> _active_uploads; std::unordered_map _active_downloads; @@ -45,7 +54,8 @@ class DirectRouter : public IRouter, public std::enable_shared_from_this seed) override; void download(DownloadRequest request) override; private: @@ -54,7 +64,12 @@ class DirectRouter : public IRouter, public std::enable_shared_from_this get_active_paths() { return {}; }; virtual std::vector get_all_used_nodes() { return {}; }; virtual void send_request(Request request, network_response_callback_t callback) = 0; + [[deprecated("use upload_file() instead")]] virtual void upload(UploadRequest request) = 0; + /// Upload a file from disk with streaming encryption. The seed is consumed immediately + /// (before this returns) to initialize the encryption key derivation state. + virtual void upload_file(FileUploadRequest request, std::span seed) = 0; virtual void download(DownloadRequest request) = 0; }; diff --git a/include/session/network/routing/onion_request_router.hpp b/include/session/network/routing/onion_request_router.hpp index 1dbbbcee3..42885d481 100644 --- a/include/session/network/routing/onion_request_router.hpp +++ b/include/session/network/routing/onion_request_router.hpp @@ -100,6 +100,9 @@ inline PathCategory to_path_category(RequestCategory category) { return PathCategory::standard; // Should not be reached } +/// Runs on loops it does not own; its owner outlives it. Its own jobs capture `this` bare -- see +/// _jq -- while callbacks handed to the transport or the snode pool, which outlive this router, +/// keep a weak guard. class OnionRequestRouter : public IRouter, public std::enable_shared_from_this { private: friend class TestOnionRequestRouter; @@ -107,8 +110,9 @@ class OnionRequestRouter : public IRouter, public std::enable_shared_from_this _loop; - std::shared_ptr _disk_loop; + oxen::quic::Loop& _loop; + // Only ever given jobs that capture what they need by value, so it needs no queue of its own. + oxen::quic::Loop& _disk_loop; std::weak_ptr _snode_pool; std::weak_ptr _transport; @@ -135,11 +139,18 @@ class OnionRequestRouter : public IRouter, public std::enable_shared_from_this loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, std::weak_ptr snode_pool, std::weak_ptr transport); ~OnionRequestRouter() override; @@ -153,7 +164,8 @@ class OnionRequestRouter : public IRouter, public std::enable_shared_from_this get_active_paths() override; std::vector get_all_used_nodes() override; void send_request(Request request, network_response_callback_t callback) override; - void upload(UploadRequest request) override; + void upload(UploadRequest request) override; // deprecated: use upload_file() + void upload_file(FileUploadRequest request, std::span seed) override; void download(DownloadRequest request) override; private: @@ -172,6 +184,16 @@ class OnionRequestRouter : public IRouter, public std::enable_shared_from_this is_cancelled, + std::function, bool)> on_result); void _download_internal(DownloadRequest request); void _build_path( diff --git a/include/session/network/routing/session_router_router.hpp b/include/session/network/routing/session_router_router.hpp index 64e5c4e23..0f02c068c 100644 --- a/include/session/network/routing/session_router_router.hpp +++ b/include/session/network/routing/session_router_router.hpp @@ -9,6 +9,7 @@ #include #include +#include "session/network/backends/quic_file_client.hpp" #include "session/network/backends/session_file_server.hpp" #include "session/network/request_queue.hpp" #include "session/network/routing/network_router.hpp" @@ -31,22 +32,42 @@ namespace config { }; } // namespace config +// A tunnel we are holding open, and whether it is usable yet. Defined in the .cpp: naming +// session-router's claim type here would drag into every consumer of this +// header, and what we hold a tunnel with is nobody else's business. +struct ActiveTunnel; + class SessionRouter : public IRouter, public std::enable_shared_from_this { private: bool _ready = false; bool _suspended = false; config::SessionRouter _config; + // The one component that keeps a share of the loop rather than a reference: the session-router + // library takes a `shared_ptr` (router.hpp:137), so ownership has to be shared with it. + // What keeps that from destroying the loop from its own thread is the same invariant as + // everywhere else here -- this object is destroyed by whoever owns it, off the loop. std::shared_ptr _loop; std::shared_ptr srouter; std::weak_ptr _snode_pool; std::weak_ptr _transport; - std::unordered_map _active_tunnels; + // Pool of QUIC file server clients, keyed by Ed25519 pubkey. Multiple requests to the + // same server share one client (and thus one connection with idle timeout). + std::unordered_map> _file_clients; + std::unordered_map> _active_tunnels; std::unordered_map>> _pending_requests; + std::vector> _pending_operations; std::unordered_map> _active_uploads; std::unordered_map _active_downloads; + /// This router's own jobs, rather than the loop's shared queue, so that ~SessionRouter can take + /// them away from the loop before anything is torn down: `stop()` waits out whatever is running + /// and cancels the rest. That is what lets the jobs capture `this` bare. + /// + /// Declared last so that it is also the first member destroyed. + oxen::quic::JobQueue _jq{*_loop}; + public: static std::shared_ptr make( config::SessionRouter config, @@ -63,7 +84,8 @@ class SessionRouter : public IRouter, public std::enable_shared_from_this get_active_paths() override; void send_request(Request request, network_response_callback_t callback) override; - void upload(UploadRequest request) override; + void upload(UploadRequest request) override; // deprecated: use upload_file() + void upload_file(FileUploadRequest request, std::span seed) override; void download(DownloadRequest request) override; private: @@ -84,15 +106,50 @@ class SessionRouter : public IRouter, public std::enable_shared_from_this enc, + FileUploadRequest request, + file_server::SRouterTarget target); + void _upload_internal_legacy(UploadRequest request, std::string upload_id); void _download_internal(DownloadRequest request); + void _download_internal_legacy(DownloadRequest request, std::string download_id); + void _cleanup_upload(const std::string& upload_id); + QuicFileClient& _get_file_client( + const ed25519_pubkey& pubkey, + std::string_view address, + uint16_t port, + std::optional max_udp_payload = std::nullopt); + + void _quic_upload_via_tunnel( + UploadRequest upload_request, + std::string upload_id, + std::vector data, + session::router::tunnel_info info); + void _quic_download_via_tunnel( + DownloadRequest request, + std::string download_id, + std::string file_id, + session::router::tunnel_info info); void _establish_tunnel( - std::span& remote_pubkey, + std::span remote_pubkey, const uint16_t remote_port, const std::string& initiating_req_id); + // Takes the tunnel's endpoint by value rather than a reference into `_active_tunnels`: a + // response callback that runs inline can fail the tunnel out from under us. void _send_via_tunnel( - session::router::tunnel_info tunnel, + std::string tunnel_remote, + uint16_t tunnel_local_port, Request request, network_response_callback_t callback); + + // The tunnel entry for a node, created empty if we have none yet. + ActiveTunnel& _tunnel(const std::string& pubkey_hex); + + // Fails every request waiting on a tunnel to `pubkey_hex`, and tells the SnodePool so that the + // node stops being selected. `unreachable` distinguishes a node Session Router has no relay + // contact for -- which will stay unreachable until it rejoins the network -- from one that + // merely failed to establish this time. + void _fail_tunnel(const std::string& pubkey_hex, bool unreachable); }; } // namespace session::network diff --git a/include/session/network/service_node.hpp b/include/session/network/service_node.hpp index a2306f083..aa96b6606 100644 --- a/include/session/network/service_node.hpp +++ b/include/session/network/service_node.hpp @@ -44,14 +44,14 @@ struct service_node { uint64_t requested_unlock_height; oxen::quic::RemoteAddress to_https_address() const { - return oxen::quic::RemoteAddress{remote_pubkey, ip, https_port}; + return oxen::quic::RemoteAddress{remote_pubkey.view(), ip, https_port}; } - oxen::quic::RemoteAddress to_omq_address() const { - return oxen::quic::RemoteAddress{remote_pubkey, ip, omq_port}; + oxen::quic::RemoteAddress to_quic_address() const { + return oxen::quic::RemoteAddress{remote_pubkey.view(), ip, omq_port}; } - std::span view_remote_key() const { return remote_pubkey; } + std::span view_remote_key() const { return remote_pubkey; } std::string host() const { return ip.to_string(); } std::string to_string() const; diff --git a/include/session/network/session_network.h b/include/session/network/session_network.h index 43451f18d..d3a63568c 100644 --- a/include/session/network/session_network.h +++ b/include/session/network/session_network.h @@ -53,7 +53,6 @@ typedef struct session_network_config { uint16_t custom_file_server_port; const char* custom_file_server_pubkey_hex; uint64_t custom_file_server_max_file_size; - bool file_server_use_stream_encryption; // General options bool increase_no_file_limit; @@ -94,7 +93,10 @@ typedef struct session_network_config { // Quic transport options (for transport == SESSION_NETWORK_TRANSPORT_QUIC) uint32_t quic_handshake_timeout_seconds; uint32_t quic_keep_alive_seconds; - bool quic_disable_mtu_discovery; + bool quic_disable_mtu_discovery; // deprecated: use quic_max_udp_payload instead + /// Maximum QUIC UDP payload size for PMTUD; 0 for default (no cap). + /// If quic_disable_mtu_discovery is true and this is 0, acts as if set to 1200. + size_t quic_max_udp_payload; } session_network_config; @@ -282,7 +284,6 @@ LIBSESSION_EXPORT session_upload_handle_t* session_network_upload( /// - `stall_timeout_ms` -- [in] timeout if no progress for this duration /// - `request_timeout_ms` -- [in] timeout for the request itself /// - `overall_timeout_ms` -- [in] timeout including pre-flight operations (0 to ignore) -/// - `partial_min_interval_ms` -- [in] minimum interval between on_data calls (default 250ms) /// /// Returns: handle to the download, or NULL on error. Caller must free with session_download_free() LIBSESSION_EXPORT session_download_handle_t* session_network_download( @@ -292,7 +293,6 @@ LIBSESSION_EXPORT session_download_handle_t* session_network_download( int64_t stall_timeout_ms, int64_t request_timeout_ms, int64_t overall_timeout_ms, - int64_t partial_min_interval_ms, int8_t desired_path_index); /// Cancels an in-progress upload diff --git a/include/session/network/session_network.hpp b/include/session/network/session_network.hpp index 9c8f07eb9..33ee4476a 100644 --- a/include/session/network/session_network.hpp +++ b/include/session/network/session_network.hpp @@ -5,6 +5,7 @@ #include #include +#include "session/clock.hpp" #include "session/network/backends/session_file_server.hpp" #include "session/network/network_config.hpp" #include "session/network/routing/network_router.hpp" @@ -13,6 +14,10 @@ #include "session/platform.hpp" #include "session/types.hpp" +namespace session { +class TestHelper; +} + namespace session::network { namespace detail { @@ -21,7 +26,13 @@ namespace detail { namespace fs = std::filesystem; // NOLINT(misc-unused-alias-decls) -class Network : public std::enable_shared_from_this { +/// Owns the loops everything below it runs on, and is itself singly owned: a Network is held by one +/// `unique_ptr` and nothing else, so no callback can keep it alive and its destructor always runs +/// on whichever thread its owner drops it from. That is what lets it join the loop threads at the +/// end of ~Network -- joining them from a callback of their own would abort. +class Network { + friend class session::TestHelper; // for unit tests: see _set_router + private: const config::Config config; std::shared_ptr _loop; // Main loop for network events and syncronization @@ -40,6 +51,16 @@ class Network : public std::enable_shared_from_this { std::function, bool)>>>> _clock_resync_download_queue; + /// Our own jobs, rather than the loop's shared queue, so that ~Network can take them away from + /// the loop: `stop()` waits out whatever is running and cancels the rest. Together with the + /// components below being destroyed before it is stopped, that is what lets every job and + /// callback here capture `this` bare. + /// + /// Declared last so that it is also the first member destroyed. Held in an optional because + /// the loop it runs on is created in the constructor body -- after the file-descriptor limit + /// has been raised, which has to come first -- rather than in the initialiser list. + std::optional _jq; + public: const config::FileServer file_server_config; @@ -52,14 +73,16 @@ class Network : public std::enable_shared_from_this { requires(!std::is_same_v< std::decay_t>>, config::Config>) - Network(Opt&&... opts) : Network(Config(std::forward(opts)...)){}; + Network(Opt&&... opts) : Network{config::Config{std::forward(opts)...}} {}; explicit Network(config::Config config); virtual ~Network(); bool has_retrieved_time_offset() const { return (_last_successful_clock_resync == std::chrono::steady_clock::time_point{}); }; - std::chrono::milliseconds network_time_offset() const { return _network_time_offset; }; + std::chrono::milliseconds network_time_offset() const { + return std::chrono::duration_cast(AdjustedClock::get_offset()); + }; fork_versions fork() const { return _fork_versions.load(); }; uint16_t hardfork() const { return _fork_versions.load().hardfork; }; uint16_t softfork() const { return _fork_versions.load().softfork; }; @@ -83,8 +106,10 @@ class Network : public std::enable_shared_from_this { /// - 'ignore_strike_count' - [in] flag indicating whether node strikes should be ignored when /// retrieving the swarm. /// - 'callback' - [in] callback to be called with the retrieved swarm (in the case of an error - /// the callback will be called with an empty list). - void get_swarm( + /// the callback will be called with an empty list). The order of items in the swarm vector + /// will be shuffled (but may prioritize some nodes over others depend on observed past + /// behaviour; see SnodePool::get_swarm). + virtual void get_swarm( session::network::x25519_pubkey swarm_pubkey, bool ignore_strike_count, std::function swarm)> callback); @@ -101,13 +126,14 @@ class Network : public std::enable_shared_from_this { void get_random_nodes( uint16_t count, std::function nodes)> callback); - void send_request(Request request, network_response_callback_t callback); + virtual void send_request(Request request, network_response_callback_t callback); + [[deprecated("use upload_file() instead")]] void upload(UploadRequest request); - void download(DownloadRequest request); + virtual void upload_file(FileUploadRequest request, std::span seed); + virtual void download(DownloadRequest request); private: std::atomic _status{ConnectionStatus::unknown}; - std::atomic _network_time_offset{0ms}; std::atomic _fork_versions{{0, 0}}; void configure(); @@ -118,6 +144,19 @@ class Network : public std::enable_shared_from_this { void _update_network_state(const std::string& body); void _handle_421_retry(Request original_request, network_response_callback_t final_callback); + // Re-sends a request to the next member of the same swarm, after the one it was sent to could + // not be reached. Distinct from the 421 path: there the swarm information was wrong and is + // thrown away, here it is right and only one member of it is unusable. Gives up when + // selection has no member left that has not already failed, reporting the original failure + // rather than one of its own invention. + void _retry_next_swarm_node( + Request original_request, + bool timeout, + int16_t status_code, + std::vector> headers, + std::optional body, + network_response_callback_t final_callback); + void _resync_clock( std::optional original_request, network_response_callback_t request_callback); void _launch_next_clock_out_of_sync_request( diff --git a/include/session/network/session_network_types.hpp b/include/session/network/session_network_types.hpp index 266cbf12b..620ea33d6 100644 --- a/include/session/network/session_network_types.hpp +++ b/include/session/network/session_network_types.hpp @@ -1,13 +1,17 @@ #pragma once +#include +#include #include #include #include #include +#include "session/attachments.hpp" #include "session/network/key_types.hpp" #include "session/network/service_node.hpp" #include "session/network/session_network_types.h" +#include "session/sodium_array.hpp" namespace session::network { @@ -30,6 +34,7 @@ constexpr int16_t ERROR_FAILED_GENERATE_ONION_PAYLOAD = -10010; constexpr int16_t ERROR_FAILED_TO_GET_STREAM = -10011; constexpr int16_t ERROR_BUILD_TIMEOUT = -10100; constexpr int16_t ERROR_REQUEST_CANCELLED = -10200; +constexpr int16_t ERROR_FILE_SERVER_UNAVAILABLE = -10300; constexpr int16_t ERROR_UNKNOWN = -11000; const std::pair content_type_plain_text = { @@ -146,7 +151,7 @@ struct Request { std::string request_id; network_destination destination; std::string endpoint; - std::optional> body; + std::optional> body; RequestCategory category; /// Timeout for an in-flight request after it has been sent via the transport mechanism. @@ -180,12 +185,28 @@ struct Request { /// `overall_timeout` has been exceeded. std::chrono::steady_clock::time_point creation_time = std::chrono::steady_clock::now(); - int retry_count = 0; + /// How many times this request has been redirected after a 421, bounded by + /// `config.redirect_retry_count`. Counts redirects only -- a 421 means our swarm information + /// was wrong, so recovery is to re-resolve the swarm from scratch. It has nothing to do with + /// `failed_nodes` below, which is the opposite situation. + int retry_421_count = 0; + + /// Swarm members that could not be reached for this request, in the order they were tried. + /// + /// A node that cannot be reached says nothing about the swarm -- unlike a 421, which says the + /// swarm itself is wrong -- so recovery is to keep the swarm and move to the next-best member, + /// excluding these. Running out of members is what ends it, so this is a set rather than a + /// count: "once per node" cannot be expressed as a number, since choosing the next one has to + /// know which have already been spent. + /// + /// Empty for anything not addressed to a swarm; a request with no `swarm_pubkey` has no other + /// member to move to. + std::vector failed_nodes; Request(std::string request_id, network_destination destination, std::string endpoint, - std::optional> body, + std::optional> body, RequestCategory category, std::chrono::milliseconds request_timeout, std::optional overall_timeout = std::nullopt, @@ -194,7 +215,7 @@ struct Request { Request(network_destination destination, std::string endpoint, - std::optional> body, + std::optional> body, RequestCategory category, std::chrono::milliseconds request_timeout, std::optional overall_timeout = std::nullopt, @@ -223,9 +244,10 @@ struct file_metadata { }; struct FileTransferRequest { - std::chrono::milliseconds stall_timeout; + std::chrono::milliseconds stall_timeout = 25s; std::chrono::milliseconds request_timeout; std::optional overall_timeout; + std::chrono::milliseconds progress_interval = 1s; std::optional desired_path_index; // This shared ptr is designed to be held by the caller (without the rest of the request object) @@ -240,22 +262,44 @@ struct FileTransferRequest { // Called when transfer completes (file_metadata) or fails (int16_t error code) std::function result, bool timeout)> on_complete; + + // Called periodically during a transfer with progress information, at most once per + // progress_interval, and only when progress has been made since the last call. + // For uploads, progress_bytes is total bytes acked by the remote; for downloads, it is + // total bytes received. + std::function on_progress; }; struct UploadRequest : FileTransferRequest { - std::function()> next_data; + std::function()> next_data; std::optional file_name; std::optional ttl; }; +struct FileUploadRequest : FileTransferRequest { + std::filesystem::path file; + attachment::Domain domain = attachment::Domain::ATTACHMENT; + bool allow_large = false; + std::optional ttl; + + // Hides FileTransferRequest::on_complete: this version includes the decryption key + // alongside the file metadata on success. + std::function, int16_t> result, bool timeout)> + on_complete; +}; + struct DownloadRequest : FileTransferRequest { std::string download_url; - // Called as data arrives (can be called multiple times) - std::function data)> on_data; - - // Minimum interval between on_data calls (to control callback overhead vs memory usage) - std::chrono::milliseconds partial_min_interval = 250ms; + // Called as data arrives, once per chunk received, with a non-owning view of that chunk and the + // file's metadata. `info.size` is the total, and is known before the first chunk arrives, so a + // caller wanting transfer progress accumulates the chunk sizes itself rather than being told. + // + // Any coalescing of these belongs above this layer: throttling here would mean holding payload + // to save an in-process call, whereas a consumer relaying progress across a process boundary + // can drop redundant notifications for free. + std::function data)> on_data; }; using node_failure_reporter_t = std::function; diff --git a/include/session/network/snode_pool.hpp b/include/session/network/snode_pool.hpp index 787e2c582..2352f78d1 100644 --- a/include/session/network/snode_pool.hpp +++ b/include/session/network/snode_pool.hpp @@ -12,6 +12,10 @@ #include "session/network/service_node.hpp" #include "swarm.hpp" +namespace session { +class TestHelper; +} + namespace session::network { namespace config { @@ -40,16 +44,18 @@ class empty_file_exception : public std::runtime_error { }; class SnodePool : public std::enable_shared_from_this { + friend class session::TestHelper; // for unit tests + public: using network_fetcher_t = std::function; using fetcher_connectivity_check_t = std::function; SnodePool( config::SnodePool config, - std::shared_ptr loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, network_fetcher_t direct_fetcher); - ~SnodePool() = default; + virtual ~SnodePool(); void suspend(); void resume(); @@ -89,8 +95,9 @@ class SnodePool : public std::enable_shared_from_this { bool _suspended = false; config::SnodePool _config; - std::shared_ptr _loop; - std::shared_ptr _disk_loop; + oxen::quic::Loop& _loop; + // Only ever given jobs that capture what they need by value, so it needs no queue of its own. + oxen::quic::Loop& _disk_loop; network_fetcher_t _direct_fetcher; std::optional _routed_fetcher; std::optional _routed_fetcher_connectivity_check; @@ -115,6 +122,14 @@ class SnodePool : public std::enable_shared_from_this { std::vector> _snode_refresh_results; std::vector> _after_snode_cache_refresh; + /// This pool's own jobs, rather than the loop's shared queue, so that ~SnodePool can take them + /// away from the loop before anything is torn down: `stop()` waits out whatever is running and + /// cancels the rest, including anything scheduled with call_later. That is what lets the jobs + /// capture `this` bare. + /// + /// Declared last so that it is also the first member destroyed. + oxen::quic::JobQueue _jq{_loop}; + // Disk I/O functions void _load_from_disk(); static void _clear_disk_cache(const std::filesystem::path& path); diff --git a/include/session/network/transport/network_transport.hpp b/include/session/network/transport/network_transport.hpp index fd914a39e..124262677 100644 --- a/include/session/network/transport/network_transport.hpp +++ b/include/session/network/transport/network_transport.hpp @@ -15,7 +15,7 @@ class ITransport { virtual void close_connections() = 0; virtual ConnectionStatus get_status() const = 0; - virtual void set_node_failure_reporter(node_failure_reporter_t /*reporter*/) {} + virtual void set_node_failure_reporter(node_failure_reporter_t) {} virtual void verify_connectivity( service_node node, std::chrono::milliseconds timeout, @@ -29,4 +29,4 @@ class ITransport { virtual void send_request(Request request, network_response_callback_t callback) = 0; }; -} // namespace session::network \ No newline at end of file +} // namespace session::network diff --git a/include/session/network/transport/quic_transport.hpp b/include/session/network/transport/quic_transport.hpp index 12f220b71..bbfc7cea4 100644 --- a/include/session/network/transport/quic_transport.hpp +++ b/include/session/network/transport/quic_transport.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include "session/network/transport/network_transport.hpp" namespace oxen::quic { -class Loop; class Endpoint; struct ConnectionID; } // namespace oxen::quic @@ -24,15 +24,17 @@ namespace config { std::chrono::milliseconds handshake_timeout; std::chrono::seconds keep_alive; - bool disable_mtu_discovery; + std::optional max_udp_payload; }; } // namespace config -class QuicTransport : public ITransport, public std::enable_shared_from_this { +/// Runs on a loop it does not own. Its jobs, and the callbacks it hands to libquic, capture `this` +/// bare: see _jq and ~QuicTransport for what makes that safe. +class QuicTransport : public ITransport { private: bool _suspended = false; config::QuicTransport _config; - std::shared_ptr _loop; + oxen::quic::Loop& _loop; std::shared_ptr _endpoint; std::unordered_map _active_connection_ids; @@ -44,7 +46,7 @@ class QuicTransport : public ITransport, public std::enable_shared_from_this>> _failure_listeners; public: - explicit QuicTransport(config::QuicTransport config, std::shared_ptr loop); + explicit QuicTransport(config::QuicTransport config, oxen::quic::Loop& loop); ~QuicTransport() override; void suspend() override; @@ -76,6 +78,13 @@ class QuicTransport : public ITransport, public std::enable_shared_from_this custom_error); }; -} // namespace session::network \ No newline at end of file +} // namespace session::network diff --git a/include/session/onionreq/builder.hpp b/include/session/onionreq/builder.hpp index 99890cbe6..3ebda64f8 100644 --- a/include/session/onionreq/builder.hpp +++ b/include/session/onionreq/builder.hpp @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -64,14 +66,14 @@ class Builder { }; void set_destination(network::network_destination destination); - void add_hop(std::span remote_key); + void add_hop(std::span remote_key); void add_hop(std::pair keys) { hops_.push_back(keys); } - std::vector build(std::vector payload); - std::vector generate_onion_blob( - const std::optional>& plaintext_body); + std::vector build(std::vector payload); + std::vector generate_onion_blob( + const std::optional>& plaintext_body); private: std::vector> hops_ = {}; @@ -91,8 +93,7 @@ class Builder { std::optional>> headers_ = std::nullopt; std::optional>> query_params_ = std::nullopt; - std::vector _generate_payload( - std::optional> body) const; + std::vector _generate_payload(std::optional> body) const; }; } // namespace session::onionreq diff --git a/include/session/onionreq/hop_encryption.hpp b/include/session/onionreq/hop_encryption.hpp index 47bb1f28b..45b92a53d 100644 --- a/include/session/onionreq/hop_encryption.hpp +++ b/include/session/onionreq/hop_encryption.hpp @@ -25,20 +25,20 @@ class HopEncryption { // Encrypts `plaintext` message using encryption `type`. `pubkey` is the recipients public key. // `reply` should be false for a client-to-snode message, and true on a returning // snode-to-client message. - std::vector encrypt( + std::vector encrypt( EncryptType type, - std::vector plaintext, + std::vector plaintext, const network::x25519_pubkey& pubkey) const; - std::vector decrypt( + std::vector decrypt( EncryptType type, - std::vector ciphertext, + std::vector ciphertext, const network::x25519_pubkey& pubkey) const; // AES-GCM encryption. - std::vector encrypt_aesgcm( - std::vector plainText, const network::x25519_pubkey& pubKey) const; - std::vector decrypt_aesgcm( - std::vector cipherText, const network::x25519_pubkey& pubKey) const; + std::vector encrypt_aesgcm( + std::vector plainText, const network::x25519_pubkey& pubKey) const; + std::vector decrypt_aesgcm( + std::span cipherText, const network::x25519_pubkey& pubKey) const; // xchacha20-poly1305 encryption; for a message sent from client Alice to server Bob we use a // shared key of a Blake2B 32-byte (i.e. crypto_aead_xchacha20poly1305_ietf_KEYBYTES) hash of @@ -48,10 +48,10 @@ class HopEncryption { // When Bob (the server) encrypts a method for Alice (the client), he uses shared key // H(bA || A || B) (note that this is *different* that what would result if Bob was a client // sending to Alice the client). - std::vector encrypt_xchacha20( - std::vector plaintext, const network::x25519_pubkey& pubKey) const; - std::vector decrypt_xchacha20( - std::vector ciphertext, const network::x25519_pubkey& pubKey) const; + std::vector encrypt_xchacha20( + std::vector plaintext, const network::x25519_pubkey& pubKey) const; + std::vector decrypt_xchacha20( + std::span ciphertext, const network::x25519_pubkey& pubKey) const; private: const network::x25519_seckey private_key_; diff --git a/include/session/onionreq/parser.hpp b/include/session/onionreq/parser.hpp index 91857904a..bffd8b94f 100644 --- a/include/session/onionreq/parser.hpp +++ b/include/session/onionreq/parser.hpp @@ -15,33 +15,33 @@ class OnionReqParser { HopEncryption enc; EncryptType enc_type = EncryptType::aes_gcm; network::x25519_pubkey remote_pk; - std::vector payload_; + std::vector payload_; public: /// Constructs a parser, parsing the given request sent to us. Throws if parsing or decryption /// fails. OnionReqParser( - std::span x25519_pubkey, - std::span x25519_privkey, - std::span req, + std::span x25519_pubkey, + std::span x25519_privkey, + std::span req, size_t max_size = DEFAULT_MAX_SIZE); /// plaintext payload, decrypted from the incoming request during construction. - std::span payload() const { return to_span(payload_); } + std::span payload() const { return payload_; } /// Extracts payload from this object (via a std::move); after the call the object's payload /// will be empty. - std::vector move_payload() { - std::vector ret{std::move(payload_)}; + std::vector move_payload() { + std::vector ret{std::move(payload_)}; payload_.clear(); // Guarantee empty, even if SSO active return ret; } - std::span remote_pubkey() const { return to_span(remote_pk.view()); } + std::span remote_pubkey() const { return remote_pk; } /// Encrypts a reply using the appropriate encryption as determined when parsing the /// request. - std::vector encrypt_reply(std::span reply) const; + std::vector encrypt_reply(std::span reply) const; }; } // namespace session::onionreq diff --git a/include/session/onionreq/response_parser.hpp b/include/session/onionreq/response_parser.hpp index 6a1d7c0a4..6638dbcea 100644 --- a/include/session/onionreq/response_parser.hpp +++ b/include/session/onionreq/response_parser.hpp @@ -34,7 +34,7 @@ class ResponseParser { static bool response_long_enough(EncryptType enc_type, size_t response_size); - std::vector decrypt(std::vector ciphertext) const; + std::vector decrypt(std::vector ciphertext) const; DecryptedResponse decrypted_response(const std::string& encrypted_response); private: diff --git a/include/session/pro_backend.h b/include/session/pro_backend.h index bc0f2e70e..8959431bc 100644 --- a/include/session/pro_backend.h +++ b/include/session/pro_backend.h @@ -298,9 +298,9 @@ void session_pro_backend_get_payment_details_response_free( /// - `ts` -- Unix timestamp (seconds) for the request. LIBSESSION_EXPORT session_pro_backend_request session_pro_backend_generate_pro_proof_request_build( - const uint8_t* master_privkey, + const unsigned char* master_privkey, size_t master_privkey_len, - const uint8_t* rotating_privkey, + const unsigned char* rotating_privkey, size_t rotating_privkey_len, int64_t ts) NON_NULL_ARG(1, 3); @@ -326,7 +326,7 @@ session_pro_backend_request session_pro_backend_get_pro_revocations_request_buil /// - `ts` -- Unix timestamp (seconds) for the request. LIBSESSION_EXPORT session_pro_backend_request session_pro_backend_get_pro_status_request_build( - const uint8_t* master_privkey, size_t master_privkey_len, int64_t ts) NON_NULL_ARG(1); + const unsigned char* master_privkey, size_t master_privkey_len, int64_t ts) NON_NULL_ARG(1); /// API: session_pro_backend/get_payment_details_request_build /// @@ -344,7 +344,7 @@ session_pro_backend_request session_pro_backend_get_pro_status_request_build( /// empty string for the newest page. Pass through verbatim; do not parse or synthesize it. LIBSESSION_EXPORT session_pro_backend_request session_pro_backend_get_payment_details_request_build( - const uint8_t* master_privkey, + const unsigned char* master_privkey, size_t master_privkey_len, int64_t ts, uint32_t limit, diff --git a/include/session/pro_backend.hpp b/include/session/pro_backend.hpp index c0236d852..38cc89e21 100644 --- a/include/session/pro_backend.hpp +++ b/include/session/pro_backend.hpp @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -61,12 +63,10 @@ namespace session::pro_backend { -using namespace oxenc::literals; - /// The Session Pro Backend's Ed25519 public key: verify that a proof was issued by the backend by /// checking its signature against this key (see ProProof::verify_signature). This is the current /// backend signing key (test deployment, expected to carry through to production). -constexpr auto PUBKEY = "479ffca8bcec7b4a0f0f7afe48b8a6d15635a8c7ff15ad16add05752c19414d4"_hex_u; +constexpr auto PUBKEY = "479ffca8bcec7b4a0f0f7afe48b8a6d15635a8c7ff15ad16add05752c19414d4"_hex_b; static_assert(PUBKEY.size() == 32); /// The X25519 form of `PUBKEY` (the same key converted via crypto_sign_ed25519_pk_to_curve25519), @@ -74,7 +74,7 @@ static_assert(PUBKEY.size() == 32); /// channel to the backend) without doing the conversion themselves. A unit test asserts these bytes /// match the runtime conversion of `PUBKEY`, so the two cannot drift. constexpr auto PUBKEY_X25519 = - "ce5a75f64b6c43db6c1374d362c3ea9d85951c4f42a3d04cf94f87822d4f803b"_hex_u; + "ce5a75f64b6c43db6c1374d362c3ea9d85951c4f42a3d04cf94f87822d4f803b"_hex_b; static_assert(PUBKEY_X25519.size() == 32); /// The Session Pro Backend's production base URL: POST a request body to `/` (see @@ -92,6 +92,10 @@ constexpr std::string_view PAYMENT_PROVIDER_APP_STORE = "app_store"; // STF = the Session Technology Foundation's out-of-band grant (not a purchasable store). constexpr std::string_view PAYMENT_PROVIDER_STF = "stf"; +/// Domain used with ed25519::derive_subkey to derive the Session Pro signing keypair from the +/// account's root Ed25519 seed. +constexpr auto pro_subkey_domain = "SessionProRandom"_bytes; + /// Response outcome category (the wire `status`, spec §5). CLOSED/exhaustive by design: the backend /// will never add a fourth value, so libsession treats any unrecognized wire status as a protocol /// error (fail-closed) rather than passing it through. New categories/detail arrive via @@ -104,7 +108,7 @@ enum class ResponseStatus { }; struct ResponseBase { - /// Outcome category; `success()` is the usual check. See ResponseStatus. + /// Outcome category; the `explicit operator bool()` is the usual check. See ResponseStatus. ResponseStatus status = ResponseStatus::Ok; /// On non-Ok, a stable machine-readable slug identifying the outcome (spec §5.1), e.g. @@ -124,11 +128,6 @@ struct ResponseBase { explicit operator bool() const { return status == ResponseStatus::Ok; } }; -struct MasterRotatingSignatures { - array_uc64 master_sig; - array_uc64 rotating_sig; -}; - /// Per-provider support/management URLs (from provider_urls()). These are identical for every user /// (not translation data), so libsession owns them as the single source of truth rather than each /// client duplicating them; the human-readable provider/store *names* are translation data and @@ -234,9 +233,9 @@ GenerateProProofResponse parse_pro_proof(std::string_view json); /// - `master_privkey` / `rotating_privkey` -- 32-byte Ed25519 seed or 64-byte libsodium private key /// - `unix_ts` -- Unix timestamp for the request ProRequest pro_proof_request( - std::span master_privkey, - std::span rotating_privkey, - sys_seconds unix_ts); + const ed25519::PrivKeySpan& master_privkey, + const ed25519::PrivKeySpan& rotating_privkey, + std::chrono::sys_seconds unix_ts); /// Build a request for the current Session Pro revocation list (endpoint `get_pro_revocations`). /// This request is unsigned. The caller retains each returned item for the response's `retain_for` @@ -251,10 +250,10 @@ ProRequest revocations_request(std::int64_t ticket); struct ProRevocationItem { /// 32-byte opaque revocation tag identifying a proof - array_uc32 revocation_tag; + b32 revocation_tag; /// A matching proof is revoked once the client's clock reaches this unix timestamp (not before) - sys_seconds effective_at; + std::chrono::sys_seconds effective_at; }; struct GetProRevocationsResponse : ResponseBase { @@ -286,7 +285,8 @@ GetProRevocationsResponse parse_revocations(std::string_view json); /// Inputs: /// - `master_privkey` -- 32-byte Ed25519 seed or 64-byte libsodium master private key /// - `unix_ts` -- Unix timestamp for the request -ProRequest pro_status_request(std::span master_privkey, sys_seconds unix_ts); +ProRequest pro_status_request( + const ed25519::PrivKeySpan& master_privkey, std::chrono::sys_seconds unix_ts); /// Query a master key's Session Pro payment history (endpoint `get_payment_details`), one keyset /// page at a time. Builds the whole request, signing internally with the master key, and returns @@ -300,8 +300,8 @@ ProRequest pro_status_request(std::span master_privkey, sys_secon /// PaymentDetailsResponse); the empty string requests the newest page. Pass it through verbatim; /// it must not be parsed or synthesized. ProRequest payment_details_request( - std::span master_privkey, - sys_seconds unix_ts, + const ed25519::PrivKeySpan& master_privkey, + std::chrono::sys_seconds unix_ts, uint32_t limit, std::string_view before); @@ -352,7 +352,7 @@ struct ProPaymentItem { sys_ms purchased_at; /// Unix timestamp of when the payment was expiry. 0 if not activated - sys_seconds expiry_at; + std::chrono::sys_seconds expiry_at; /// The dunning window this ONE payment's provider declared -- raw store data, and NOT the same /// quantity as `ProStatusResponse::grace_period_duration`, which is account-level and adds the @@ -365,7 +365,7 @@ struct ProPaymentItem { /// Unix deadline timestamp of when the user is able to refund the subscription via the payment /// provider. Thereafter the user must initiate a refund manually via Session support. - sys_seconds platform_refund_expiry_at; + std::chrono::sys_seconds platform_refund_expiry_at; /// Provider revocation instant (when the payment was revoked). Epoch (0) if not applicable. /// Carries the provider's sub-second precision as a millisecond-resolution `sys_ms`; the wire @@ -403,7 +403,7 @@ struct ProStatusResponse : ResponseBase { /// This timestamp may be in the past if the user no longer has active payments. Overtime the /// Pro Backend may prune user history and so after long lapses of activity, a user's /// subscription history may be deleted. - sys_seconds expiry_at; + std::chrono::sys_seconds expiry_at; /// How much longer entitlement continues PAST `expiry_at`: the payment provider's dunning /// window (the leeway it allows itself to retry a failed renewal) plus the backend's own diff --git a/include/session/random.hpp b/include/session/random.hpp index 9968b1e90..1acfac821 100644 --- a/include/session/random.hpp +++ b/include/session/random.hpp @@ -24,7 +24,7 @@ struct CSRNG { uint64_t operator()() const { uint64_t i; - randombytes((uint8_t*)&i, sizeof(i)); + randombytes_buf(&i, sizeof(i)); return i; }; }; @@ -36,6 +36,23 @@ inline constexpr CSRNG csrng{}; namespace session::random { +/// API: random/random_fill +/// +/// Wrapper around the randombytes_buf function. +/// +/// Inputs: +/// - `buf` -- span to fill with random bytes +/// +/// Outputs: None. +void fill(std::span buf); +void fill(std::span buf); + +/// API: random/random_fill_deterministic +/// +/// Wrapper around randombytes_buf_deterministic: fills `buf` with deterministic pseudorandom +/// bytes derived from the given 32-byte seed. +void fill_deterministic(std::span buf, std::span seed); + /// API: random/random /// /// Wrapper around the randombytes_buf function. @@ -45,7 +62,7 @@ namespace session::random { /// /// Outputs: /// - random bytes of the specified length. -std::vector random(size_t size); +std::vector random(size_t size); /// API: random/random_base32 /// @@ -67,7 +84,7 @@ std::string random_base32(size_t size); /// /// Outputs: /// - generated id string. -std::string unique_id(std::string_view prefix); +std::string unique_id(std::string_view prefix, size_t random_len = 4); /// API: random/get_uniform_distribution /// diff --git a/include/session/session_encrypt.h b/include/session/session_encrypt.h index 6ded17298..1cf8bf11d 100644 --- a/include/session/session_encrypt.h +++ b/include/session/session_encrypt.h @@ -239,7 +239,7 @@ typedef struct session_decrypt_group_message_result { size_t index; // Index of the key that successfully decrypted the message char session_id[66]; // In hex span_u8 plaintext; // Decrypted message on success. Must be freed by calling the CRT's `free` - char error_len_incl_null_terminator; + size_t error_len_incl_null_terminator; } session_decrypt_group_message_result; /// API: crypto/session_decrypt_group_message @@ -343,7 +343,7 @@ LIBSESSION_EXPORT bool session_decrypt_push_notification( /// Inputs: /// - `plaintext_in` -- [in] the data to encrypt. /// - `plaintext_len` -- [in] the length of `plaintext_in`. -/// - `enc_key_in` -- [in] the key to use for encryption (32 bytes). +/// - `key_in` -- [in] the 32-byte symmetric key. /// - `ciphertext_out` -- [out] Pointer-pointer to an output buffer; a new buffer is allocated, the /// encrypted data written to it, and then the pointer to that buffer is stored here. /// This buffer must be `free()`d by the caller when done with it *unless* the function returns @@ -356,7 +356,7 @@ LIBSESSION_EXPORT bool session_decrypt_push_notification( LIBSESSION_EXPORT bool session_encrypt_xchacha20( const unsigned char* plaintext_in, size_t plaintext_len, - const unsigned char* enc_key_in, /* 32 bytes */ + const unsigned char* key_in, /* 32 bytes */ unsigned char** ciphertext_out, size_t* ciphertext_len); @@ -367,7 +367,7 @@ LIBSESSION_EXPORT bool session_encrypt_xchacha20( /// Inputs: /// - `ciphertext_in` -- [in] the data to decrypt. /// - `ciphertext_len` -- [in] the length of `ciphertext_in`. -/// - `enc_key_in` -- [in] the key to use for decryption (32 bytes). +/// - `key_in` -- [in] the 32-byte symmetric key. /// - `plaintext_out` -- [out] Pointer-pointer to an output buffer; a new buffer is allocated, the /// decrypted data written to it, and then the pointer to that buffer is stored here. /// This buffer must be `free()`d by the caller when done with it *unless* the function returns @@ -380,7 +380,7 @@ LIBSESSION_EXPORT bool session_encrypt_xchacha20( LIBSESSION_EXPORT bool session_decrypt_xchacha20( const unsigned char* ciphertext_in, size_t ciphertext_len, - const unsigned char* enc_key_in, /* 32 bytes */ + const unsigned char* key_in, /* 32 bytes */ unsigned char** plaintext_out, size_t* plaintext_len); diff --git a/include/session/session_encrypt.hpp b/include/session/session_encrypt.hpp index 7635c77e2..243039c1e 100644 --- a/include/session/session_encrypt.hpp +++ b/include/session/session_encrypt.hpp @@ -2,11 +2,15 @@ #include +#include #include #include +#include #include #include +#include "crypto/ed25519.hpp" + // Helper functions for the "Session Protocol" encryption mechanism. This is the encryption used // for DMs sent from one Session user to another. // @@ -53,9 +57,8 @@ namespace session { /// Performs session protocol encryption, typically for a DM sent between Session users. /// /// Inputs: -/// - `ed25519_privkey` -- the libsodium-style secret key of the sender, 64 bytes. Can also be -/// passed as a 32-byte seed, but the 64-byte value is preferrable (to avoid needing to -/// recompute the public key from the seed). +/// - `ed25519_privkey` -- the Ed25519 private key of the sender; accepts a 32-byte seed or +/// 64-byte libsodium key (the latter avoids recomputing the public key from the seed). /// - `recipient_pubkey` -- the recipient X25519 pubkey, either as a 0x05-prefixed session ID /// (33 bytes) or an unprefixed pubkey (32 bytes). /// - `message` -- the message to encrypt for the recipient. @@ -63,10 +66,10 @@ namespace session { /// Outputs: /// - The encrypted ciphertext to send. /// - Throw if encryption fails or (which typically means invalid keys provided) -std::vector encrypt_for_recipient( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message); +std::vector encrypt_for_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message); /// API: crypto/encrypt_for_recipient_deterministic /// @@ -84,10 +87,10 @@ std::vector encrypt_for_recipient( /// /// Outputs: /// Identical to `encrypt_for_recipient`. -std::vector encrypt_for_recipient_deterministic( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message); +std::vector encrypt_for_recipient_deterministic( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message); /// API: crypto/session_encrypt_for_blinded_recipient /// @@ -104,11 +107,188 @@ std::vector encrypt_for_recipient_deterministic( /// Outputs: /// - The encrypted ciphertext to send. /// - Throw if encryption fails or (which typically means invalid keys provided) -std::vector encrypt_for_blinded_recipient( - std::span ed25519_privkey, - std::span server_pk, - std::span recipient_blinded_id, - std::span message); +std::vector encrypt_for_blinded_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span server_pk, + std::span recipient_blinded_id, + std::span message); + +/// API: crypto/encrypt_for_recipient_v2 +/// +/// Encrypts a v2 Session DM (PFS + post-quantum) for a recipient. +/// +/// The wire format of the returned ciphertext is: +/// 0x00 0x02 | ki (2B) | E (32B) | mlkem_ct (1088B) | xchacha20poly1305_ciphertext +/// +/// where: +/// - `ki` is an encrypted key indicator used by the recipient to cheaply identify which of +/// their current account keys was used, without revealing it to outside observers. +/// - `E` is an ephemeral X25519 pubkey. +/// - `mlkem_ct` is an ML-KEM-768 ciphertext. +/// - The xchacha20poly1305 ciphertext contains the signed, padded inner plaintext. +/// +/// The inner plaintext is a bt-encoded dict with: +/// - "S": the sender's Ed25519 pubkey (32 bytes) +/// - "c": the message content (typically a serialized protobuf Content) +/// - "~": a 64-byte Ed25519 signature over a BLAKE2b-64 hash of the preceding content, +/// keyed with the recipient's 33-byte Session ID (personalized "SessionV2Message") +/// - "~P": optional Session Pro Ed25519 signature verifying the sender's Pro status. The public +/// key for verifying this is embedded within the protobuf Content. Present only when the +/// message uses Session Pro features; absent otherwise. +/// +/// Inputs: +/// - `sender_ed25519_privkey` -- sender's 32-byte seed or 64-byte Ed25519 secret key +/// - `recipient_session_id` -- 33-byte 0x05-prefixed long-term X25519 pubkey (S with prefix) +/// - `recipient_account_x25519` -- 32-byte recently-fetched PFS account X25519 pubkey (X) +/// - `recipient_account_mlkem768` -- 1184-byte recently-fetched PFS account ML-KEM-768 pubkey (M) +/// - `content` -- the plaintext message content to encrypt (typically a serialized protobuf) +/// - `pro_ed25519_privkey` -- optional Session Pro rotating Ed25519 private key (32-byte seed or +/// 64-byte libsodium key). When provided, a `~P` signature is appended to the inner bt-dict, +/// signing all preceding dict content. Pass nullopt / omit when not using Session Pro. +/// +/// Outputs: +/// - The encrypted v2 ciphertext to send to the swarm. +/// - Throws on invalid keys or encryption failure. +std::vector encrypt_for_recipient_v2( + const ed25519::PrivKeySpan& sender_ed25519_privkey, + std::span recipient_session_id, + std::span recipient_account_x25519, + std::span recipient_account_mlkem768, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_ed25519_privkey = std::nullopt); + +/// Exception thrown when a v2 message could not be decrypted with a given account key. The +/// caller should catch this and try the next candidate key. Other exceptions (e.g., +/// std::runtime_error for invalid message format, or std::invalid_argument for bad keys) are +/// unrecoverable and should not be caught per-key. +struct DecryptV2Error : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/// Result of decrypt_incoming_v2. +struct DecryptV2Result { + std::vector content; ///< Decrypted message content. + b33 sender_session_id; ///< 05-prefixed Session ID of the sender. + std::optional pro_signature; ///< Pro sig, if present. +}; + +/// API: crypto/decrypt_incoming_v2_prefix +/// +/// Extracts and decrypts the 2-byte key indicator from a v2 Session DM ciphertext, returning +/// the first 2 bytes of the ML-KEM-768 public key that was used to encrypt the message. +/// +/// This is a cheap pre-filter step: the caller uses the returned prefix to look up which of +/// their PFS account keys match, then passes the matching key(s) to `decrypt_incoming_v2`. +/// +/// Inputs: +/// - `x25519_sec` -- 32-byte long-term X25519 secret key of the recipient (the raw key, *not* +/// the Ed25519 key). +/// - `x25519_pub` -- 32-byte long-term X25519 public key of the recipient (i.e. the Session ID +/// bytes without the `0x05` prefix). +/// - `ciphertext` -- wire-format v2 ciphertext as produced by `encrypt_for_recipient_v2`. +/// +/// Outputs: +/// - The recovered 2-byte ML-KEM-768 public key prefix. +/// - Throws `std::runtime_error` if the ciphertext is too short or has the wrong prefix bytes. +std::array decrypt_incoming_v2_prefix( + std::span x25519_sec, + std::span x25519_pub, + std::span ciphertext); + +/// API: crypto/decrypt_incoming_v2 +/// +/// Inverse of `encrypt_for_recipient_v2`: decrypts a v2 Session DM using a single PFS account +/// key. Verifies the X-Wing (ML-KEM-768 + X25519) shared secret derivation and the inner +/// Ed25519 message signature. +/// +/// Typical usage: call `decrypt_incoming_v2_prefix` to get the 2-byte ML-KEM prefix, look up +/// all PFS account keys whose ML-KEM-768 public key begins with that prefix, then call this +/// function for each candidate, catching `DecryptV2Error` and trying the next key on failure: +/// +/// auto prefix = decrypt_incoming_v2_prefix(x25519_sec, x25519_pub, ciphertext); +/// for (auto& key : pfs_keys_by_prefix(prefix)) { +/// try { +/// return decrypt_incoming_v2(session_id, key.x_sec, key.x_pub, +/// key.mlkem_sec, ciphertext); +/// } catch (const DecryptV2Error&) { continue; } +/// } +/// throw std::runtime_error{"no PFS account key could decrypt the message"}; +/// +/// Inputs: +/// - `recipient_session_id` -- 33-byte 0x05-prefixed Session ID of the recipient. Used to +/// verify the inner Ed25519 message signature; no private key material is needed here. +/// - `account_pfs_x25519_sec` -- 32-byte X25519 secret key of the PFS account key to try. +/// - `account_pfs_x25519_pub` -- 32-byte X25519 public key of the PFS account key to try. +/// - `account_pfs_mlkem768_sec` -- 2400-byte ML-KEM-768 secret key of the PFS account key +/// to try. +/// - `ciphertext` -- wire-format v2 ciphertext as produced by `encrypt_for_recipient_v2`. +/// +/// Outputs: +/// - `DecryptV2Result` with the decrypted content, 33-byte (05-prefixed) sender Session ID, +/// and an optional 64-byte Session Pro signature. +/// - Throws `DecryptV2Error` if the key did not decrypt the message (try the next candidate). +/// - Throws `std::runtime_error` for unrecoverable errors (invalid format, signature failure). +DecryptV2Result decrypt_incoming_v2( + std::span recipient_session_id, + std::span account_pfs_x25519_sec, + std::span account_pfs_x25519_pub, + std::span account_pfs_mlkem768_sec, + std::span ciphertext); + +/// API: crypto/encrypt_for_recipient_v2_nopfs +/// +/// Encrypts a v2 Session DM using the non-PFS fallback (long-term X25519 DH only). +/// +/// Wire format is identical to `encrypt_for_recipient_v2`: +/// 0x00 0x02 | ki (2B) | E (32B) | outer_ct (1088B) | xchacha20poly1305_ciphertext +/// +/// but `ki` and `outer_ct` carry no key material — they are random bytes used only to make +/// non-PFS messages externally indistinguishable from PFS+PQ messages. The actual shared +/// secret is derived as: +/// ss = eR (X25519 DH with ephemeral secret e and recipient long-term pubkey R) +/// ss = SHA3-256(ss || R || E || "SessionV2NonPFS") +/// k,n = SHAKE256("SessionV2NonPFSSS", ss) → 32-byte key + 24-byte nonce +/// +/// Inputs: +/// - `sender_ed25519_privkey` -- sender's 32-byte seed or 64-byte Ed25519 secret key +/// - `recipient_session_id` -- 33-byte 0x05-prefixed long-term X25519 pubkey of the recipient +/// - `content` -- the plaintext message content to encrypt +/// - `pro_ed25519_privkey` -- optional Session Pro rotating Ed25519 private key. When provided, +/// a `~P` signature is appended to the inner bt-dict. Pass nullopt / omit when not using Pro. +/// +/// Outputs: +/// - Wire-format v2 ciphertext (non-PFS). +/// - Throws on key or encryption failure. +std::vector encrypt_for_recipient_v2_nopfs( + const ed25519::PrivKeySpan& sender_ed25519_privkey, + std::span recipient_session_id, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_ed25519_privkey = std::nullopt); + +/// API: crypto/decrypt_incoming_v2_nopfs +/// +/// Decrypts a v2 Session DM using the non-PFS fallback (long-term X25519 DH only). +/// +/// Performs the inverse of `encrypt_for_recipient_v2_nopfs`. The `ki` and ML-KEM fields in +/// the wire format are ignored; only the ephemeral pubkey E and the recipient's long-term +/// X25519 key pair are used. +/// +/// Inputs: +/// - `recipient_session_id` -- 33-byte 0x05-prefixed Session ID of the recipient (used to +/// verify the inner Ed25519 message signature). +/// - `x25519_sec` -- 32-byte long-term X25519 secret key of the recipient. +/// - `x25519_pub` -- 32-byte long-term X25519 public key of the recipient. +/// - `ciphertext` -- wire-format v2 ciphertext. +/// +/// Outputs: +/// - `DecryptV2Result` with the decrypted content, sender Session ID, and optional Pro sig. +/// - Throws `DecryptV2Error` if AEAD authentication fails (wrong key — try PFS path instead). +/// - Throws `std::runtime_error` for unrecoverable errors (invalid format, signature failure). +DecryptV2Result decrypt_incoming_v2_nopfs( + std::span recipient_session_id, + std::span x25519_sec, + std::span x25519_pub, + std::span ciphertext); static constexpr size_t GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE = 1'000'000; @@ -163,8 +343,8 @@ static constexpr size_t GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE = 1'000'000; /// exhaustion attacks. /// /// Inputs: -/// - `user_ed25519_privkey` -- the private key of the user. Can be a 32-byte seed, or a 64-byte -/// libsodium secret key. The latter is a bit faster as it doesn't have to re-compute the pubkey +/// - `user_ed25519_privkey` -- the Ed25519 private key of the user; accepts a 32-byte seed or +/// 64-byte libsodium key (the latter avoids recomputing the public key from the seed). /// - `group_ed25519_pubkey` -- The 32 byte public key of the group /// - group_enc_key -- The group's encryption key (32 bytes or 64-byte libsodium key) for groups v2 /// messages, typically the latest key for the group (e.g., Keys::group_enc_key). @@ -177,11 +357,11 @@ static constexpr size_t GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE = 1'000'000; /// /// Outputs: /// - `ciphertext` -- the encrypted, etc. value to send to the swarm -std::vector encrypt_for_group( - std::span user_ed25519_privkey, - std::span group_ed25519_pubkey, - std::span group_enc_key, - std::span plaintext, +std::vector encrypt_for_group( + const ed25519::PrivKeySpan& user_ed25519_privkey, + std::span group_ed25519_pubkey, + std::span group_enc_key, + std::span plaintext, bool compress, size_t padding); @@ -204,14 +384,15 @@ std::vector encrypt_for_group( /// signed message. /// /// Inputs: -/// - `ed25519_privkey` -- the seed (32 bytes) or secret key (64 bytes) of the sender +/// - `ed25519_privkey` -- the Ed25519 private key of the sender; accepts a 32-byte seed or +/// 64-byte libsodium key. /// - `recipient_pubkey` -- the recipient X25519 pubkey, which may or may not be prefixed with the /// 0x05 session id prefix (33 bytes if prefixed, 32 if not prefixed). /// - `message` -- the message to embed and sign. -std::vector sign_for_recipient( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message); +std::vector sign_for_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message); /// API: crypto/decrypt_incoming /// @@ -219,18 +400,16 @@ std::vector sign_for_recipient( /// pubkey, and verifies that the sender Ed25519 signature on the message. /// /// Inputs: -/// - `ed25519_privkey` -- the private key of the recipient. Can be a 32-byte seed, or a 64-byte -/// libsodium secret key. The latter is a bit faster as it doesn't have to re-compute the pubkey -/// from the seed. +/// - `ed25519_privkey` -- the Ed25519 private key of the recipient; accepts a 32-byte seed or +/// 64-byte libsodium key. /// - `ciphertext` -- the encrypted data /// /// Outputs: -/// - `std::pair, std::vector>` -- the plaintext binary -/// data that was encrypted and the -/// sender's ED25519 pubkey, *if* the message decrypted and validated successfully. Throws on -/// error. -std::pair, std::vector> decrypt_incoming( - std::span ed25519_privkey, std::span ciphertext); +/// - `std::pair, b32>` -- the plaintext binary data that was encrypted +/// and the sender's Ed25519 pubkey, *if* the message decrypted and validated successfully. +/// Throws on error. +std::pair, b32> decrypt_incoming( + const ed25519::PrivKeySpan& ed25519_privkey, std::span ciphertext); /// API: crypto/decrypt_incoming /// @@ -246,14 +425,13 @@ std::pair, std::vector> decrypt_incomi /// - `ciphertext` -- the encrypted data /// /// Outputs: -/// - `std::pair, std::vector>` -- the plaintext binary -/// data that was encrypted and the -/// sender's ED25519 pubkey, *if* the message decrypted and validated successfully. Throws on -/// error. -std::pair, std::vector> decrypt_incoming( - std::span x25519_pubkey, - std::span x25519_seckey, - std::span ciphertext); +/// - `std::pair, b32>` -- the plaintext binary data that was encrypted +/// and the sender's Ed25519 pubkey, *if* the message decrypted and validated successfully. +/// Throws on error. +std::pair, b32> decrypt_incoming( + std::span x25519_pubkey, + std::span x25519_seckey, + std::span ciphertext); /// API: crypto/decrypt_incoming /// @@ -261,17 +439,16 @@ std::pair, std::vector> decrypt_incomi /// signature on the message and converts the extracted sender's Ed25519 pubkey into a session ID. /// /// Inputs: -/// - `ed25519_privkey` -- the private key of the recipient. Can be a 32-byte seed, or a 64-byte -/// libsodium secret key. The latter is a bit faster as it doesn't have to re-compute the pubkey -/// from the seed. +/// - `ed25519_privkey` -- the Ed25519 private key of the recipient; accepts a 32-byte seed or +/// 64-byte libsodium key. /// - `ciphertext` -- the encrypted data /// /// Outputs: /// - `std::pair, std::string>` -- the plaintext binary data that was /// encrypted and the /// session ID (in hex), *if* the message decrypted and validated successfully. Throws on error. -std::pair, std::string> decrypt_incoming_session_id( - std::span ed25519_privkey, std::span ciphertext); +std::pair, std::string> decrypt_incoming_session_id( + const ed25519::PrivKeySpan& ed25519_privkey, std::span ciphertext); /// API: crypto/decrypt_incoming /// @@ -286,13 +463,13 @@ std::pair, std::string> decrypt_incoming_session_id( /// - `ciphertext` -- the encrypted data /// /// Outputs: -/// - `std::pair, std::string>` -- the plaintext binary data that was +/// - `std::pair, std::string>` -- the plaintext binary data that was /// encrypted and the /// session ID (in hex), *if* the message decrypted and validated successfully. Throws on error. -std::pair, std::string> decrypt_incoming_session_id( - std::span x25519_pubkey, - std::span x25519_seckey, - std::span ciphertext); +std::pair, std::string> decrypt_incoming_session_id( + std::span x25519_pubkey, + std::span x25519_seckey, + std::span ciphertext); /// API: crypto/decrypt_from_blinded_recipient /// @@ -301,10 +478,8 @@ std::pair, std::string> decrypt_incoming_session_id( /// the `ciphertext` is an outgoing message and decrypts it as such. /// /// Inputs: -/// - `ed25519_privkey` -- the Ed25519 private key of the receiver. Can be a 32-byte seed, or a -/// 64-byte -/// libsodium secret key. The latter is a bit faster as it doesn't have to re-compute the pubkey -/// from the seed. +/// - `ed25519_privkey` -- the Ed25519 private key of the receiver; accepts a 32-byte seed or +/// 64-byte libsodium key. /// - `server_pk` -- the public key of the community server to route the blinded message through /// (32 bytes). /// - `sender_id` -- the blinded id of the sender including the blinding prefix (33 bytes), @@ -317,17 +492,17 @@ std::pair, std::string> decrypt_incoming_session_id( /// - `std::pair, std::string>` -- the plaintext binary data that was /// encrypted and the /// session ID (in hex), *if* the message decrypted and validated successfully. Throws on error. -std::pair, std::string> decrypt_from_blinded_recipient( - std::span ed25519_privkey, - std::span server_pk, - std::span sender_id, - std::span recipient_id, - std::span ciphertext); +std::pair, std::string> decrypt_from_blinded_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span server_pk, + std::span sender_id, + std::span recipient_id, + std::span ciphertext); struct DecryptGroupMessage { size_t index; // Index of the key that successfully decrypted the message std::string session_id; // In hex - std::vector plaintext; + std::vector plaintext; }; /// API: crypto/decrypt_group_message @@ -357,9 +532,9 @@ struct DecryptGroupMessage { /// (and possibly log) but otherwise ignore such exceptions and just not process the message if /// it throws. DecryptGroupMessage decrypt_group_message( - std::span> decrypt_ed25519_privkey_list, - std::span group_ed25519_pubkey, - std::span ciphertext); + std::span> group_enc_keys, + std::span group_ed25519_pubkey, + std::span ciphertext); /// API: crypto/decrypt_ons_response /// @@ -375,8 +550,8 @@ DecryptGroupMessage decrypt_group_message( /// a session ID. Throws on error/failure. std::string decrypt_ons_response( std::string_view lowercase_name, - std::span ciphertext, - std::optional> nonce); + std::span ciphertext, + std::optional> nonce); /// API: crypto/decrypt_push_notification /// @@ -391,8 +566,8 @@ std::string decrypt_ons_response( /// - `std::vector` -- the decrypted push notification payload, *if* the decryption /// was /// successful. Throws on error/failure. -std::vector decrypt_push_notification( - std::span payload, std::span enc_key); +std::vector decrypt_push_notification( + std::span payload, std::span enc_key); /// API: crypto/encrypt_xchacha20 /// @@ -400,12 +575,12 @@ std::vector decrypt_push_notification( /// /// Inputs: /// - `plaintext` -- the data to encrypt. -/// - `enc_key` -- the key to use for encryption (32 bytes). +/// - `key` -- the 32-byte symmetric key. /// /// Outputs: /// - `std::vector` -- the resulting ciphertext. -std::vector encrypt_xchacha20( - std::span plaintext, std::span enc_key); +std::vector encrypt_xchacha20( + std::span plaintext, std::span key); /// API: crypto/decrypt_xchacha20 /// @@ -413,11 +588,11 @@ std::vector encrypt_xchacha20( /// /// Inputs: /// - `ciphertext` -- the data to decrypt. -/// - `enc_key` -- the key to use for decryption (32 bytes). +/// - `key` -- the 32-byte symmetric key. /// /// Outputs: -/// - `std::vector` -- the resulting plaintext. -std::vector decrypt_xchacha20( - std::span ciphertext, std::span enc_key); +/// - `std::vector` -- the resulting plaintext. +std::vector decrypt_xchacha20( + std::span ciphertext, std::span key); } // namespace session diff --git a/include/session/session_protocol.h b/include/session/session_protocol.h index 970eb9a53..21201772f 100644 --- a/include/session/session_protocol.h +++ b/include/session/session_protocol.h @@ -72,69 +72,32 @@ typedef struct session_protocol_pro_signed_message { } session_protocol_pro_signed_message; typedef struct session_protocol_pro_proof { - bytes32 revocation_tag; - bytes32 rotating_pubkey; + cbytes32 revocation_tag; + cbytes32 rotating_pubkey; int64_t expiry_ts; - bytes64 sig; + cbytes64 sig; } session_protocol_pro_proof; -// Feature flags for profile features where each enum value indicates the bit position in the -// corresponding bitset, e.g. (1 << ENUM_VAL) -typedef enum SESSION_PROTOCOL_PRO_PROFILE_FEATURES { - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE, - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR, - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_COUNT, -} SESSION_PROTOCOL_PRO_PROFILE_FEATURES; - -// Strongly typed bitset for profile features. Each profile enum value corresponds to the bit -// position to set on the bitset (e.g. 1 << ENUM_VALUE). This bitset is wrapped in a struct and has -// helper functions (`session_protocol_pro_profile_bitset_*` family of functions) that accepts the -// typed-enum to mitigate against mixing up the profile features with the message features. +// Session Pro feature flag bits. These are plain `uint64_t` bit masks (`1 << position`) that are +// OR'd together into a profile/message feature bitset (itself just a `uint64_t`). They mirror the +// C++ `session::ProProfileFlags` / `session::ProMessageFlags` enum classes, which are the source +// of truth; these constants are defined from those enum values in session_protocol.cpp. // -// The enums are kept as bit positions (ENUM_VAL = 1 << N) instead of bit values (ENUM_VAL = 1) -// for ergonomic usage in the way we sync and store these bitsets on the protocol swarms. These -// bitsets are stored as sets which allows us to do diffs and deltas on the set of values. The -// syncing scheme does not allow bit-level deltas which makes handling conflicts between competing -// synced configurations, awkward. -typedef struct session_protocol_pro_profile_bitset { - uint64_t data; -} session_protocol_pro_profile_bitset; - -// Feature flags for message features where each enum value indicates the bit position in the -// corresponding bitset. -typedef enum SESSION_PROTOCOL_PRO_MESSAGE_FEATURES { - SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT, -} SESSION_PROTOCOL_PRO_MESSAGE_FEATURES; - -// Strongly typed bitset for Session Pro message features (see -// `session_protocol_pro_profile_bitset`) -typedef struct session_protocol_pro_message_bitset { - uint64_t data; -} session_protocol_pro_message_bitset; +// Manipulate a bitset directly with the standard bitwise operators, e.g.: +// uint64_t features = 0; +// features |= SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE; // set +// features &= ~SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE; // unset +// if (features & SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE) { ... } // test +extern const uint64_t SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE; +extern const uint64_t SESSION_PROTOCOL_PRO_PROFILE_FEATURE_ANIMATED_AVATAR; + +extern const uint64_t SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT; typedef enum SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS { // See session::ProFeaturesForMsgStatus SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_SUCCESS, SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_EXCEEDS_CHARACTER_LIMIT, } SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS; -typedef enum SESSION_PROTOCOL_DESTINATION_TYPE { // See session::DestinationType - SESSION_PROTOCOL_DESTINATION_TYPE_SYNC_OR_1O1, - SESSION_PROTOCOL_DESTINATION_TYPE_GROUP, - SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX, - SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY, -} SESSION_PROTOCOL_DESTINATION_TYPE; - -typedef struct session_protocol_destination { // See session::Destination - SESSION_PROTOCOL_DESTINATION_TYPE type; - const void* pro_rotating_ed25519_privkey; - size_t pro_rotating_ed25519_privkey_len; - bytes33 recipient_pubkey; - uint64_t sent_timestamp_ms; - bytes32 community_inbox_server_pubkey; - bytes33 group_ed25519_pubkey; - bytes32 group_enc_key; -} session_protocol_destination; - // Indicates which optional fields in the envelope has been populated out of the optional fields in // an envelope after it has been parsed off the wire. typedef uint32_t SESSION_PROTOCOL_ENVELOPE_FLAGS; @@ -149,10 +112,10 @@ enum ENVELOPE_FLAGS_ { typedef struct session_protocol_envelope { SESSION_PROTOCOL_ENVELOPE_FLAGS flags; uint64_t timestamp_ms; - bytes33 source; + cbytes33 source; uint32_t source_device; uint64_t server_timestamp; - bytes64 pro_sig; + cbytes64 pro_sig; } session_protocol_envelope; typedef struct session_protocol_decode_envelope_keys { @@ -164,8 +127,9 @@ typedef struct session_protocol_decode_envelope_keys { typedef struct session_protocol_decoded_pro { SESSION_PROTOCOL_PRO_STATUS status; session_protocol_pro_proof proof; - session_protocol_pro_message_bitset msg_bitset; - session_protocol_pro_profile_bitset profile_bitset; + // Bitsets of SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_* / SESSION_PROTOCOL_PRO_PROFILE_FEATURE_* + uint64_t msg_bitset; + uint64_t profile_bitset; } session_protocol_decoded_pro; typedef struct session_protocol_decoded_envelope { @@ -174,8 +138,8 @@ typedef struct session_protocol_decoded_envelope { bool success; session_protocol_envelope envelope; span_u8 content_plaintext; - bytes32 sender_ed25519_pubkey; - bytes32 sender_x25519_pubkey; + cbytes32 sender_ed25519_pubkey; + cbytes32 sender_x25519_pubkey; session_protocol_decoded_pro pro; size_t error_len_incl_null_terminator; } session_protocol_decoded_envelope; @@ -200,7 +164,7 @@ typedef struct session_protocol_encoded_for_destination { size_t error_len_incl_null_terminator; } session_protocol_encoded_for_destination; -/// API: session_protocol/session_protocol_encode_for_destination_free +/// API: session_protocol/session_protocol_encrypt_for_destination_free /// /// Free the encryption result for a destination produced by /// `session_protocol_encrypt_for_destination`. It is safe to pass a `NULL` or any result returned @@ -218,7 +182,7 @@ typedef struct session_protocol_decoded_community_message { session_protocol_envelope envelope; span_u8 content_plaintext; bool has_pro; - bytes64 pro_sig; + cbytes64 pro_sig; session_protocol_decoded_pro pro; size_t error_len_incl_null_terminator; } session_protocol_decoded_community_message; @@ -235,41 +199,9 @@ typedef struct session_protocol_decoded_community_message { LIBSESSION_EXPORT void session_protocol_decode_for_community_free( session_protocol_decoded_community_message* community_msg); -/// API: session_protocol/session_protocol_pro_profile_bitset_is_set -/// -/// Check if the feature flag is set on the bitset -LIBSESSION_EXPORT bool session_protocol_pro_profile_bitset_is_set( - session_protocol_pro_profile_bitset value, SESSION_PROTOCOL_PRO_PROFILE_FEATURES features); - -/// API: session_protocol/session_protocol_pro_profile_bitset_set -/// -/// Set the feature flag on the bitset -LIBSESSION_EXPORT void session_protocol_pro_profile_bitset_set( - session_protocol_pro_profile_bitset* value, SESSION_PROTOCOL_PRO_PROFILE_FEATURES features); - -/// API: session_protocol/session_protocol_pro_profile_bitset_unset -/// -/// Unset the feature flag on the bitset -LIBSESSION_EXPORT void session_protocol_pro_profile_bitset_unset( - session_protocol_pro_profile_bitset* value, SESSION_PROTOCOL_PRO_PROFILE_FEATURES features); - -/// API: session_protocol/session_protocol_pro_profile_bitset_is_set -/// -/// Check if the feature flag is set on the bitset -LIBSESSION_EXPORT bool session_protocol_pro_message_bitset_is_set( - session_protocol_pro_message_bitset value, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features); - -/// API: session_protocol/session_protocol_pro_profile_bitset_set -/// -/// Set the feature flag on the bitset -LIBSESSION_EXPORT void session_protocol_pro_message_bitset_set( - session_protocol_pro_message_bitset* value, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features); - -/// API: session_protocol/session_protocol_pro_profile_bitset_unset -/// -/// Unset the feature flag on the bitset -LIBSESSION_EXPORT void session_protocol_pro_message_bitset_unset( - session_protocol_pro_message_bitset* value, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features); +// The Pro feature bitsets are plain `uint64_t` masks of SESSION_PROTOCOL_PRO_*_FEATURES_* bits; +// set/unset/test them with the standard bitwise operators (see the feature constants above). No +// accessor functions are needed. /// API: session_protocol/session_protocol_pro_proof_verify_signature /// @@ -382,7 +314,7 @@ typedef struct session_protocol_pro_features_for_msg { /// On error (status != OK), a static, null-terminated English diagnostic string; NULL when /// there is no error. const char* error; - session_protocol_pro_message_bitset bitset; + uint64_t bitset; // Mask of SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_* bits } session_protocol_pro_features_for_msg; /// API: session_protocol/session_protocol_pro_features_for_message @@ -403,13 +335,13 @@ LIBSESSION_EXPORT session_protocol_pro_features_for_msg session_protocol_pro_features_for_message( size_t codepoint_count); -/// API: session_protocol_encode_for_1o1 +/// API: session_protocol_encode_dm_v1 /// /// Encode a plaintext message for a one-on-one (1o1) conversation or sync message in the Session /// Protocol. This function wraps the plaintext in the necessary structures and encrypts it for /// transmission to a single recipient. /// -/// See: session_protocol/encode_for_1o1 for more information +/// See: session_protocol/encode_dm_v1 for more information /// /// The encoded result must be freed with session_protocol_encrypt_for_destination_free when /// the caller is done with the result. @@ -449,13 +381,13 @@ session_protocol_pro_features_for_msg session_protocol_pro_features_for_message( /// required to write the error. Both counts include the null-terminator. The user must allocate /// at minimum the requested length for the error message to be preserved in full. LIBSESSION_EXPORT -session_protocol_encoded_for_destination session_protocol_encode_for_1o1( +session_protocol_encoded_for_destination session_protocol_encode_dm_v1( const void* plaintext, size_t plaintext_len, const void* ed25519_privkey, size_t ed25519_privkey_len, uint64_t sent_timestamp_ms, - const bytes33* recipient_pubkey, + const cbytes33* recipient_pubkey, OPTIONAL const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, OPTIONAL char* error, @@ -514,8 +446,8 @@ session_protocol_encoded_for_destination session_protocol_encode_for_community_i size_t plaintext_len, const void* ed25519_privkey, size_t ed25519_privkey_len, - const bytes33* recipient_pubkey, - const bytes32* community_pubkey, + const cbytes33* recipient_pubkey, + const cbytes32* community_pubkey, OPTIONAL const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, OPTIONAL char* error, @@ -628,62 +560,13 @@ session_protocol_encoded_for_destination session_protocol_encode_for_group( const void* ed25519_privkey, size_t ed25519_privkey_len, uint64_t sent_timestamp_ms, - const bytes33* group_ed25519_pubkey, - const bytes32* group_enc_key, + const cbytes33* group_ed25519_pubkey, + const cbytes32* group_enc_key, OPTIONAL const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, OPTIONAL char* error, size_t error_len) NON_NULL_ARG(1, 3, 6, 7); -/// API: session_protocol/session_protocol_encrypt_for_destination -/// -/// Given an unencrypted plaintext representation of the content (i.e.: protobuf encoded stream of -/// `Content`), encrypt and/or wrap the plaintext in the necessary structures for transmission on -/// the Session Protocol. -/// -/// See: session_protocol/encrypt_for_destination for more information -/// -/// The encoded result must be freed with `session_protocol_encrypt_for_destination_free` when -/// the caller is done with the result. -/// -/// Inputs: -/// - `plaintext` -- the protobuf serialised payload containing the protobuf encoded stream, -/// `Content`. It must not be already be encrypted. -/// - `ed25519_privkey` -- the libsodium-style secret key of the sender, 64 bytes. Can also be -/// passed as a 32-byte seed. Used to encrypt the plaintext. -/// - `dest` -- the extra metadata indicating the destination of the message and the necessary data -/// to encrypt a message for that destination. -/// - `error` -- Pointer to the character buffer to be populated with the error message if the -/// returned `success` was false, untouched otherwise. If this is set to `NULL`, then on failure, -/// the returned `error_len_incl_null_terminator` is the number of bytes required by the user to -/// receive the error. The message may be truncated if the buffer is too small, but it's always -/// guaranteed that `error` is null-terminated on failure when a buffer is passed in even if the -/// error must be truncated to fit in the buffer. -/// - `error_len` -- The capacity of the character buffer passed by the user. This should be 0 if -/// `error` is NULL. This function will fill the buffer up to `error_len - 1` characters with the -/// last character reserved for the null-terminator. -/// -/// Outputs: -/// - `success` -- True if encoding was successful, if the underlying implementation threw -/// an exception then this is caught internally and success is set to false. All remaining fields -/// are to be ignored in the result on failure. -/// - `ciphertext` -- Encryption result for the plaintext. The retured payload is suitable for -/// sending on the wire (i.e: it has been protobuf encoded/wrapped if necessary). -/// - `error_len_incl_null_terminator` The length of the error message if `success` was false. If -/// the user passes in an non-`NULL` error buffer this is amount of characters written to the -/// error buffer. If the user passes in a `NULL` error buffer, this is the amount of characters -/// required to write the error. Both counts include the null-terminator. The user must allocate -/// at minimum the requested length for the error message to be preserved in full. -LIBSESSION_EXPORT -session_protocol_encoded_for_destination session_protocol_encode_for_destination( - const void* plaintext, - size_t plaintext_len, - OPTIONAL const void* ed25519_privkey, - size_t ed25519_privkey_len, - const session_protocol_destination* dest, - OPTIONAL char* error, - size_t error_len) NON_NULL_ARG(1, 5); - /// API: session_protocol/session_protocol_decode_envelope /// /// Given an envelope payload (i.e.: protobuf encoded stream of `WebsocketRequestMessage` which diff --git a/include/session/session_protocol.hpp b/include/session/session_protocol.hpp index e553d3c81..6a89685cd 100644 --- a/include/session/session_protocol.hpp +++ b/include/session/session_protocol.hpp @@ -5,9 +5,13 @@ #include #include #include +#include +#include +#include #include #include #include +#include /// A complimentary file to session encrypt (which has the low level encryption function for Session /// protocol types). This file contains high-level helper functions for decoding payloads on the @@ -31,27 +35,6 @@ /// messages is libsession itself and that it will provide wrapper/proxy types for and handle /// converting those into the wire format. -// NOTE: In the CPP file we use C-style enums for bitfields and CPP-style enums for non-bitfield -// enums where we can to benefit from the type-safety of strong enums. -// -// CPP doesn't support named bitfields without casting or operator overloads but C-style -// enums support it very well. The only issue is that using a native C-style enum enforces some type -// restrictions that compilers dislike when attempting to manipulate bit fields. For example: -// -// enum Feature {x = 1 << 0, y = 1 << 1} -// Feature f = x | y -// -// Causes the compiler to complain about trying to do bit ops/assign an unsigned integer to an enum -// `Feature`. We use a common C pattern/trick by suffixing an underscore to the the original enum, -// then type define the non-suffixed enum to an unsigned integer: -// -// enum Feature_ {x = 1 << 0, y = 1 << 1} -// typedef U64 Feature -// Feature f = x | y -// -// Does not trigger errors as the underlying type of `f` is actually an unsigned integer. The type -// define is merely a hint to the user to what flags are to be used when manipulating the variable. - namespace session { using namespace std::literals; @@ -69,23 +52,6 @@ inline constexpr int STANDARD_PINNED_CONVERSATION_LIMIT = 5; /// envelope. inline constexpr int COMMUNITY_OR_1O1_MSG_PADDING = 160; -// Session Pro 16-byte signing domain prefixes; each prefixes the Ed25519-signed message for its -// endpoint (pro-wire-protocol.md §2 proof, §3 signed requests). ASCII, `_`-right-padded to 16 -// bytes (formerly the BLAKE2b personalisation, back when messages were pre-hashed). -// -// BUILD_PROOF_DOMAIN is also what pins a proof to its format: the `_v0` in it is part of the signed -// bytes, so a proof of some future format signed under its own domain simply fails verification -// here. That is why a proof carries no version field of its own -- and why this literal is a wire -// constant that must not be "tidied up" along with any C++ renaming. -inline constexpr std::string_view GENERATE_PROOF_DOMAIN = "ProGenerateProof"; -inline constexpr std::string_view BUILD_PROOF_DOMAIN = "ProProof_v0_____"; -inline constexpr std::string_view GET_PRO_STATUS_DOMAIN = "ProGetProStatus_"; -inline constexpr std::string_view GET_PAYMENT_DETAILS_DOMAIN = "ProGetPayDetails"; -static_assert(GENERATE_PROOF_DOMAIN.size() == 16); -static_assert(BUILD_PROOF_DOMAIN.size() == 16); -static_assert(GET_PRO_STATUS_DOMAIN.size() == 16); -static_assert(GET_PAYMENT_DETAILS_DOMAIN.size() == 16); - /// Rotation window for the Session Pro rotating key: ProProof::rotating_seed yields the same seed /// for all timestamps within one such period and a fresh one at each boundary. inline constexpr auto PRO_ROTATING_SEED_PERIOD = 7 * 24h; @@ -112,6 +78,23 @@ inline constexpr auto PRO_RENEWAL_BOUNDARY_DEFER = 1min; /// still leave at least this much of the current proof's validity. inline constexpr auto PRO_RENEWAL_BOUNDARY_MIN_VALIDITY = 5min; +// Session Pro 16-byte signing domain prefixes; each prefixes the Ed25519-signed message for its +// endpoint (pro-wire-protocol.md §2 proof, §3 signed requests). ASCII, `_`-right-padded to 16 +// bytes. +// +// BUILD_PROOF_DOMAIN is also what pins a proof to its format: the `_v0` in it is part of the signed +// bytes, so a proof of some future format signed under its own domain simply fails verification +// here. That is why a proof carries no version field of its own -- and why this literal is a wire +// constant that must not be "tidied up" along with any C++ renaming. +inline constexpr std::string_view GENERATE_PROOF_DOMAIN = "ProGenerateProof"; +inline constexpr std::string_view BUILD_PROOF_DOMAIN = "ProProof_v0_____"; +inline constexpr std::string_view GET_PRO_STATUS_DOMAIN = "ProGetProStatus_"; +inline constexpr std::string_view GET_PAYMENT_DETAILS_DOMAIN = "ProGetPayDetails"; +static_assert(GENERATE_PROOF_DOMAIN.size() == 16); +static_assert(BUILD_PROOF_DOMAIN.size() == 16); +static_assert(GET_PRO_STATUS_DOMAIN.size() == 16); +static_assert(GET_PAYMENT_DETAILS_DOMAIN.size() == 16); + enum class ProStatus { // Pro proof sig was not signed by the Pro backend key InvalidProBackendSig = SESSION_PROTOCOL_PRO_STATUS_INVALID_PRO_BACKEND_SIG, @@ -133,20 +116,20 @@ enum class ProStatus { class ProProof { public: /// Opaque revocation tag identifying this proof (from the Session Pro backend) - array_uc32 revocation_tag; + b32 revocation_tag; /// The public key that the Session client registers their Session Pro entitlement under. /// Session clients must sign messages with this key along side the sending of this proof for /// the network to authenticate their usage of the proof - array_uc32 rotating_pubkey; + b32 rotating_pubkey; /// Unix epoch timestamp to which this proof's entitlement to Session Pro features is valid to - sys_seconds expiry_at; + std::chrono::sys_seconds expiry_at; /// Signature over the contents of the proof. It is signed by the Session Pro Backend key which /// is the entity responsible for issueing tamper-proof Sesison Pro certificates for Session /// clients. - array_uc64 sig; + b64 sig; /// API: pro/Proof::verify_signature /// @@ -162,7 +145,7 @@ class ProProof { /// /// Outputs: /// - `bool` - True if the given key was the signatory of the proof, false otherwise - bool verify_signature(const std::span& verify_pubkey) const; + bool verify_signature(std::span verify_pubkey) const; /// API: pro/Proof::verify_message /// @@ -177,7 +160,7 @@ class ProProof { /// /// Outputs: /// - `bool` - True if the message was signed by the embedded `rotating_pubkey` false otherwise. - bool verify_message(std::span sig, const std::span msg) const; + bool verify_message(std::span sig, std::span msg) const; /// API: pro/Proof::is_active /// @@ -190,7 +173,7 @@ class ProProof { /// /// Outputs: /// - `bool` - True if proof is active (i.e. has not expired), false otherwise. - bool is_active(sys_seconds unix_ts) const; + bool is_active(std::chrono::sys_seconds unix_ts) const; /// API: pro/Proof::status /// @@ -217,17 +200,17 @@ class ProProof { /// not set then this function can never return `ProStatus::InvalidUserSig` from the set of /// possible enum values. Otherwise this funtion can return all possible values. ProStatus status( - std::span verify_pubkey, - sys_seconds unix_ts, - std::optional> user_sig = std::nullopt, - std::span signed_msg = {}) const; + std::span verify_pubkey, + std::chrono::sys_seconds unix_ts, + std::optional> user_sig = std::nullopt, + std::span signed_msg = {}) const; /// API: pro/Proof::signed_message /// /// Build the exact byte string that the backend signs to produce this proof's `sig`, and that /// verification reconstructs to check it (pro-wire-protocol.md §2, per §1.1). The message is /// Ed25519-signed directly — there is no pre-hash. - std::vector signed_message() const; + std::vector signed_message() const; /// API: pro/Proof::rotating_seed /// @@ -249,8 +232,8 @@ class ProProof { /// /// Outputs: /// - The 32-byte rotating seed (secret; zeroed on destruction). - static cleared_uc32 rotating_seed( - std::span master_seed, std::chrono::sys_seconds now); + static cleared_b32 rotating_seed( + std::span master_seed, std::chrono::sys_seconds now); bool operator==(const ProProof& other) const { return revocation_tag == other.revocation_tag && rotating_pubkey == other.rotating_pubkey && @@ -265,61 +248,68 @@ enum class ProFeaturesForMsgStatus { ExceedsCharacterLimit = SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_EXCEEDS_CHARACTER_LIMIT, }; -struct ProProfileBitset { - uint64_t data; - void set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features); - void unset(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features); - bool is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) const; +// Session Pro profile feature flags. The enumerator values are the single-bit masks (`1 << +// position`) and are the source of truth for these feature bits; the C API re-exposes the same +// values as `extern const uint64_t SESSION_PROTOCOL_PRO_PROFILE_FEATURE_*` constants. Combine and +// test them with the bitwise operators defined below. +enum class ProProfileFlags : uint64_t { + None = 0, + ProBadge = 1ull << 0, + AnimatedAvatar = 1ull << 1, }; -struct ProMessageBitset { - uint64_t data; - void set(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features); - void unset(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features); - bool is_set(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) const; +// Session Pro message feature flags (see ProProfileFlags). +enum class ProMessageFlags : uint64_t { + None = 0, + CharLimit10k = 1ull << 0, }; +namespace detail { + template + inline constexpr bool is_pro_flags = false; + template <> + inline constexpr bool is_pro_flags = true; + template <> + inline constexpr bool is_pro_flags = true; + + template + concept ProFlags = is_pro_flags; +} // namespace detail + +// Bitwise algebra for the Pro feature flag sets above. Defined once for every flag enum via the +// `ProFlags` concept so the two types can't be accidentally mixed and so the logic lives in a +// single place. +template +constexpr E operator|(E a, E b) { + return static_cast(static_cast(a) | static_cast(b)); +} +template +constexpr E& operator|=(E& a, E b) { + return a = a | b; +} +template +constexpr E operator&(E a, E b) { + return static_cast(static_cast(a) & static_cast(b)); +} +template +constexpr E& operator&=(E& a, E b) { + return a = a & b; +} +template +constexpr E operator~(E a) { + return static_cast(~static_cast(a)); +} + +// True if every bit set in `flag` is also set in `flags` (i.e. `flags` contains `flag`). +template +constexpr bool contains(E flags, E flag) { + return (flags & flag) == flag; +} + struct ProFeaturesForMsg { ProFeaturesForMsgStatus status; std::string_view error; - ProMessageBitset bitset; -}; - -enum class DestinationType { - SyncOr1o1 = SESSION_PROTOCOL_DESTINATION_TYPE_SYNC_OR_1O1, - /// Both legacy and non-legacy groups are to be identified as `Group`. A non-legacy - /// group is detected by the (0x03) prefix byte on the given `dest_group_pubkey` specified in - /// Destination. - Group = SESSION_PROTOCOL_DESTINATION_TYPE_GROUP, - CommunityInbox = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX, - Community = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY, -}; - -struct Destination { - DestinationType type; - - // Optional rotating Session Pro Ed25519 private key to sign the message with on behalf of the - // caller. The Session Pro signature must _not_ be set in the plaintext content passed into the - // encoding function. - std::span pro_rotating_ed25519_privkey; - - // The timestamp to assign to the message envelope - std::chrono::milliseconds sent_timestamp_ms; - - // When type => (CommunityInbox || SyncMessage || Contact): set to the recipient's Session - // public key - array_uc33 recipient_pubkey; - - // When type => CommunityInbox: set this pubkey to the server's key - array_uc32 community_inbox_server_pubkey; - - // When type => Group: set to the group public keys for a 0x03 prefix (e.g. groups v2) - // `group_pubkey` to encrypt the message for. - array_uc33 group_ed25519_pubkey; - - // When type => Group: Set the encryption key of the group for groups v2 messages. Typically - // the latest key for the group, e.g: `Keys::group_enc_key` or `groups_keys_group_enc_key` - cleared_uc32 group_enc_key; + ProMessageFlags flags; }; struct Envelope { @@ -329,12 +319,12 @@ struct Envelope { // Optional fields. These fields are set if the appropriate flag has been set in `flags` // otherwise the corresponding values are to be ignored and those fields will be // zero-initialised. - array_uc33 source; + b33 source; uint32_t source_device; uint64_t server_timestamp; // Signature by the sending client's rotating key - array_uc64 pro_sig; + b64 pro_sig; }; struct DecodedPro { @@ -342,8 +332,8 @@ struct DecodedPro { // Session Pro proof that was embedded in the envelope, this is always populated irrespective of // the status but the validity of the contents should be verified by checking `status` ProProof proof; - ProMessageBitset msg_bitset; - ProProfileBitset profile_bitset; + ProMessageFlags msg_flags; + ProProfileFlags profile_flags; }; struct DecodedEnvelope { @@ -351,16 +341,16 @@ struct DecodedEnvelope { Envelope envelope; // Decoded envelope content into plaintext with padding stripped - std::vector content_plaintext; + std::vector content_plaintext; // Sender public key extracted from the encrypted content payload. This is not set if the // envelope was a groups v2 envelope where the envelope was encrypted and only the x25519 pubkey // was available. - array_uc32 sender_ed25519_pubkey; + b32 sender_ed25519_pubkey; // The x25519 pubkey, always populated on successful parse. Either it's present from decrypting // a Groups v2 envelope or it's re-derived from the Ed25519 pubkey. - array_uc32 sender_x25519_pubkey; + b32 sender_x25519_pubkey; // Set if the envelope included a pro payload. The caller must check the status to determine if // the embedded pro data/proof was valid, invalid or whether or not the proof has expired. @@ -375,46 +365,20 @@ struct DecodedCommunityMessage { std::optional envelope; // The protobuf encoded `Content` with padding stripped - std::vector content_plaintext; + std::vector content_plaintext; // The signature if it was present in the payload. If the envelope is set and the envelope has // the pro signature flag set, then this signature was extracted from the envelope. When the // signature is sourced from the envelope, the envelope's `pro_sig` field is also set to the // same signature as this instance for consistency. Otherwise the signature, if set was // extracted from the community-exclusive pro signature field in the content message. - std::optional pro_sig; + std::optional pro_sig; // Set if the envelope included a pro payload. The caller must check the status to determine if // the embedded pro data/proof was valid, invalid or whether or not the proof has expired. std::optional pro; }; -struct DecodeEnvelopeKey { - // Set the key to decrypt the envelope. If this key is set then it's assumed that the envelope - // payload is encrypted (e.g. groups v2) and that the contents are unencrypted. If this key is - // not set the it's assumed the envelope is not encrypted but the contents are encrypted (e.g.: - // 1o1 or legacy group). - std::optional> group_ed25519_pubkey; - - // List of libsodium-style secret key to decrypt the envelope from. Can also be passed as a 32 - // byte secret key. The public key component is not used. - // - // If the `group_ed25519_pubkey` is set then a list of keys is accepted to attempt to decrypt - // the envelope. For envelopes generated by a group message, we assume that the envelope is - // encrypted and must be decrypted by the group keys associated with it (of which there may be - // many candidate keys depending on how many times the group has been rekeyed). It's recommended - // to pass `Keys::group_keys()` or in the C API use the `groups_keys_size` and - // `group_keys_get_keys` combo to retrieve the keys to attempt to use to decrypt this message. - // - // If `group_ed25519_pubkey` is _not_ set then this function assumes the envelope is unencrypted - // but the content is encrypted (e.g.: 1o1 and legacy group messages). The function will attempt - // to decrypt the envelope's contents with the given keys. Typically in these cases you will - // pass exactly 1 ed25519 private key for decryption but this function makes no pre - // existing assumptions on the number of keys and will attempt all given keys specified - // regardless until it finds one that successfully decrypts the envelope contents. - std::span> decrypt_keys; -}; - /// API: session_protocol/pro_features_for_message /// /// Determine the Pro features required for a conversation message of a given length. @@ -427,7 +391,7 @@ struct DecodeEnvelopeKey { /// - `status` -- Success, or ExceedsCharacterLimit if the message is over the maximum limit. When /// not Success, only `error` is meaningful. /// - `error` -- On a non-Success `status`, a read-only description of the failure; empty otherwise. -/// - `bitset` -- Feature flags suitable for writing directly into the protobuf +/// - `flags` -- Feature flags suitable for writing directly into the protobuf /// `ProMessage.messageFeatures` ProFeaturesForMsg pro_features_for_message(size_t codepoint_count); @@ -435,58 +399,51 @@ ProFeaturesForMsg pro_features_for_message(size_t codepoint_count); /// /// Pad a message to the required alignment for 1o1/community messages (160 bytes) including space /// for the padding-terminating byte. -std::vector pad_message(std::span payload); +std::vector pad_message(std::span payload); -/// API: session_protocol/encode_for_1o1 +/// API: session_protocol/encode_dm_v1 /// -/// Encode a plaintext message for a one-on-one (1o1) conversation or sync message in the Session -/// Protocol. This function wraps the plaintext in the necessary structures and encrypts it for -/// transmission to a single recipient. -/// -/// This is a high-level convenience function that internally calls encode_for_destination with -/// the appropriate Destination configuration for a 1o1 or sync message. +/// Encode a plaintext "v1" message for a one-on-one conversation message (either text message, or +/// conversation metadata) in the Session Protocol. This function wraps the plaintext in the +/// necessary structures and encrypts it for transmission to a single recipient. /// /// This function throws if any input argument is invalid (e.g., incorrect key sizes). /// /// Inputs: -/// - plaintext -- The protobuf serialized payload containing the Content to be encrypted. Must -/// not be already encrypted and must not be padded. -/// - ed25519_privkey -- The sender's libsodium-style secret key (64 bytes). Can also be passed as -/// a 32-byte seed. Used to encrypt the plaintext. -/// - sent_timestamp -- The timestamp to assign to the message envelope, in milliseconds. This -/// should match the protobuf encoded Content's `sigtimestamp` in the given `plaintext`. -/// - recipient_pubkey -- The recipient's Session public key (33 bytes). -/// - pro_rotating_ed25519_privkey -- Optional libsodium-style secret key (64 bytes) that is the -/// secret component of the user's Session Pro Proof `rotating_pubkey`. This key is authorised to -/// entitle the message with Pro features by signing it. Can also be passed as a 32-byte seed. -/// Pass in the empty span to opt-out of Pro feature entitlement. +/// - plaintext -- The protobuf serialized Content plaintext, unpadded payload of the message. +/// - ed25519_privkey -- The sender's Ed25519 private key; accepts a 32-byte seed or 64-byte +/// libsodium "secret". Used to identify the sender and sign the payload. +/// - sent_timestamp -- The timestamp to assign to the message envelope, in unix epoch milliseconds. +/// This must match the protobuf encoded Content's `sigTimestamp` in the given `plaintext`. +/// - recipient_pubkey -- The recipient's Session ID (33 bytes: 0x05 prefix + X25519 pubkey). +/// - pro_rotating_ed25519_privkey -- Optional libsodium-style secret key (64 bytes) or seed (32 +/// bytes) of the user's Session Pro Proof `rotating_pubkey`. This key is authorised to entitle +/// the message with Pro features by signing it. Can also be passed as a 32-byte seed. Omit +/// (or pass std::nullopt) to opt-out of Pro feature entitlement. /// /// Outputs: -/// - Encryption result for the plaintext. The retured payload is suitable for sending on the wire -/// (i.e: it has been protobuf encoded/wrapped if necessary). -std::vector encode_for_1o1( - std::span plaintext, - std::span ed25519_privkey, - std::chrono::milliseconds sent_timestamp, - const array_uc33& recipient_pubkey, - std::optional> pro_rotating_ed25519_privkey); +/// - Encrypted, encoded payload, with all required protobuf encoding and wrapping. +std::vector encode_dm_v1( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, + sys_ms sent_timestamp, + std::span recipient_pubkey, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey = std::nullopt); /// API: session_protocol/encode_for_community_inbox /// -/// Encode a plaintext message for a community inbox in the Session Protocol. This function wraps -/// the plaintext in the necessary structures and encrypts it for transmission to a community inbox -/// server. -/// -/// This is a high-level convenience function that internally calls encode_for_destination with -/// the appropriate Destination configuration for a community inbox message. +/// Encode a plaintext message for a community-handled direct message in the Session Protocol. Such +/// DMs are used to initiate contact with blinded users without needing to expose their Session ID +/// unless they accept the contact. This function wraps the plaintext in the necessary structures +/// and encrypts it for transmission to a community inbox server. /// /// This function throws if any input argument is invalid (e.g., incorrect key sizes). /// /// Inputs: /// - plaintext -- The protobuf serialized payload containing the Content to be encrypted. Must /// not be already encrypted and must not be padded. -/// - ed25519_privkey -- The sender's libsodium-style secret key (64 bytes). Can also be passed as -/// a 32-byte seed. Used to encrypt the plaintext. +/// - ed25519_privkey -- The sender's Ed25519 private key; accepts a 32-byte seed or 64-byte +/// libsodium key. Used to encrypt the plaintext. /// - recipient_pubkey -- The recipient's Session public key (33 bytes). /// - community_pubkey -- The community inbox server's public key (32 bytes). /// - pro_rotating_ed25519_privkey -- Optional libsodium-style secret key (64 bytes) that is the @@ -497,19 +454,19 @@ std::vector encode_for_1o1( /// Outputs: /// - Encryption result for the plaintext. The retured payload is suitable for sending on the wire /// (i.e: it has been protobuf encoded/wrapped if necessary). -std::vector encode_for_community_inbox( - std::span plaintext, - std::span ed25519_privkey, - const array_uc33& recipient_pubkey, - const array_uc32& community_pubkey, - std::optional> pro_rotating_ed25519_privkey); +std::vector encode_for_community_inbox( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span community_pubkey, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey); -/// API: session_protocol/encode_for_community +/// API: session_protocol/encode_community_message /// -/// Encode a plaintext `Content` message for a community in the Session Protocol. This function -/// encodes Session Pro metadata including generating and embedding the Session Pro signature, when -/// given a Session Pro rotating Ed25519 key into the final payload suitable for transmission on the -/// wire. +/// Encode a plaintext `Content` message for sending to a community in the Session Protocol. This +/// function encodes Session Pro metadata including generating and embedding the Session Pro +/// signature, when given a Session Pro rotating Ed25519 key into the final payload suitable for +/// transmission on the wire. /// /// This function throws if any input argument is invalid (e.g., incorrect key sizes). It also /// throws if the pro signature is already set in the plaintext `Content` or the `plaintext` cannot @@ -526,27 +483,24 @@ std::vector encode_for_community_inbox( /// Outputs: /// - Encryption result for the plaintext. The retured payload is suitable for sending on the wire /// (i.e: it has been protobuf encoded/wrapped if necessary). -std::vector encode_for_community( - std::span plaintext, - std::optional> pro_rotating_ed25519_privkey); +std::vector encode_for_community( + std::span plaintext, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey); /// API: session_protocol/encode_for_group /// /// Encode a plaintext message for a group in the Session Protocol. This function wraps the /// plaintext in the necessary structures and encrypts it for transmission to a group, using the -/// group's encryption key. Only v2 groups, (0x03) prefixed keys are supported. Passing a legacy -/// group (0x05) prefixed key will cause the function to throw. -/// -/// This is a high-level convenience function that internally calls encode_for_destination with -/// the appropriate Destination configuration for a group message. +/// group's encryption key. Only v2 groups (with 0x03 prefixed keys) are supported. Passing a legacy +/// group (0x05 prefix) will cause the function to throw. /// /// This function throws if any input argument is invalid (e.g., incorrect key sizes). /// /// Inputs: /// - plaintext -- The protobuf serialized payload containing the Content to be encrypted. Must /// not be already encrypted and must not be padded. -/// - ed25519_privkey -- The sender's libsodium-style secret key (64 bytes). Can also be passed as -/// a 32-byte seed. Used to encrypt the plaintext. +/// - ed25519_privkey -- The sender's Ed25519 private key; accepts a 32-byte seed or 64-byte +/// libsodium key. Used to encrypt the plaintext. /// - sent_timestamp -- The timestamp to assign to the message envelope, in milliseconds. /// - group_ed25519_pubkey -- The group's public key (33 bytes) for encryption with a 0x03 prefix /// - group_enc_key -- The group's encryption key (32 bytes) for groups v2 messages, typically the @@ -559,55 +513,20 @@ std::vector encode_for_community( /// Outputs: /// - Encryption result for the plaintext. The retured payload is suitable for sending on the wire /// (i.e: it has been protobuf encoded/wrapped if necessary). -std::vector encode_for_group( - std::span plaintext, - std::span ed25519_privkey, +std::vector encode_for_group( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, std::chrono::milliseconds sent_timestamp, - const array_uc33& group_ed25519_pubkey, - const cleared_uc32& group_enc_key, - std::optional> pro_rotating_ed25519_privkey); - -/// API: session_protocol/encode_for_destination -/// -/// Given an unencrypted plaintext representation of the content (i.e.: protobuf encoded stream of -/// `Content`), encrypt and/or wrap the plaintext in the necessary structures for transmission on -/// the Session Protocol. -/// -/// Calling this function requires filling out the options in the `Destination` struct with the -/// appropriate values for the desired destination. Check the annotation on `Destination` for more -/// information on how to fill this struct. Alternatively, there are higher level functions, encrypt -/// for 1o1, group and community functions which thunk into this low-level function for convenience. -/// -/// This function throws if the API is misused (i.e.: A field was not set, but was required to be -/// set for the given destination and namespace. For example the group keys not being set -/// when sending to a group prefixed [0x3] key in a group) -/// but otherwise returns a struct with values. -/// -/// Inputs: -/// - `plaintext` -- the protobuf serialised payload containing the protobuf encoded stream, -/// `Content`. It must not be already be encrypted and must not be padded. -/// - `ed25519_privkey` -- the libsodium-style secret key of the sender, 64 bytes. Can also be -/// passed as a 32-byte seed. Used to encrypt the plaintext. -/// - `dest` -- the extra metadata indicating the destination of the message and the necessary data -/// to encrypt a message for that destination. -/// -/// Outputs: -/// - Encryption result for the plaintext. The retured payload is suitable for sending on the wire -/// (i.e: it has been protobuf encoded/wrapped if necessary). -std::vector encode_for_destination( - std::span plaintext, - std::span ed25519_privkey, - const Destination& dest); + std::span group_ed25519_pubkey, + std::span group_enc_key, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey); -/// API: session_protocol/decode_envelope +/// API: session_protocol/decode_dm_envelope /// -/// Given an envelope payload (i.e.: protobuf encoded stream of `WebsocketRequestMessage` which -/// wraps an `Envelope` for 1o1 messages/sync messages, or `Envelope` encrypted using a Groups v2 -/// key) parse (or decrypt) the envelope and return the envelope content decrypted if necessary. -/// -/// A groups v2 envelope will get decrypted with the group keys. A non-groups v2 envelope will get -/// decrypted with the specified Ed25519 private key in the `keys` object. Only one of these keys -/// need to be set depending on the type of envelope payload passed into the function. +/// Decode a 1-on-1 (or legacy group) envelope: a WebSocket-wrapped protobuf `Envelope` whose inner +/// `Content` is encrypted with the Session protocol (Ed25519 DH). Parse the envelope, decrypt the +/// content, and return the plaintext along with any Session Pro metadata. (Groups v2 envelopes, +/// where the envelope itself is encrypted with a group key, are handled by decode_group_envelope.) /// /// If the message does not use Session Pro features, the `pro` object will be set to nil. Otherwise /// the pro fields will be populated with data about the Session Pro proof embedded in the envelope @@ -625,40 +544,45 @@ std::vector encode_for_destination( /// field to verify if the Session Pro was present and/or valid or invalid. /// /// Inputs: -/// - `keys` -- the keys to decrypt either the envelope or the envelope contents. Groups v2 -/// envelopes where the envelope is encrypted must set the group key. Envelopes with an encrypted -/// content must set the the libsodium-style secret key of the receiver, 64 bytes. Can also be -/// passed as a 32-byte seed. -/// -/// If a group decryption key is specified, the recipient key is ignored and vice versa. Only one -/// of the keys should be set depending on the type of envelope. -/// -/// - `envelope_payload` -- the envelope payload either encrypted (groups v2 style) or unencrypted -/// (1o1 or legacy groups). +/// - `ed25519_privkey` -- the receiver's Ed25519 private key used to decrypt the envelope content; +/// a libsodium-style 64-byte secret key, or a 32-byte seed. +/// - `envelope_payload` -- the WebSocket-wrapped envelope payload (the inner content is encrypted). /// - `pro_backend_pubkey` -- the Session Pro backend public key to verify the signature embedded in /// the proof, validating whether or not the attached proof was indeed issued by an authorised /// issuer /// /// Outputs: -/// - `envelope` -- Envelope structure that was decrypted/parsed from the `envelope_plaintext` +/// - `envelope` -- Envelope structure that was parsed from the payload /// - `content_plaintext` -- Decrypted contents of the envelope structure. This is the protobuf /// encoded stream that can be parsed into a protobuf `Content` structure. /// - `sender_ed25519_pubkey` -- The sender's ed25519 public key embedded in the encrypted payload. -/// This is only set for session message envelopes. Groups envelopes only embed the sender's -/// x25519 public key in which case this field is set to the zero public key. -/// - `sender_x25519_pubkey` -- The sender's x25519 public key. It's always set on successful -/// decryption either by extracting the key from the encrypted groups envelope, or, by deriving -/// the x25519 key from the sender's ed25519 key in the case of a session message envelope. -/// - `pro` -- Optional object that is set if there was pro metadata associatd with the envelope, if -/// any. The `status` field in the decrypted pro object should be used to determine whether or not +/// - `sender_x25519_pubkey` -- The sender's x25519 public key, derived from the sender's ed25519 +/// key. +/// - `pro` -- Optional object that is set if there was pro metadata associated with the envelope, +/// if +/// any. The `status` field in the decoded pro object should be used to determine whether or not /// the caller can respect the contents of the `proof` and `features`. /// -/// If the `status` is set to valid the the caller can proceed with entitling the envelope with +/// If the `status` is set to valid the caller can proceed with entitling the envelope with /// access to pro features if it's using any. -DecodedEnvelope decode_envelope( - const DecodeEnvelopeKey& keys, - std::span envelope_payload, - const array_uc32& pro_backend_pubkey); +DecodedEnvelope decode_dm_envelope( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span envelope_payload, + std::span pro_backend_pubkey); + +/// Decodes a groups v2 envelope. The envelope payload is encrypted with a group symmetric key +/// and decrypted via `decrypt_group_message`. The inner content is plaintext. +/// +/// `group_keys` is a list of recent symmetric group encryption keys to try; multiple keys are +/// needed because the key rotates periodically, and retrieved messages may still be encrypted +/// with a pre-rotation key if they were sent before the rotation occurred. +/// +/// Throws on parse or decryption failure. +DecodedEnvelope decode_group_envelope( + std::span> group_keys, + std::span group_ed25519_pubkey, + std::span envelope_payload, + std::span pro_backend_pubkey); /// API: session_protocol/decode_for_community /// @@ -690,7 +614,8 @@ DecodedEnvelope decode_envelope( /// If the `status` is set to valid the the caller can proceed with entitling the envelope with /// access to pro features if it's using any. DecodedCommunityMessage decode_for_community( - std::span content_or_envelope_payload, - sys_seconds unix_ts, - const array_uc32& pro_backend_pubkey); + std::span content_or_envelope_payload, + std::chrono::sys_seconds unix_ts, + std::span pro_backend_pubkey); + } // namespace session diff --git a/include/session/sodium_array.hpp b/include/session/sodium_array.hpp index a0a52de0a..841799e8f 100644 --- a/include/session/sodium_array.hpp +++ b/include/session/sodium_array.hpp @@ -11,48 +11,6 @@ void sodium_buffer_deallocate(void* p); // Calls sodium_memzero to zero a buffer void sodium_zero_buffer(void* ptr, size_t size); -// Works similarly to a unique_ptr, but allocations and free go via libsodium (which is slower, but -// more secure for sensitive data). -template -struct sodium_ptr { - private: - T* x; - - public: - sodium_ptr() : x{nullptr} {} - sodium_ptr(std::nullptr_t) : sodium_ptr{} {} - ~sodium_ptr() { reset(x); } - - // Allocates and constructs a new `T` in-place, forwarding any given arguments to the `T` - // constructor. If this sodium_ptr already has an object, `reset()` is first called implicitly - // to destruct and deallocate the existing object. - template - T& emplace(Args&&... args) { - if (x) - reset(); - x = static_cast(sodium_buffer_allocate(sizeof(T))); - new (x) T(std::forward(args)...); - return *x; - } - - void reset() { - if (x) { - x->~T(); - sodium_buffer_deallocate(x); - x = nullptr; - } - } - void operator=(std::nullptr_t) { reset(); } - - T& operator*() { return *x; } - const T& operator*() const { return *x; } - - T* operator->() { return x; } - const T* operator->() const { return x; } - - explicit operator bool() const { return x != nullptr; } -}; - // Wrapper around a type that uses `sodium_memzero` to zero the container on destruction; may only // be used with trivially destructible types. template >> @@ -62,145 +20,23 @@ struct sodium_cleared : T { ~sodium_cleared() { sodium_zero_buffer(this, sizeof(*this)); } }; -template -using cleared_array = sodium_cleared>; - -using cleared_uc32 = cleared_array<32>; -using cleared_uc64 = cleared_array<64>; +template +struct cleared_array : sodium_cleared> { + using sodium_cleared>::sodium_cleared; -// This is an optional (i.e. can be empty) fixed-size (at construction) buffer that does allocation -// and freeing via libsodium. It is slower and heavier than a regular allocation type but takes -// extra precautions, intended for storing sensitive values. -template -struct sodium_array { - private: - T* buf; - size_t len; - - public: - // Default constructor: makes an empty object (that is, has no buffer and has `.size()` of 0). - sodium_array() : buf{nullptr}, len{0} {} - - // Constructs an array with a given size, default-constructing the individual elements. - template >> - explicit sodium_array(size_t length) : - buf{length == 0 ? nullptr - : static_cast(sodium_buffer_allocate(length * sizeof(T)))}, - len{0} { - - if (length > 0) { - if constexpr (std::is_trivial_v) { - std::memset(buf, 0, length * sizeof(T)); - len = length; - } else if constexpr (std::is_nothrow_default_constructible_v) { - for (; len < length; len++) - new (buf[len]) T(); - } else { - try { - for (; len < length; len++) - new (buf[len]) T(); - } catch (...) { - reset(); - throw; - } - } - } + // Provide implicit conversion to fixed extent span because otherwise span's built-in is dynamic + // extent (because span uses CTAD which detects std::array but not our subclass). + operator std::span() { return std::span{static_cast&>(*this)}; } + operator std::span() const { + return std::span{static_cast&>(*this)}; } - - ~sodium_array() { reset(); } - - // Moveable: ownership is transferred to the new object and the old object becomes empty. - sodium_array(sodium_array&& other) : buf{other.buf}, len{other.len} { - other.buf = nullptr; - other.len = 0; - } - sodium_array& operator=(sodium_array&& other) { - sodium_buffer_deallocate(buf); - buf = other.buf; - len = other.len; - other.buf = nullptr; - other.len = 0; - } - - // Non-copyable - sodium_array(const sodium_array&) = delete; - sodium_array& operator=(const sodium_array&) = delete; - - // Destroys the held array; after destroying elements the allocated space is overwritten with - // 0s before being deallocated. - void reset() { - if (buf) { - if constexpr (!std::is_trivially_destructible_v) - while (len > 0) - buf[--len].~T(); - - sodium_buffer_deallocate(buf); - } - buf = nullptr; - len = 0; - } - - // Calls reset() to destroy the current value (if any) and then allocates a new - // default-constructed one of the given size. - template >> - void reset(size_t length) { - reset(); - if (length > 0) { - buf = static_cast(sodium_buffer_allocate(length * sizeof(T))); - if constexpr (std::is_trivial_v) { - std::memset(buf, 0, length * sizeof(T)); - len = length; - } else { - for (; len < length; len++) - new (buf[len]) T(); - } - } - } - - // Loads the array from a pointer and size; this first resets a value (if present), allocates a - // new array of the given size, the copies the given value(s) into the new buffer. T must be - // copyable. This is *not* safe to use if `buf` points into the currently allocated data. - template >> - void load(const T* data, size_t length) { - reset(length); - if (length == 0) - return; - - if constexpr (std::is_trivially_copyable_v) - std::memcpy(buf, data, sizeof(T) * length); - else - for (; len < length; len++) - new (buf[len]) T(data[len]); - } - - const T& operator[](size_t i) const { - assert(i < len); - return buf[i]; - } - T& operator[](size_t i) { - assert(i < len); - return buf[i]; - } - - T* data() { return buf; } - const T* data() const { return buf; } - - size_t size() const { return len; } - bool empty() const { return len == 0; } - explicit operator bool() const { return !empty(); } - - T* begin() { return buf; } - const T* begin() const { return buf; } - T* end() { return buf + len; } - const T* end() const { return buf + len; } - - using difference_type = ptrdiff_t; - using value_type = T; - using pointer = value_type*; - using reference = value_type&; - using iterator_category = std::random_access_iterator_tag; }; +template +using cleared_bytes = cleared_array; +using cleared_b32 = cleared_bytes<32>; +using cleared_b64 = cleared_bytes<64>; + // sodium Allocator wrapper; this allocates/frees via libsodium, which is designed for dealing with // sensitive data. It is as a result slower and has more overhead than a standard allocator and // intended for use with a container (such as std::vector) when storing keys. @@ -228,4 +64,28 @@ struct sodium_allocator { template using sodium_vector = std::vector>; +// Like std::allocator but zeros memory before freeing. Lighter weight than sodium_allocator +// (uses regular heap allocation) but still ensures sensitive data is wiped on deallocation. +template +struct clearing_allocator { + using value_type = T; + + [[nodiscard]] static T* allocate(std::size_t n) { return std::allocator{}.allocate(n); } + + static void deallocate(T* p, std::size_t n) { + sodium_zero_buffer(p, n * sizeof(T)); + std::allocator{}.deallocate(p, n); + } + + template + bool operator==(const clearing_allocator&) const noexcept { + return true; + } +}; + +/// Vector that zeros its buffer on deallocation (including when resizing). Lighter weight +/// than sodium_vector but still suitable for short-lived sensitive data. +template +using cleared_vector = std::vector>; + } // namespace session diff --git a/include/session/types.h b/include/session/types.h index 9e14a90aa..cd87d28e6 100644 --- a/include/session/types.h +++ b/include/session/types.h @@ -1,7 +1,6 @@ #pragma once #include -#include #ifdef __cplusplus extern "C" { @@ -17,43 +16,25 @@ extern "C" { /// C friendly buffer structure that is a pointer and length to a span of bytes. typedef struct span_u8 span_u8; struct span_u8 { - uint8_t* data; + unsigned char* data; size_t size; }; -typedef struct bytes32 bytes32; -struct bytes32 { - uint8_t data[32]; +typedef struct cbytes32 cbytes32; +struct cbytes32 { + unsigned char data[32]; }; -typedef struct bytes33 bytes33; -struct bytes33 { - uint8_t data[33]; +typedef struct cbytes33 cbytes33; +struct cbytes33 { + unsigned char data[33]; }; -typedef struct bytes64 bytes64; -struct bytes64 { - uint8_t data[64]; +typedef struct cbytes64 cbytes64; +struct cbytes64 { + unsigned char data[64]; }; -/// A wrapper around snprintf that fixes a common bug in the value the printing function returns -/// when a buffer is passed in. Irrespective of whether a buffer is passed in, snprintf is defined -/// to return: -/// -/// number of characters (not including the terminating null character) which would have been -/// written to buffer if bufsz was ignored -/// -/// This means if the user passes in a buffer to small, the return value is always the amount of -/// bytes required. This means the user always has to calculate the number of bytes written as: -/// -/// size_t bytes_written = min(snprintf(buffer, size, ...), size); -/// -/// This is error prone. This function does the `min(...)` for you so that this function -/// _always_ calculates the actual number of bytes written (not including the null-terminator). If a -/// NULL is passed in then this function returns the number of bytes actually needed to write the -/// entire string (as per normal snprintf behaviour). -int snprintf_clamped(char* buffer, size_t size, char const* fmt, ...); - #ifdef __cplusplus } #endif diff --git a/include/session/types.hpp b/include/session/types.hpp index 52500d8e0..1ea5e418b 100644 --- a/include/session/types.hpp +++ b/include/session/types.hpp @@ -1,8 +1,8 @@ #pragma once -#include #include #include +#include #include "types.h" @@ -11,10 +11,6 @@ namespace session { template static constexpr bool is_one_of = (std::is_same_v || ...); -using array_uc32 = std::array; -using array_uc33 = std::array; -using array_uc64 = std::array; - enum class SessionIDPrefix { standard = 0, group = 0x3, @@ -37,5 +33,4 @@ span_u8 span_u8_alloc_or_throw(size_t size); /// this function throws a runtime exception. The `data` pointer is span must be freed once the span /// is no longer needed. span_u8 span_u8_copy_or_throw(const void* data, size_t size); - } // namespace session diff --git a/include/session/util.hpp b/include/session/util.hpp index d818fdc28..7ebf5e386 100644 --- a/include/session/util.hpp +++ b/include/session/util.hpp @@ -30,29 +30,34 @@ namespace session { using namespace oxenc; // Helper functions to convert to/from spans -template +template inline std::span as_span(std::span sp) { return std::span{reinterpret_cast(sp.data()), sp.size()}; } -template +template inline std::span as_span(std::span sp) { return std::span{reinterpret_cast(sp.data()), sp.size()}; } -template +template inline std::span to_span(const T& c) { return {reinterpret_cast(c.data()), c.size()}; } -template +template inline std::span to_span(const char (&literal)[N]) { return {reinterpret_cast(literal), N - 1}; } -template - requires(!oxenc::bt_input_string) -inline std::span to_span(const Container& c) { - return {reinterpret_cast(c.data()), c.size()}; +template + requires( + std::convertible_to< + const Container&, + std::span> && + !oxenc::bt_input_string && oxenc::basic_char) +inline auto to_span(const Container& c) { + constexpr size_t Extent{decltype(std::span{c})::extent}; + return std::span{reinterpret_cast(c.data()), c.size()}; } // Helper functions to convert container types @@ -63,34 +68,32 @@ inline OutContainer convert(const InContainer& in) { return OutContainer(begin, begin + in.size()); } -template +template inline std::vector to_vector(std::span sp) { return convert>(sp); } -template +template inline std::vector to_vector(const T& c) { return convert>(to_span(c)); } -template +template inline std::vector to_vector(const std::array& arr) { return convert>(arr); } -template +template requires(!oxenc::bt_input_string) inline std::vector to_vector(const Container& c) { return convert>(to_span(c)); } template -inline std::array to_array(std::span sp) { - std::array result{}; +inline std::array to_array(std::span sp) { + std::array result{}; std::copy_n( - reinterpret_cast(sp.data()), - std::min(N, sp.size()), - result.begin()); + reinterpret_cast(sp.data()), std::min(N, sp.size()), result.begin()); return result; } @@ -108,7 +111,52 @@ inline std::string_view to_string_view(const Container& c) { return {reinterpret_cast(c.data()), static_cast(c.size())}; } -// Helper function to go to/from char pointers to unsigned char pointers: +/// Returns a fixed-extent std::byte span viewing N bytes starting at p. Primarily useful for +/// wrapping C API pointers (e.g. `unsigned char*`) into typed spans without a reinterpret_cast at +/// the call site. A dynamic-size overload taking a length is also provided. A third overload +/// accepts a C array directly and deduces N. +template +inline std::span to_byte_span(const Char* p) { + return std::span(reinterpret_cast(p), N); +} +template +inline std::span to_byte_span(Char* p) { + return std::span(reinterpret_cast(p), N); +} +template +inline std::span to_byte_span(const std::byte* p) { + return std::span(p, N); +} +template +inline std::span to_byte_span(std::byte* p) { + return std::span(p, N); +} +template +inline std::span to_byte_span(const Char* p, size_t n) { + return {reinterpret_cast(p), n}; +} +template +inline std::span to_byte_span(Char* p, size_t n) { + return {reinterpret_cast(p), n}; +} +inline std::span to_byte_span(const std::byte* p, size_t n) { + return {p, n}; +} +inline std::span to_byte_span(std::byte* p, size_t n) { + return {p, n}; +} +// Array overloads: deduce N from a C array, returning a fixed-extent span. +template +inline std::span to_byte_span(const Char (&arr)[N]) { + return std::span(reinterpret_cast(arr), N); +} +template +inline std::span to_byte_span(Char (&arr)[N]) { + return std::span(reinterpret_cast(arr), N); +} + +// Helper functions to reinterpret char-type pointers as unsigned char* or std::byte*. +// These are primarily for passing binary data to/from C APIs. template inline const unsigned char* to_unsigned(const Char* x) { return reinterpret_cast(x); @@ -131,69 +179,31 @@ inline unsigned char* to_unsigned(unsigned char* x) { return x; } -// The same as std::chrono::system_clock::now(), except that it allows you to get it in a different -// precision. E.g. sysclock_now gives a timepoint with seconds precision (aka -// std::chrono::sys_seconds). -template -inline std::chrono::sys_time sysclock_now() { - return std::chrono::floor(std::chrono::system_clock::now()); -} -// Shortcut for sysclock_now(); -inline std::chrono::sys_seconds sysclock_now_s() { - return sysclock_now(); -} -using sys_ms = std::chrono::sys_time; -using sys_seconds = std::chrono::sys_seconds; -// Shortcut for sysclock_now>(); -inline sys_ms sysclock_now_ms() { - return sysclock_now(); -} - -// Returns the duration count of the given duration cast into ToDuration. Example: -// duration_count(30000ms) // returns 30 -// This function requires that the target type is no more precise than d, that is, it will not allow -// you to cast from seconds to milliseconds because such a cast indicates that the sub-second -// precision has already been lost. -template - requires std::is_convertible_v> -constexpr int64_t duration_count(const std::chrono::duration& d) { - return std::chrono::duration_cast(d).count(); -} -// Returns the seconds count of the given duration -template - requires std::is_convertible_v> -constexpr int64_t duration_seconds(const std::chrono::duration& d) { - return duration_count(d); -} -// Returns the milliseconds count of the given duration -template - requires std::is_convertible_v> -constexpr int64_t duration_ms(const std::chrono::duration& d) { - return duration_count(d); -} - -// Returns the time-since-epoch count of the given time point, cast into ToDuration. The given time -// point must be at least as precise as ToDuration, i.e. this will not allow you to cast to a more -// precise time point as that would mean the intended precision has already been lost by an earlier -// cast. -template - requires std::is_convertible_v -constexpr int64_t epoch_count(const std::chrono::time_point& t) { - return duration_count(t.time_since_epoch()); -} -// Returns the seconds-since-epoch count of the given time point. The given time point must be at -// least as precise as seconds. -template - requires std::is_convertible_v -constexpr int64_t epoch_seconds(const std::chrono::time_point& t) { - return duration_seconds(t.time_since_epoch()); -} -// Returns the milliseconds-since-epoch count of the given time point. The given time point must -// have at least milliseconds precision. -template - requires std::is_convertible_v -constexpr int64_t epoch_ms(const std::chrono::time_point& t) { - return duration_ms(t.time_since_epoch()); +template +inline const std::byte* to_bytes(const Char* x) { + return reinterpret_cast(x); +} +template +inline std::byte* to_bytes(Char* x) { + return reinterpret_cast(x); +} +// These do nothing, but having them makes template metaprogramming easier: +inline const std::byte* to_bytes(const std::byte* x) { + return x; +} +inline std::byte* to_bytes(std::byte* x) { + return x; +} + +/// Returns the data pointer of a std::byte span as an `unsigned char*`, for passing to C APIs +/// that expect `unsigned char*`. The const overload returns `const unsigned char*`. +template +inline unsigned char* ucdata(std::span sp) { + return reinterpret_cast(sp.data()); +} +template +inline const unsigned char* ucdata(std::span sp) { + return reinterpret_cast(sp.data()); } /// Returns true if the first string is equal to the second string, compared case-insensitively. @@ -204,15 +214,19 @@ inline bool string_iequal(std::string_view s1, std::string_view s2) { }); } +using b32 = std::array; +using b33 = std::array; +using b64 = std::array; + using uc32 = std::array; using uc33 = std::array; using uc64 = std::array; -/// Takes a container of string-like binary values and returns a vector of unsigned char spans -/// viewing those values. This can be used on a container of any type with a `.data()` and a -/// `.size()` where `.data()` is a one-byte value pointer; std::string, std::string_view, -/// std::vector, std::span, etc. apply, as does std::array -/// of 1-byte char types. +/// Takes a container of string-like binary values and returns a vector of std::byte spans viewing +/// those values. This can be used on a container of any type with a `.data()` and a `.size()` +/// where `.data()` is a one-byte value pointer; std::string, std::string_view, +/// std::vector, std::span, etc. apply, as does std::array of 1-byte +/// char types. /// /// This is useful in various libsession functions that require such a vector. Note that the /// returned vector's views are valid only as the original container remains alive; this is @@ -223,25 +237,25 @@ using uc64 = std::array; /// There are two versions of this: the first takes a generic iterator pair; the second takes a /// single container. template -std::vector> to_view_vector(It begin, It end) { - std::vector> vec; +std::vector> to_view_vector(It begin, It end) { + std::vector> vec; vec.reserve(std::distance(begin, end)); for (; begin != end; ++begin) { if constexpr (std::is_same_v, char*>) // C strings - vec.emplace_back(*begin); + vec.emplace_back(reinterpret_cast(*begin), std::strlen(*begin)); else { static_assert( sizeof(*begin->data()) == 1, "to_view_vector can only be used with containers of string-like types of " "1-byte characters"); - vec.emplace_back(reinterpret_cast(begin->data()), begin->size()); + vec.emplace_back(reinterpret_cast(begin->data()), begin->size()); } } return vec; } template -std::vector> to_view_vector(const Container& c) { +std::vector> to_view_vector(const Container& c) { return to_view_vector(c.begin(), c.end()); } @@ -251,6 +265,9 @@ std::vector> to_view_vector(const Container& c) { /// vector of string_views each viewing one character. If `trim` is true then leading and trailing /// empty values will be suppressed. /// +/// The returned vector always contains at least one element when `trim` is false (even for an empty +/// input string). With `trim` true, an empty input returns an empty vector. +/// /// auto v = split("ab--c----de", "--"); // v is {"ab", "c", "", "de"} /// auto v = split("abc", ""); // v is {"a", "b", "c"} /// auto v = split("abc", "c"); // v is {"ab", ""} @@ -363,15 +380,48 @@ static_assert(std::is_same_v< /// ZSTD-compresses a value. `prefix` can be prepended on the returned value, if needed. Throws on /// serious error. -std::vector zstd_compress( - std::span data, - int level = 1, - std::span prefix = {}); +std::vector zstd_compress( + std::span data, int level = 1, std::span prefix = {}); /// ZSTD-decompresses a value. Returns nullopt if decompression fails. If max_size is non-zero /// then this returns nullopt if the decompressed size would exceed that limit. -std::optional> zstd_decompress( - std::span data, size_t max_size = 0); +std::optional> zstd_decompress( + std::span data, size_t max_size = 0); + +/// Wrapper for formatting byte sizes with SI prefixes via fmt/oxen-logging. Provides a +/// `format_as` friend function discoverable via ADL, so no fmt headers are needed here. +/// Usage: `log::info(cat, "Size: {}", human_size{12345});` => "Size: 12.3 kB" +struct human_size { + int64_t bytes; + friend std::string format_as(human_size s); +}; + +/// NTTP helper struct for the `_bytes` user-defined literal. +template +struct BytesLiteral { + std::byte data[N - 1]; + static constexpr size_t size = N - 1; + consteval BytesLiteral(const char (&s)[N]) { + for (size_t i = 0; i < N - 1; ++i) + data[i] = static_cast(static_cast(s[i])); + } +}; + +inline namespace literals { + + /// User-defined literal that returns a compile-time `std::span` view over the + /// string literal's bytes (excluding the null terminator). Example: + /// + /// using namespace session::literals; // or `using namespace session;` + /// constexpr auto domain = "MyDomainKey"_bytes; + /// + template + consteval std::span operator""_bytes() { + return std::span(Lit.data, Lit.size); + } + +} // namespace literals + } // namespace session #ifndef _WIN32 diff --git a/include/session/xed25519.h b/include/session/xed25519.h index 4963ce89c..d65307287 100644 --- a/include/session/xed25519.h +++ b/include/session/xed25519.h @@ -29,7 +29,7 @@ LIBSESSION_EXPORT bool session_xed25519_verify( /// in a given curve25519 pubkey: this always returns the positive value. You can get the other /// possibility (the negative) by flipping the sign bit, i.e. `returned_pubkey[31] |= 0x80`. /// Returns 0 on success, non-0 on failure. -LIBSESSION_EXPORT bool session_xed25519_pubkey( +LIBSESSION_EXPORT void session_xed25519_pubkey( unsigned char* ed25519_pubkey /* 32-byte output buffer */, const unsigned char* curve25519_pubkey /* 32 bytes */); diff --git a/include/session/xed25519.hpp b/include/session/xed25519.hpp index 4cf85c6ad..e5ed56e3e 100644 --- a/include/session/xed25519.hpp +++ b/include/session/xed25519.hpp @@ -1,27 +1,31 @@ #pragma once #include +#include #include #include #include -#include + +#include "util.hpp" namespace session::xed25519 { /// XEd25519-signs a message given the curve25519 privkey and message. -std::array sign( - std::span curve25519_privkey, std::span msg); +b64 sign(std::span curve25519_privkey, std::span msg); -/// "Softer" version that takes and returns strings of regular chars +/// "Softer" version that takes and returns strings of regular chars. Throws invalid_argument if +/// the privkey is not 32 bytes. std::string sign(std::string_view curve25519_privkey /* 32 bytes */, std::string_view msg); /// Verifies a curve25519 message allegedly signed by the given curve25519 pubkey [[nodiscard]] bool verify( - std::span signature, - std::span curve25519_pubkey, - std::span msg); + std::span signature, + std::span curve25519_pubkey, + std::span msg); -/// "Softer" version that takes strings of regular chars +/// "Softer" version that takes strings of regular chars. Throws invalid_argument if the signature +/// is not 64 bytes or the pubkey is not 32 bytes (a wrong-sized input is a caller bug, not a failed +/// verification, so it is not reported as a false return). [[nodiscard]] bool verify( std::string_view signature /* 64 bytes */, std::string_view curve25519_pubkey /* 32 bytes */, @@ -31,19 +35,20 @@ std::string sign(std::string_view curve25519_privkey /* 32 bytes */, std::string /// however, that there are *two* possible Ed25519 pubkeys that could result in a given curve25519 /// pubkey: this always returns the positive value. You can get the other possibility (the /// negative) by setting the sign bit, i.e. `returned_pubkey[31] |= 0x80`. -std::array pubkey(std::span curve25519_pubkey); +b32 pubkey(std::span curve25519_pubkey) noexcept; -/// "Softer" version that takes/returns strings of regular chars +/// "Softer" version that takes/returns strings of regular chars. Throws invalid_argument if the +/// input is not 32 bytes. std::string pubkey(std::string_view curve25519_pubkey); /// Utility function that provides a constant-time `if (b) f = g;` implementation for byte arrays. template void constant_time_conditional_assign( - std::array& f, const std::array& g, bool b) { - std::array x; + std::array& f, const std::array& g, bool b) { + std::array x; for (size_t i = 0; i < x.size(); i++) x[i] = f[i] ^ g[i]; - unsigned char mask = (unsigned char)(-(signed char)b); + auto mask = static_cast(-(signed char)b); for (size_t i = 0; i < x.size(); i++) x[i] &= mask; for (size_t i = 0; i < x.size(); i++) diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 324b116b8..52c3a6c37 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -4,9 +4,12 @@ libsession_static_bundle(protobuf::libprotobuf-lite) add_library(protos SessionProtos.pb.cc - WebSocketResources.pb.cc) + WebSocketResources.pb.cc + debug_print.cpp) target_include_directories(protos PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(protos PUBLIC protobuf::libprotobuf-lite) +# `common` is for debug_print.cpp, which needs fmt and oxenc to render; PRIVATE because nothing +# about the generated message classes exposes either. +target_link_libraries(protos PUBLIC protobuf::libprotobuf-lite PRIVATE common) set_target_properties( protos PROPERTIES OUTPUT_NAME session-protos @@ -23,16 +26,16 @@ endif() libsession_static_bundle(protos) add_library(libsession::protos ALIAS protos) -export( - TARGETS protos - NAMESPACE libsession:: - FILE libsessionTargets.cmake -) list(APPEND libsession_export_targets protos) set(libsession_export_targets "${libsession_export_targets}" PARENT_SCOPE) +# Regenerates both the protobuf classes and the text dumper built on them. They come from the same +# descriptors and are checked in together, which is what keeps the dumper from falling behind the +# schema -- it has to be generated, because LITE_RUNTIME leaves the classes with no reflection for a +# TextFormat printer to use. Needs python3-protobuf, and only when run. add_custom_target(regen-protobuf - protoc --cpp_out=. SessionProtos.proto WebSocketResources.proto + COMMAND protoc --cpp_out=. SessionProtos.proto WebSocketResources.proto + COMMAND sh -c "protoc --descriptor_set_out=/dev/stdout SessionProtos.proto WebSocketResources.proto | ./gen_debug_print.py ." WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" ) diff --git a/proto/SessionProtos.pb.cc b/proto/SessionProtos.pb.cc index 747f53bbb..e0c6bc4d6 100644 --- a/proto/SessionProtos.pb.cc +++ b/proto/SessionProtos.pb.cc @@ -58,7 +58,8 @@ PROTOBUF_CONSTEXPR UnsendRequest::UnsendRequest( /*decltype(_impl_._has_bits_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_.author_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} - , /*decltype(_impl_.timestamp_)*/uint64_t{0u}} {} + , /*decltype(_impl_.msgtimestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgid_)*/int64_t{0}} {} struct UnsendRequestDefaultTypeInternal { PROTOBUF_CONSTEXPR UnsendRequestDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} @@ -100,7 +101,8 @@ PROTOBUF_CONSTEXPR Content::Content( , /*decltype(_impl_.promessage_)*/nullptr , /*decltype(_impl_.expirationtype_)*/0 , /*decltype(_impl_.expirationtimer_)*/0u - , /*decltype(_impl_.sigtimestamp_)*/uint64_t{0u}} {} + , /*decltype(_impl_.sigtimestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgid_)*/int64_t{0}} {} struct ContentDefaultTypeInternal { PROTOBUF_CONSTEXPR ContentDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} @@ -128,26 +130,14 @@ struct CallMessageDefaultTypeInternal { }; }; PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 CallMessageDefaultTypeInternal _CallMessage_default_instance_; -PROTOBUF_CONSTEXPR KeyPair::KeyPair( - ::_pbi::ConstantInitialized): _impl_{ - /*decltype(_impl_._has_bits_)*/{} - , /*decltype(_impl_._cached_size_)*/{} - , /*decltype(_impl_.publickey_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} - , /*decltype(_impl_.privatekey_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}}} {} -struct KeyPairDefaultTypeInternal { - PROTOBUF_CONSTEXPR KeyPairDefaultTypeInternal() - : _instance(::_pbi::ConstantInitialized{}) {} - ~KeyPairDefaultTypeInternal() {} - union { - KeyPair _instance; - }; -}; -PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 KeyPairDefaultTypeInternal _KeyPair_default_instance_; PROTOBUF_CONSTEXPR DataExtractionNotification::DataExtractionNotification( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_._has_bits_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_.timestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgtimestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgid_)*/int64_t{0} + , /*decltype(_impl_.attindex_)*/0 , /*decltype(_impl_.type_)*/1} {} struct DataExtractionNotificationDefaultTypeInternal { PROTOBUF_CONSTEXPR DataExtractionNotificationDefaultTypeInternal() @@ -198,7 +188,8 @@ PROTOBUF_CONSTEXPR DataMessage_Quote::DataMessage_Quote( , /*decltype(_impl_.attachments_)*/{} , /*decltype(_impl_.author_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.text_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} - , /*decltype(_impl_.id_)*/uint64_t{0u}} {} + , /*decltype(_impl_.msgtimestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgid_)*/int64_t{0}} {} struct DataMessage_QuoteDefaultTypeInternal { PROTOBUF_CONSTEXPR DataMessage_QuoteDefaultTypeInternal() : _instance(::_pbi::ConstantInitialized{}) {} @@ -230,7 +221,8 @@ PROTOBUF_CONSTEXPR DataMessage_Reaction::DataMessage_Reaction( , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_.author_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} , /*decltype(_impl_.emoji_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} - , /*decltype(_impl_.id_)*/uint64_t{0u} + , /*decltype(_impl_.msgtimestamp_)*/uint64_t{0u} + , /*decltype(_impl_.msgid_)*/int64_t{0} , /*decltype(_impl_.action_)*/0} {} struct DataMessage_ReactionDefaultTypeInternal { PROTOBUF_CONSTEXPR DataMessage_ReactionDefaultTypeInternal() @@ -287,6 +279,7 @@ PROTOBUF_CONSTEXPR ReceiptMessage::ReceiptMessage( /*decltype(_impl_._has_bits_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_.timestamp_)*/{} + , /*decltype(_impl_.msgid_)*/{} , /*decltype(_impl_.type_)*/0} {} struct ReceiptMessageDefaultTypeInternal { PROTOBUF_CONSTEXPR ReceiptMessageDefaultTypeInternal() @@ -2047,12 +2040,15 @@ std::string TypingMessage::GetTypeName() const { class UnsendRequest::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); - static void set_has_timestamp(HasBits* has_bits) { + static void set_has_msgtimestamp(HasBits* has_bits) { (*has_bits)[0] |= 2u; } static void set_has_author(HasBits* has_bits) { (*has_bits)[0] |= 1u; } + static void set_has_msgid(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } static bool MissingRequiredFields(const HasBits& has_bits) { return ((has_bits[0] & 0x00000003) ^ 0x00000003) != 0; } @@ -2071,7 +2067,8 @@ UnsendRequest::UnsendRequest(const UnsendRequest& from) decltype(_impl_._has_bits_){from._impl_._has_bits_} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.author_){} - , decltype(_impl_.timestamp_){}}; + , decltype(_impl_.msgtimestamp_){} + , decltype(_impl_.msgid_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); _impl_.author_.InitDefault(); @@ -2082,7 +2079,9 @@ UnsendRequest::UnsendRequest(const UnsendRequest& from) _this->_impl_.author_.Set(from._internal_author(), _this->GetArenaForAllocation()); } - _this->_impl_.timestamp_ = from._impl_.timestamp_; + ::memcpy(&_impl_.msgtimestamp_, &from._impl_.msgtimestamp_, + static_cast(reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.msgid_)); // @@protoc_insertion_point(copy_constructor:SessionProtos.UnsendRequest) } @@ -2094,7 +2093,8 @@ inline void UnsendRequest::SharedCtor( decltype(_impl_._has_bits_){} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.author_){} - , decltype(_impl_.timestamp_){uint64_t{0u}} + , decltype(_impl_.msgtimestamp_){uint64_t{0u}} + , decltype(_impl_.msgid_){int64_t{0}} }; _impl_.author_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -2130,7 +2130,11 @@ void UnsendRequest::Clear() { if (cached_has_bits & 0x00000001u) { _impl_.author_.ClearNonDefaultToEmpty(); } - _impl_.timestamp_ = uint64_t{0u}; + if (cached_has_bits & 0x00000006u) { + ::memset(&_impl_.msgtimestamp_, 0, static_cast( + reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.msgid_)); + } _impl_._has_bits_.Clear(); _internal_metadata_.Clear(); } @@ -2142,11 +2146,11 @@ const char* UnsendRequest::_InternalParse(const char* ptr, ::_pbi::ParseContext* uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { - // required uint64 timestamp = 1; + // required uint64 msgTimestamp = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { - _Internal::set_has_timestamp(&has_bits); - _impl_.timestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + _Internal::set_has_msgtimestamp(&has_bits); + _impl_.msgtimestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; @@ -2160,6 +2164,15 @@ const char* UnsendRequest::_InternalParse(const char* ptr, ::_pbi::ParseContext* } else goto handle_unusual; continue; + // optional sfixed64 msgId = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 25)) { + _Internal::set_has_msgid(&has_bits); + _impl_.msgid_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -2191,10 +2204,10 @@ uint8_t* UnsendRequest::_InternalSerialize( (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - // required uint64 timestamp = 1; + // required uint64 msgTimestamp = 1; if (cached_has_bits & 0x00000002u) { target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_timestamp(), target); + target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_msgtimestamp(), target); } // required string author = 2; @@ -2203,6 +2216,12 @@ uint8_t* UnsendRequest::_InternalSerialize( 2, this->_internal_author(), target); } + // optional sfixed64 msgId = 3; + if (cached_has_bits & 0x00000004u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSFixed64ToArray(3, this->_internal_msgid(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -2222,9 +2241,9 @@ size_t UnsendRequest::RequiredFieldsByteSizeFallback() const { this->_internal_author()); } - if (_internal_has_timestamp()) { - // required uint64 timestamp = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_timestamp()); + if (_internal_has_msgtimestamp()) { + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); } return total_size; @@ -2239,8 +2258,8 @@ size_t UnsendRequest::ByteSizeLong() const { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_author()); - // required uint64 timestamp = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_timestamp()); + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); } else { total_size += RequiredFieldsByteSizeFallback(); @@ -2249,6 +2268,12 @@ size_t UnsendRequest::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; + // optional sfixed64 msgId = 3; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000004u) { + total_size += 1 + 8; + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); } @@ -2271,12 +2296,15 @@ void UnsendRequest::MergeFrom(const UnsendRequest& from) { (void) cached_has_bits; cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { + if (cached_has_bits & 0x00000007u) { if (cached_has_bits & 0x00000001u) { _this->_internal_set_author(from._internal_author()); } if (cached_has_bits & 0x00000002u) { - _this->_impl_.timestamp_ = from._impl_.timestamp_; + _this->_impl_.msgtimestamp_ = from._impl_.msgtimestamp_; + } + if (cached_has_bits & 0x00000004u) { + _this->_impl_.msgid_ = from._impl_.msgid_; } _this->_impl_._has_bits_[0] |= cached_has_bits; } @@ -2305,7 +2333,12 @@ void UnsendRequest::InternalSwap(UnsendRequest* other) { &_impl_.author_, lhs_arena, &other->_impl_.author_, rhs_arena ); - swap(_impl_.timestamp_, other->_impl_.timestamp_); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(UnsendRequest, _impl_.msgid_) + + sizeof(UnsendRequest::_impl_.msgid_) + - PROTOBUF_FIELD_OFFSET(UnsendRequest, _impl_.msgtimestamp_)>( + reinterpret_cast(&_impl_.msgtimestamp_), + reinterpret_cast(&other->_impl_.msgtimestamp_)); } std::string UnsendRequest::GetTypeName() const { @@ -2672,6 +2705,9 @@ class Content::_Internal { static void set_has_prosigforcommunitymessageonly(HasBits* has_bits) { (*has_bits)[0] |= 1u; } + static void set_has_msgid(HasBits* has_bits) { + (*has_bits)[0] |= 8192u; + } }; const ::SessionProtos::DataMessage& @@ -2734,7 +2770,8 @@ Content::Content(const Content& from) , decltype(_impl_.promessage_){nullptr} , decltype(_impl_.expirationtype_){} , decltype(_impl_.expirationtimer_){} - , decltype(_impl_.sigtimestamp_){}}; + , decltype(_impl_.sigtimestamp_){} + , decltype(_impl_.msgid_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); _impl_.prosigforcommunitymessageonly_.InitDefault(); @@ -2773,8 +2810,8 @@ Content::Content(const Content& from) _this->_impl_.promessage_ = new ::SessionProtos::ProMessage(*from._impl_.promessage_); } ::memcpy(&_impl_.expirationtype_, &from._impl_.expirationtype_, - static_cast(reinterpret_cast(&_impl_.sigtimestamp_) - - reinterpret_cast(&_impl_.expirationtype_)) + sizeof(_impl_.sigtimestamp_)); + static_cast(reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.expirationtype_)) + sizeof(_impl_.msgid_)); // @@protoc_insertion_point(copy_constructor:SessionProtos.Content) } @@ -2798,6 +2835,7 @@ inline void Content::SharedCtor( , decltype(_impl_.expirationtype_){0} , decltype(_impl_.expirationtimer_){0u} , decltype(_impl_.sigtimestamp_){uint64_t{0u}} + , decltype(_impl_.msgid_){int64_t{0}} }; _impl_.prosigforcommunitymessageonly_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -2882,10 +2920,10 @@ void Content::Clear() { _impl_.promessage_->Clear(); } } - if (cached_has_bits & 0x00001c00u) { + if (cached_has_bits & 0x00003c00u) { ::memset(&_impl_.expirationtype_, 0, static_cast( - reinterpret_cast(&_impl_.sigtimestamp_) - - reinterpret_cast(&_impl_.expirationtype_)) + sizeof(_impl_.sigtimestamp_)); + reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.expirationtype_)) + sizeof(_impl_.msgid_)); } _impl_._has_bits_.Clear(); _internal_metadata_.Clear(); @@ -3010,6 +3048,15 @@ const char* Content::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) } else goto handle_unusual; continue; + // optional sfixed64 msgId = 18; + case 18: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 145)) { + _Internal::set_has_msgid(&has_bits); + _impl_.msgid_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -3129,6 +3176,12 @@ uint8_t* Content::_InternalSerialize( 17, this->_internal_prosigforcommunitymessageonly(), target); } + // optional sfixed64 msgId = 18; + if (cached_has_bits & 0x00002000u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSFixed64ToArray(18, this->_internal_msgid(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -3204,7 +3257,7 @@ size_t Content::ByteSizeLong() const { } } - if (cached_has_bits & 0x00001f00u) { + if (cached_has_bits & 0x00003f00u) { // optional .SessionProtos.SharedConfigMessage sharedConfigMessage = 11; if (cached_has_bits & 0x00000100u) { total_size += 1 + @@ -3235,6 +3288,11 @@ size_t Content::ByteSizeLong() const { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_sigtimestamp()); } + // optional sfixed64 msgId = 18; + if (cached_has_bits & 0x00002000u) { + total_size += 2 + 8; + } + } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); @@ -3291,7 +3349,7 @@ void Content::MergeFrom(const Content& from) { from._internal_messagerequestresponse()); } } - if (cached_has_bits & 0x00001f00u) { + if (cached_has_bits & 0x00003f00u) { if (cached_has_bits & 0x00000100u) { _this->_internal_mutable_sharedconfigmessage()->::SessionProtos::SharedConfigMessage::MergeFrom( from._internal_sharedconfigmessage()); @@ -3309,6 +3367,9 @@ void Content::MergeFrom(const Content& from) { if (cached_has_bits & 0x00001000u) { _this->_impl_.sigtimestamp_ = from._impl_.sigtimestamp_; } + if (cached_has_bits & 0x00002000u) { + _this->_impl_.msgid_ = from._impl_.msgid_; + } _this->_impl_._has_bits_[0] |= cached_has_bits; } _this->_internal_metadata_.MergeFrom(from._internal_metadata_); @@ -3360,8 +3421,8 @@ void Content::InternalSwap(Content* other) { &other->_impl_.prosigforcommunitymessageonly_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< - PROTOBUF_FIELD_OFFSET(Content, _impl_.sigtimestamp_) - + sizeof(Content::_impl_.sigtimestamp_) + PROTOBUF_FIELD_OFFSET(Content, _impl_.msgid_) + + sizeof(Content::_impl_.msgid_) - PROTOBUF_FIELD_OFFSET(Content, _impl_.datamessage_)>( reinterpret_cast(&_impl_.datamessage_), reinterpret_cast(&other->_impl_.datamessage_)); @@ -3757,312 +3818,28 @@ std::string CallMessage::GetTypeName() const { } -// =================================================================== - -class KeyPair::_Internal { - public: - using HasBits = decltype(std::declval()._impl_._has_bits_); - static void set_has_publickey(HasBits* has_bits) { - (*has_bits)[0] |= 1u; - } - static void set_has_privatekey(HasBits* has_bits) { - (*has_bits)[0] |= 2u; - } - static bool MissingRequiredFields(const HasBits& has_bits) { - return ((has_bits[0] & 0x00000003) ^ 0x00000003) != 0; - } -}; - -KeyPair::KeyPair(::PROTOBUF_NAMESPACE_ID::Arena* arena, - bool is_message_owned) - : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena, is_message_owned) { - SharedCtor(arena, is_message_owned); - // @@protoc_insertion_point(arena_constructor:SessionProtos.KeyPair) -} -KeyPair::KeyPair(const KeyPair& from) - : ::PROTOBUF_NAMESPACE_ID::MessageLite() { - KeyPair* const _this = this; (void)_this; - new (&_impl_) Impl_{ - decltype(_impl_._has_bits_){from._impl_._has_bits_} - , /*decltype(_impl_._cached_size_)*/{} - , decltype(_impl_.publickey_){} - , decltype(_impl_.privatekey_){}}; - - _internal_metadata_.MergeFrom(from._internal_metadata_); - _impl_.publickey_.InitDefault(); - #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - _impl_.publickey_.Set("", GetArenaForAllocation()); - #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (from._internal_has_publickey()) { - _this->_impl_.publickey_.Set(from._internal_publickey(), - _this->GetArenaForAllocation()); - } - _impl_.privatekey_.InitDefault(); - #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - _impl_.privatekey_.Set("", GetArenaForAllocation()); - #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (from._internal_has_privatekey()) { - _this->_impl_.privatekey_.Set(from._internal_privatekey(), - _this->GetArenaForAllocation()); - } - // @@protoc_insertion_point(copy_constructor:SessionProtos.KeyPair) -} - -inline void KeyPair::SharedCtor( - ::_pb::Arena* arena, bool is_message_owned) { - (void)arena; - (void)is_message_owned; - new (&_impl_) Impl_{ - decltype(_impl_._has_bits_){} - , /*decltype(_impl_._cached_size_)*/{} - , decltype(_impl_.publickey_){} - , decltype(_impl_.privatekey_){} - }; - _impl_.publickey_.InitDefault(); - #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - _impl_.publickey_.Set("", GetArenaForAllocation()); - #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - _impl_.privatekey_.InitDefault(); - #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - _impl_.privatekey_.Set("", GetArenaForAllocation()); - #endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING -} - -KeyPair::~KeyPair() { - // @@protoc_insertion_point(destructor:SessionProtos.KeyPair) - if (auto *arena = _internal_metadata_.DeleteReturnArena()) { - (void)arena; - return; - } - SharedDtor(); -} - -inline void KeyPair::SharedDtor() { - GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); - _impl_.publickey_.Destroy(); - _impl_.privatekey_.Destroy(); -} - -void KeyPair::SetCachedSize(int size) const { - _impl_._cached_size_.Set(size); -} - -void KeyPair::Clear() { -// @@protoc_insertion_point(message_clear_start:SessionProtos.KeyPair) - uint32_t cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - if (cached_has_bits & 0x00000001u) { - _impl_.publickey_.ClearNonDefaultToEmpty(); - } - if (cached_has_bits & 0x00000002u) { - _impl_.privatekey_.ClearNonDefaultToEmpty(); - } - } - _impl_._has_bits_.Clear(); - _internal_metadata_.Clear(); -} - -const char* KeyPair::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) { -#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure - _Internal::HasBits has_bits{}; - while (!ctx->Done(&ptr)) { - uint32_t tag; - ptr = ::_pbi::ReadTag(ptr, &tag); - switch (tag >> 3) { - // required bytes publicKey = 1; - case 1: - if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 10)) { - auto str = _internal_mutable_publickey(); - ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); - CHK_(ptr); - } else - goto handle_unusual; - continue; - // required bytes privateKey = 2; - case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 18)) { - auto str = _internal_mutable_privatekey(); - ptr = ::_pbi::InlineGreedyStringParser(str, ptr, ctx); - CHK_(ptr); - } else - goto handle_unusual; - continue; - default: - goto handle_unusual; - } // switch - handle_unusual: - if ((tag == 0) || ((tag & 7) == 4)) { - CHK_(ptr); - ctx->SetLastTag(tag); - goto message_done; - } - ptr = UnknownFieldParse( - tag, - _internal_metadata_.mutable_unknown_fields(), - ptr, ctx); - CHK_(ptr != nullptr); - } // while -message_done: - _impl_._has_bits_.Or(has_bits); - return ptr; -failure: - ptr = nullptr; - goto message_done; -#undef CHK_ -} - -uint8_t* KeyPair::_InternalSerialize( - uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { - // @@protoc_insertion_point(serialize_to_array_start:SessionProtos.KeyPair) - uint32_t cached_has_bits = 0; - (void) cached_has_bits; - - cached_has_bits = _impl_._has_bits_[0]; - // required bytes publicKey = 1; - if (cached_has_bits & 0x00000001u) { - target = stream->WriteBytesMaybeAliased( - 1, this->_internal_publickey(), target); - } - - // required bytes privateKey = 2; - if (cached_has_bits & 0x00000002u) { - target = stream->WriteBytesMaybeAliased( - 2, this->_internal_privatekey(), target); - } - - if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { - target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), - static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:SessionProtos.KeyPair) - return target; -} - -size_t KeyPair::RequiredFieldsByteSizeFallback() const { -// @@protoc_insertion_point(required_fields_byte_size_fallback_start:SessionProtos.KeyPair) - size_t total_size = 0; - - if (_internal_has_publickey()) { - // required bytes publicKey = 1; - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize( - this->_internal_publickey()); - } - - if (_internal_has_privatekey()) { - // required bytes privateKey = 2; - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize( - this->_internal_privatekey()); - } - - return total_size; -} -size_t KeyPair::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:SessionProtos.KeyPair) - size_t total_size = 0; - - if (((_impl_._has_bits_[0] & 0x00000003) ^ 0x00000003) == 0) { // All required fields are present. - // required bytes publicKey = 1; - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize( - this->_internal_publickey()); - - // required bytes privateKey = 2; - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize( - this->_internal_privatekey()); - - } else { - total_size += RequiredFieldsByteSizeFallback(); - } - uint32_t cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { - total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); - } - int cached_size = ::_pbi::ToCachedSize(total_size); - SetCachedSize(cached_size); - return total_size; -} - -void KeyPair::CheckTypeAndMergeFrom( - const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { - MergeFrom(*::_pbi::DownCast( - &from)); -} - -void KeyPair::MergeFrom(const KeyPair& from) { - KeyPair* const _this = this; - // @@protoc_insertion_point(class_specific_merge_from_start:SessionProtos.KeyPair) - GOOGLE_DCHECK_NE(&from, _this); - uint32_t cached_has_bits = 0; - (void) cached_has_bits; - - cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - if (cached_has_bits & 0x00000001u) { - _this->_internal_set_publickey(from._internal_publickey()); - } - if (cached_has_bits & 0x00000002u) { - _this->_internal_set_privatekey(from._internal_privatekey()); - } - } - _this->_internal_metadata_.MergeFrom(from._internal_metadata_); -} - -void KeyPair::CopyFrom(const KeyPair& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:SessionProtos.KeyPair) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool KeyPair::IsInitialized() const { - if (_Internal::MissingRequiredFields(_impl_._has_bits_)) return false; - return true; -} - -void KeyPair::InternalSwap(KeyPair* other) { - using std::swap; - auto* lhs_arena = GetArenaForAllocation(); - auto* rhs_arena = other->GetArenaForAllocation(); - _internal_metadata_.InternalSwap(&other->_internal_metadata_); - swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); - ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( - &_impl_.publickey_, lhs_arena, - &other->_impl_.publickey_, rhs_arena - ); - ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::InternalSwap( - &_impl_.privatekey_, lhs_arena, - &other->_impl_.privatekey_, rhs_arena - ); -} - -std::string KeyPair::GetTypeName() const { - return "SessionProtos.KeyPair"; -} - - // =================================================================== class DataExtractionNotification::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); static void set_has_type(HasBits* has_bits) { - (*has_bits)[0] |= 2u; + (*has_bits)[0] |= 16u; } static void set_has_timestamp(HasBits* has_bits) { (*has_bits)[0] |= 1u; } + static void set_has_msgtimestamp(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_msgid(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } + static void set_has_attindex(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } static bool MissingRequiredFields(const HasBits& has_bits) { - return ((has_bits[0] & 0x00000002) ^ 0x00000002) != 0; + return ((has_bits[0] & 0x00000010) ^ 0x00000010) != 0; } }; @@ -4079,6 +3856,9 @@ DataExtractionNotification::DataExtractionNotification(const DataExtractionNotif decltype(_impl_._has_bits_){from._impl_._has_bits_} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.timestamp_){} + , decltype(_impl_.msgtimestamp_){} + , decltype(_impl_.msgid_){} + , decltype(_impl_.attindex_){} , decltype(_impl_.type_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); @@ -4096,6 +3876,9 @@ inline void DataExtractionNotification::SharedCtor( decltype(_impl_._has_bits_){} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.timestamp_){uint64_t{0u}} + , decltype(_impl_.msgtimestamp_){uint64_t{0u}} + , decltype(_impl_.msgid_){int64_t{0}} + , decltype(_impl_.attindex_){0} , decltype(_impl_.type_){1} }; } @@ -4124,8 +3907,10 @@ void DataExtractionNotification::Clear() { (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - _impl_.timestamp_ = uint64_t{0u}; + if (cached_has_bits & 0x0000001fu) { + ::memset(&_impl_.timestamp_, 0, static_cast( + reinterpret_cast(&_impl_.attindex_) - + reinterpret_cast(&_impl_.timestamp_)) + sizeof(_impl_.attindex_)); _impl_.type_ = 1; } _impl_._has_bits_.Clear(); @@ -4161,6 +3946,33 @@ const char* DataExtractionNotification::_InternalParse(const char* ptr, ::_pbi:: } else goto handle_unusual; continue; + // optional uint64 msgTimestamp = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 24)) { + _Internal::set_has_msgtimestamp(&has_bits); + _impl_.msgtimestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; + // optional sfixed64 msgId = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 33)) { + _Internal::set_has_msgid(&has_bits); + _impl_.msgid_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; + // optional sint32 attIndex = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 40)) { + _Internal::set_has_attindex(&has_bits); + _impl_.attindex_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarintZigZag32(&ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -4193,7 +4005,7 @@ uint8_t* DataExtractionNotification::_InternalSerialize( cached_has_bits = _impl_._has_bits_[0]; // required .SessionProtos.DataExtractionNotification.Type type = 1; - if (cached_has_bits & 0x00000002u) { + if (cached_has_bits & 0x00000010u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 1, this->_internal_type(), target); @@ -4205,6 +4017,24 @@ uint8_t* DataExtractionNotification::_InternalSerialize( target = ::_pbi::WireFormatLite::WriteUInt64ToArray(2, this->_internal_timestamp(), target); } + // optional uint64 msgTimestamp = 3; + if (cached_has_bits & 0x00000002u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteUInt64ToArray(3, this->_internal_msgtimestamp(), target); + } + + // optional sfixed64 msgId = 4; + if (cached_has_bits & 0x00000004u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSFixed64ToArray(4, this->_internal_msgid(), target); + } + + // optional sint32 attIndex = 5; + if (cached_has_bits & 0x00000008u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSInt32ToArray(5, this->_internal_attindex(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -4226,12 +4056,29 @@ size_t DataExtractionNotification::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // optional uint64 timestamp = 2; cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_timestamp()); - } + if (cached_has_bits & 0x0000000fu) { + // optional uint64 timestamp = 2; + if (cached_has_bits & 0x00000001u) { + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_timestamp()); + } + + // optional uint64 msgTimestamp = 3; + if (cached_has_bits & 0x00000002u) { + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); + } + // optional sfixed64 msgId = 4; + if (cached_has_bits & 0x00000004u) { + total_size += 1 + 8; + } + + // optional sint32 attIndex = 5; + if (cached_has_bits & 0x00000008u) { + total_size += ::_pbi::WireFormatLite::SInt32SizePlusOne(this->_internal_attindex()); + } + + } if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); } @@ -4254,11 +4101,20 @@ void DataExtractionNotification::MergeFrom(const DataExtractionNotification& fro (void) cached_has_bits; cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { + if (cached_has_bits & 0x0000001fu) { if (cached_has_bits & 0x00000001u) { _this->_impl_.timestamp_ = from._impl_.timestamp_; } if (cached_has_bits & 0x00000002u) { + _this->_impl_.msgtimestamp_ = from._impl_.msgtimestamp_; + } + if (cached_has_bits & 0x00000004u) { + _this->_impl_.msgid_ = from._impl_.msgid_; + } + if (cached_has_bits & 0x00000008u) { + _this->_impl_.attindex_ = from._impl_.attindex_; + } + if (cached_has_bits & 0x00000010u) { _this->_impl_.type_ = from._impl_.type_; } _this->_impl_._has_bits_[0] |= cached_has_bits; @@ -4282,7 +4138,12 @@ void DataExtractionNotification::InternalSwap(DataExtractionNotification* other) using std::swap; _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); - swap(_impl_.timestamp_, other->_impl_.timestamp_); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(DataExtractionNotification, _impl_.attindex_) + + sizeof(DataExtractionNotification::_impl_.attindex_) + - PROTOBUF_FIELD_OFFSET(DataExtractionNotification, _impl_.timestamp_)>( + reinterpret_cast(&_impl_.timestamp_), + reinterpret_cast(&other->_impl_.timestamp_)); swap(_impl_.type_, other->_impl_.type_); } @@ -4956,7 +4817,7 @@ std::string DataMessage_Quote_QuotedAttachment::GetTypeName() const { class DataMessage_Quote::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); - static void set_has_id(HasBits* has_bits) { + static void set_has_msgtimestamp(HasBits* has_bits) { (*has_bits)[0] |= 4u; } static void set_has_author(HasBits* has_bits) { @@ -4965,6 +4826,9 @@ class DataMessage_Quote::_Internal { static void set_has_text(HasBits* has_bits) { (*has_bits)[0] |= 2u; } + static void set_has_msgid(HasBits* has_bits) { + (*has_bits)[0] |= 8u; + } static bool MissingRequiredFields(const HasBits& has_bits) { return ((has_bits[0] & 0x00000005) ^ 0x00000005) != 0; } @@ -4985,7 +4849,8 @@ DataMessage_Quote::DataMessage_Quote(const DataMessage_Quote& from) , decltype(_impl_.attachments_){from._impl_.attachments_} , decltype(_impl_.author_){} , decltype(_impl_.text_){} - , decltype(_impl_.id_){}}; + , decltype(_impl_.msgtimestamp_){} + , decltype(_impl_.msgid_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); _impl_.author_.InitDefault(); @@ -5004,7 +4869,9 @@ DataMessage_Quote::DataMessage_Quote(const DataMessage_Quote& from) _this->_impl_.text_.Set(from._internal_text(), _this->GetArenaForAllocation()); } - _this->_impl_.id_ = from._impl_.id_; + ::memcpy(&_impl_.msgtimestamp_, &from._impl_.msgtimestamp_, + static_cast(reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.msgid_)); // @@protoc_insertion_point(copy_constructor:SessionProtos.DataMessage.Quote) } @@ -5018,7 +4885,8 @@ inline void DataMessage_Quote::SharedCtor( , decltype(_impl_.attachments_){arena} , decltype(_impl_.author_){} , decltype(_impl_.text_){} - , decltype(_impl_.id_){uint64_t{0u}} + , decltype(_impl_.msgtimestamp_){uint64_t{0u}} + , decltype(_impl_.msgid_){int64_t{0}} }; _impl_.author_.InitDefault(); #ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING @@ -5066,7 +4934,11 @@ void DataMessage_Quote::Clear() { _impl_.text_.ClearNonDefaultToEmpty(); } } - _impl_.id_ = uint64_t{0u}; + if (cached_has_bits & 0x0000000cu) { + ::memset(&_impl_.msgtimestamp_, 0, static_cast( + reinterpret_cast(&_impl_.msgid_) - + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.msgid_)); + } _impl_._has_bits_.Clear(); _internal_metadata_.Clear(); } @@ -5078,11 +4950,11 @@ const char* DataMessage_Quote::_InternalParse(const char* ptr, ::_pbi::ParseCont uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { - // required uint64 id = 1; + // required uint64 msgTimestamp = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { - _Internal::set_has_id(&has_bits); - _impl_.id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + _Internal::set_has_msgtimestamp(&has_bits); + _impl_.msgtimestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; @@ -5118,6 +4990,15 @@ const char* DataMessage_Quote::_InternalParse(const char* ptr, ::_pbi::ParseCont } else goto handle_unusual; continue; + // optional sfixed64 msgId = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 41)) { + _Internal::set_has_msgid(&has_bits); + _impl_.msgid_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -5149,10 +5030,10 @@ uint8_t* DataMessage_Quote::_InternalSerialize( (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - // required uint64 id = 1; + // required uint64 msgTimestamp = 1; if (cached_has_bits & 0x00000004u) { target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_id(), target); + target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_msgtimestamp(), target); } // required string author = 2; @@ -5175,6 +5056,12 @@ uint8_t* DataMessage_Quote::_InternalSerialize( InternalWriteMessage(4, repfield, repfield.GetCachedSize(), target, stream); } + // optional sfixed64 msgId = 5; + if (cached_has_bits & 0x00000008u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSFixed64ToArray(5, this->_internal_msgid(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -5194,9 +5081,9 @@ size_t DataMessage_Quote::RequiredFieldsByteSizeFallback() const { this->_internal_author()); } - if (_internal_has_id()) { - // required uint64 id = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_id()); + if (_internal_has_msgtimestamp()) { + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); } return total_size; @@ -5211,8 +5098,8 @@ size_t DataMessage_Quote::ByteSizeLong() const { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_author()); - // required uint64 id = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_id()); + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); } else { total_size += RequiredFieldsByteSizeFallback(); @@ -5236,6 +5123,11 @@ size_t DataMessage_Quote::ByteSizeLong() const { this->_internal_text()); } + // optional sfixed64 msgId = 5; + if (cached_has_bits & 0x00000008u) { + total_size += 1 + 8; + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); } @@ -5259,7 +5151,7 @@ void DataMessage_Quote::MergeFrom(const DataMessage_Quote& from) { _this->_impl_.attachments_.MergeFrom(from._impl_.attachments_); cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000007u) { + if (cached_has_bits & 0x0000000fu) { if (cached_has_bits & 0x00000001u) { _this->_internal_set_author(from._internal_author()); } @@ -5267,7 +5159,10 @@ void DataMessage_Quote::MergeFrom(const DataMessage_Quote& from) { _this->_internal_set_text(from._internal_text()); } if (cached_has_bits & 0x00000004u) { - _this->_impl_.id_ = from._impl_.id_; + _this->_impl_.msgtimestamp_ = from._impl_.msgtimestamp_; + } + if (cached_has_bits & 0x00000008u) { + _this->_impl_.msgid_ = from._impl_.msgid_; } _this->_impl_._has_bits_[0] |= cached_has_bits; } @@ -5303,7 +5198,12 @@ void DataMessage_Quote::InternalSwap(DataMessage_Quote* other) { &_impl_.text_, lhs_arena, &other->_impl_.text_, rhs_arena ); - swap(_impl_.id_, other->_impl_.id_); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(DataMessage_Quote, _impl_.msgid_) + + sizeof(DataMessage_Quote::_impl_.msgid_) + - PROTOBUF_FIELD_OFFSET(DataMessage_Quote, _impl_.msgtimestamp_)>( + reinterpret_cast(&_impl_.msgtimestamp_), + reinterpret_cast(&other->_impl_.msgtimestamp_)); } std::string DataMessage_Quote::GetTypeName() const { @@ -5639,7 +5539,7 @@ std::string DataMessage_Preview::GetTypeName() const { class DataMessage_Reaction::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); - static void set_has_id(HasBits* has_bits) { + static void set_has_msgtimestamp(HasBits* has_bits) { (*has_bits)[0] |= 4u; } static void set_has_author(HasBits* has_bits) { @@ -5649,10 +5549,13 @@ class DataMessage_Reaction::_Internal { (*has_bits)[0] |= 2u; } static void set_has_action(HasBits* has_bits) { + (*has_bits)[0] |= 16u; + } + static void set_has_msgid(HasBits* has_bits) { (*has_bits)[0] |= 8u; } static bool MissingRequiredFields(const HasBits& has_bits) { - return ((has_bits[0] & 0x0000000d) ^ 0x0000000d) != 0; + return ((has_bits[0] & 0x00000015) ^ 0x00000015) != 0; } }; @@ -5670,7 +5573,8 @@ DataMessage_Reaction::DataMessage_Reaction(const DataMessage_Reaction& from) , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.author_){} , decltype(_impl_.emoji_){} - , decltype(_impl_.id_){} + , decltype(_impl_.msgtimestamp_){} + , decltype(_impl_.msgid_){} , decltype(_impl_.action_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); @@ -5690,9 +5594,9 @@ DataMessage_Reaction::DataMessage_Reaction(const DataMessage_Reaction& from) _this->_impl_.emoji_.Set(from._internal_emoji(), _this->GetArenaForAllocation()); } - ::memcpy(&_impl_.id_, &from._impl_.id_, + ::memcpy(&_impl_.msgtimestamp_, &from._impl_.msgtimestamp_, static_cast(reinterpret_cast(&_impl_.action_) - - reinterpret_cast(&_impl_.id_)) + sizeof(_impl_.action_)); + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.action_)); // @@protoc_insertion_point(copy_constructor:SessionProtos.DataMessage.Reaction) } @@ -5705,7 +5609,8 @@ inline void DataMessage_Reaction::SharedCtor( , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.author_){} , decltype(_impl_.emoji_){} - , decltype(_impl_.id_){uint64_t{0u}} + , decltype(_impl_.msgtimestamp_){uint64_t{0u}} + , decltype(_impl_.msgid_){int64_t{0}} , decltype(_impl_.action_){0} }; _impl_.author_.InitDefault(); @@ -5752,10 +5657,10 @@ void DataMessage_Reaction::Clear() { _impl_.emoji_.ClearNonDefaultToEmpty(); } } - if (cached_has_bits & 0x0000000cu) { - ::memset(&_impl_.id_, 0, static_cast( + if (cached_has_bits & 0x0000001cu) { + ::memset(&_impl_.msgtimestamp_, 0, static_cast( reinterpret_cast(&_impl_.action_) - - reinterpret_cast(&_impl_.id_)) + sizeof(_impl_.action_)); + reinterpret_cast(&_impl_.msgtimestamp_)) + sizeof(_impl_.action_)); } _impl_._has_bits_.Clear(); _internal_metadata_.Clear(); @@ -5768,11 +5673,11 @@ const char* DataMessage_Reaction::_InternalParse(const char* ptr, ::_pbi::ParseC uint32_t tag; ptr = ::_pbi::ReadTag(ptr, &tag); switch (tag >> 3) { - // required uint64 id = 1; + // required uint64 msgTimestamp = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 8)) { - _Internal::set_has_id(&has_bits); - _impl_.id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + _Internal::set_has_msgtimestamp(&has_bits); + _impl_.msgtimestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); CHK_(ptr); } else goto handle_unusual; @@ -5808,6 +5713,15 @@ const char* DataMessage_Reaction::_InternalParse(const char* ptr, ::_pbi::ParseC } else goto handle_unusual; continue; + // optional sfixed64 msgId = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 41)) { + _Internal::set_has_msgid(&has_bits); + _impl_.msgid_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -5839,10 +5753,10 @@ uint8_t* DataMessage_Reaction::_InternalSerialize( (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - // required uint64 id = 1; + // required uint64 msgTimestamp = 1; if (cached_has_bits & 0x00000004u) { target = stream->EnsureSpace(target); - target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_id(), target); + target = ::_pbi::WireFormatLite::WriteUInt64ToArray(1, this->_internal_msgtimestamp(), target); } // required string author = 2; @@ -5858,12 +5772,18 @@ uint8_t* DataMessage_Reaction::_InternalSerialize( } // required .SessionProtos.DataMessage.Reaction.Action action = 4; - if (cached_has_bits & 0x00000008u) { + if (cached_has_bits & 0x00000010u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 4, this->_internal_action(), target); } + // optional sfixed64 msgId = 5; + if (cached_has_bits & 0x00000008u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteSFixed64ToArray(5, this->_internal_msgid(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -5883,9 +5803,9 @@ size_t DataMessage_Reaction::RequiredFieldsByteSizeFallback() const { this->_internal_author()); } - if (_internal_has_id()) { - // required uint64 id = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_id()); + if (_internal_has_msgtimestamp()) { + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); } if (_internal_has_action()) { @@ -5900,14 +5820,14 @@ size_t DataMessage_Reaction::ByteSizeLong() const { // @@protoc_insertion_point(message_byte_size_start:SessionProtos.DataMessage.Reaction) size_t total_size = 0; - if (((_impl_._has_bits_[0] & 0x0000000d) ^ 0x0000000d) == 0) { // All required fields are present. + if (((_impl_._has_bits_[0] & 0x00000015) ^ 0x00000015) == 0) { // All required fields are present. // required string author = 2; total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->_internal_author()); - // required uint64 id = 1; - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_id()); + // required uint64 msgTimestamp = 1; + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne(this->_internal_msgtimestamp()); // required .SessionProtos.DataMessage.Reaction.Action action = 4; total_size += 1 + @@ -5928,6 +5848,11 @@ size_t DataMessage_Reaction::ByteSizeLong() const { this->_internal_emoji()); } + // optional sfixed64 msgId = 5; + if (cached_has_bits & 0x00000008u) { + total_size += 1 + 8; + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); } @@ -5950,7 +5875,7 @@ void DataMessage_Reaction::MergeFrom(const DataMessage_Reaction& from) { (void) cached_has_bits; cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x0000000fu) { + if (cached_has_bits & 0x0000001fu) { if (cached_has_bits & 0x00000001u) { _this->_internal_set_author(from._internal_author()); } @@ -5958,9 +5883,12 @@ void DataMessage_Reaction::MergeFrom(const DataMessage_Reaction& from) { _this->_internal_set_emoji(from._internal_emoji()); } if (cached_has_bits & 0x00000004u) { - _this->_impl_.id_ = from._impl_.id_; + _this->_impl_.msgtimestamp_ = from._impl_.msgtimestamp_; } if (cached_has_bits & 0x00000008u) { + _this->_impl_.msgid_ = from._impl_.msgid_; + } + if (cached_has_bits & 0x00000010u) { _this->_impl_.action_ = from._impl_.action_; } _this->_impl_._has_bits_[0] |= cached_has_bits; @@ -5997,9 +5925,9 @@ void DataMessage_Reaction::InternalSwap(DataMessage_Reaction* other) { ::PROTOBUF_NAMESPACE_ID::internal::memswap< PROTOBUF_FIELD_OFFSET(DataMessage_Reaction, _impl_.action_) + sizeof(DataMessage_Reaction::_impl_.action_) - - PROTOBUF_FIELD_OFFSET(DataMessage_Reaction, _impl_.id_)>( - reinterpret_cast(&_impl_.id_), - reinterpret_cast(&other->_impl_.id_)); + - PROTOBUF_FIELD_OFFSET(DataMessage_Reaction, _impl_.msgtimestamp_)>( + reinterpret_cast(&_impl_.msgtimestamp_), + reinterpret_cast(&other->_impl_.msgtimestamp_)); } std::string DataMessage_Reaction::GetTypeName() const { @@ -7055,6 +6983,7 @@ ReceiptMessage::ReceiptMessage(const ReceiptMessage& from) decltype(_impl_._has_bits_){from._impl_._has_bits_} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.timestamp_){from._impl_.timestamp_} + , decltype(_impl_.msgid_){from._impl_.msgid_} , decltype(_impl_.type_){}}; _internal_metadata_.MergeFrom(from._internal_metadata_); @@ -7070,6 +6999,7 @@ inline void ReceiptMessage::SharedCtor( decltype(_impl_._has_bits_){} , /*decltype(_impl_._cached_size_)*/{} , decltype(_impl_.timestamp_){arena} + , decltype(_impl_.msgid_){arena} , decltype(_impl_.type_){0} }; } @@ -7086,6 +7016,7 @@ ReceiptMessage::~ReceiptMessage() { inline void ReceiptMessage::SharedDtor() { GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); _impl_.timestamp_.~RepeatedField(); + _impl_.msgid_.~RepeatedField(); } void ReceiptMessage::SetCachedSize(int size) const { @@ -7099,6 +7030,7 @@ void ReceiptMessage::Clear() { (void) cached_has_bits; _impl_.timestamp_.Clear(); + _impl_.msgid_.Clear(); _impl_.type_ = 0; _impl_._has_bits_.Clear(); _internal_metadata_.Clear(); @@ -7140,6 +7072,17 @@ const char* ReceiptMessage::_InternalParse(const char* ptr, ::_pbi::ParseContext } else goto handle_unusual; continue; + // repeated sfixed64 msgId = 3 [packed = true]; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 26)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedSFixed64Parser(_internal_mutable_msgid(), ptr, ctx); + CHK_(ptr); + } else if (static_cast(tag) == 25) { + _internal_add_msgid(::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr)); + ptr += sizeof(int64_t); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -7184,6 +7127,11 @@ uint8_t* ReceiptMessage::_InternalSerialize( target = ::_pbi::WireFormatLite::WriteUInt64ToArray(2, this->_internal_timestamp(i), target); } + // repeated sfixed64 msgId = 3 [packed = true]; + if (this->_internal_msgid_size() > 0) { + target = stream->WriteFixedPacked(3, _internal_msgid(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); @@ -7214,6 +7162,17 @@ size_t ReceiptMessage::ByteSizeLong() const { total_size += data_size; } + // repeated sfixed64 msgId = 3 [packed = true]; + { + unsigned int count = static_cast(this->_internal_msgid_size()); + size_t data_size = 8UL * count; + if (data_size > 0) { + total_size += 1 + + ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); + } + total_size += data_size; + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); } @@ -7236,6 +7195,7 @@ void ReceiptMessage::MergeFrom(const ReceiptMessage& from) { (void) cached_has_bits; _this->_impl_.timestamp_.MergeFrom(from._impl_.timestamp_); + _this->_impl_.msgid_.MergeFrom(from._impl_.msgid_); if (from._internal_has_type()) { _this->_internal_set_type(from._internal_type()); } @@ -7259,6 +7219,7 @@ void ReceiptMessage::InternalSwap(ReceiptMessage* other) { _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); _impl_.timestamp_.InternalSwap(&other->_impl_.timestamp_); + _impl_.msgid_.InternalSwap(&other->_impl_.msgid_); swap(_impl_.type_, other->_impl_.type_); } @@ -11657,10 +11618,6 @@ template<> PROTOBUF_NOINLINE ::SessionProtos::CallMessage* Arena::CreateMaybeMessage< ::SessionProtos::CallMessage >(Arena* arena) { return Arena::CreateMessageInternal< ::SessionProtos::CallMessage >(arena); } -template<> PROTOBUF_NOINLINE ::SessionProtos::KeyPair* -Arena::CreateMaybeMessage< ::SessionProtos::KeyPair >(Arena* arena) { - return Arena::CreateMessageInternal< ::SessionProtos::KeyPair >(arena); -} template<> PROTOBUF_NOINLINE ::SessionProtos::DataExtractionNotification* Arena::CreateMaybeMessage< ::SessionProtos::DataExtractionNotification >(Arena* arena) { return Arena::CreateMessageInternal< ::SessionProtos::DataExtractionNotification >(arena); diff --git a/proto/SessionProtos.pb.h b/proto/SessionProtos.pb.h index ccd3ec2e5..2a539c833 100644 --- a/proto/SessionProtos.pb.h +++ b/proto/SessionProtos.pb.h @@ -103,9 +103,6 @@ extern GroupUpdateMessageDefaultTypeInternal _GroupUpdateMessage_default_instanc class GroupUpdatePromoteMessage; struct GroupUpdatePromoteMessageDefaultTypeInternal; extern GroupUpdatePromoteMessageDefaultTypeInternal _GroupUpdatePromoteMessage_default_instance_; -class KeyPair; -struct KeyPairDefaultTypeInternal; -extern KeyPairDefaultTypeInternal _KeyPair_default_instance_; class LokiProfile; struct LokiProfileDefaultTypeInternal; extern LokiProfileDefaultTypeInternal _LokiProfile_default_instance_; @@ -152,7 +149,6 @@ template<> ::SessionProtos::GroupUpdateMemberLeftMessage* Arena::CreateMaybeMess template<> ::SessionProtos::GroupUpdateMemberLeftNotificationMessage* Arena::CreateMaybeMessage<::SessionProtos::GroupUpdateMemberLeftNotificationMessage>(Arena*); template<> ::SessionProtos::GroupUpdateMessage* Arena::CreateMaybeMessage<::SessionProtos::GroupUpdateMessage>(Arena*); template<> ::SessionProtos::GroupUpdatePromoteMessage* Arena::CreateMaybeMessage<::SessionProtos::GroupUpdatePromoteMessage>(Arena*); -template<> ::SessionProtos::KeyPair* Arena::CreateMaybeMessage<::SessionProtos::KeyPair>(Arena*); template<> ::SessionProtos::LokiProfile* Arena::CreateMaybeMessage<::SessionProtos::LokiProfile>(Arena*); template<> ::SessionProtos::MessageRequestResponse* Arena::CreateMaybeMessage<::SessionProtos::MessageRequestResponse>(Arena*); template<> ::SessionProtos::ProMessage* Arena::CreateMaybeMessage<::SessionProtos::ProMessage>(Arena*); @@ -992,7 +988,8 @@ class UnsendRequest final : enum : int { kAuthorFieldNumber = 2, - kTimestampFieldNumber = 1, + kMsgTimestampFieldNumber = 1, + kMsgIdFieldNumber = 3, }; // required string author = 2; bool has_author() const; @@ -1012,17 +1009,30 @@ class UnsendRequest final : std::string* _internal_mutable_author(); public: - // required uint64 timestamp = 1; - bool has_timestamp() const; + // required uint64 msgTimestamp = 1; + bool has_msgtimestamp() const; private: - bool _internal_has_timestamp() const; + bool _internal_has_msgtimestamp() const; public: - void clear_timestamp(); - uint64_t timestamp() const; - void set_timestamp(uint64_t value); + void clear_msgtimestamp(); + uint64_t msgtimestamp() const; + void set_msgtimestamp(uint64_t value); private: - uint64_t _internal_timestamp() const; - void _internal_set_timestamp(uint64_t value); + uint64_t _internal_msgtimestamp() const; + void _internal_set_msgtimestamp(uint64_t value); + public: + + // optional sfixed64 msgId = 3; + bool has_msgid() const; + private: + bool _internal_has_msgid() const; + public: + void clear_msgid(); + int64_t msgid() const; + void set_msgid(int64_t value); + private: + int64_t _internal_msgid() const; + void _internal_set_msgid(int64_t value); public: // @@protoc_insertion_point(class_scope:SessionProtos.UnsendRequest) @@ -1039,7 +1049,8 @@ class UnsendRequest final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr author_; - uint64_t timestamp_; + uint64_t msgtimestamp_; + int64_t msgid_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_SessionProtos_2eproto; @@ -1379,6 +1390,7 @@ class Content final : kExpirationTypeFieldNumber = 12, kExpirationTimerFieldNumber = 13, kSigTimestampFieldNumber = 15, + kMsgIdFieldNumber = 18, }; // optional bytes proSigForCommunityMessageOnly = 17; bool has_prosigforcommunitymessageonly() const; @@ -1599,6 +1611,19 @@ class Content final : void _internal_set_sigtimestamp(uint64_t value); public: + // optional sfixed64 msgId = 18; + bool has_msgid() const; + private: + bool _internal_has_msgid() const; + public: + void clear_msgid(); + int64_t msgid() const; + void set_msgid(int64_t value); + private: + int64_t _internal_msgid() const; + void _internal_set_msgid(int64_t value); + public: + // @@protoc_insertion_point(class_scope:SessionProtos.Content) private: class _Internal; @@ -1622,6 +1647,7 @@ class Content final : int expirationtype_; uint32_t expirationtimer_; uint64_t sigtimestamp_; + int64_t msgid_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_SessionProtos_2eproto; @@ -1903,176 +1929,6 @@ class CallMessage final : }; // ------------------------------------------------------------------- -class KeyPair final : - public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:SessionProtos.KeyPair) */ { - public: - inline KeyPair() : KeyPair(nullptr) {} - ~KeyPair() override; - explicit PROTOBUF_CONSTEXPR KeyPair(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); - - KeyPair(const KeyPair& from); - KeyPair(KeyPair&& from) noexcept - : KeyPair() { - *this = ::std::move(from); - } - - inline KeyPair& operator=(const KeyPair& from) { - CopyFrom(from); - return *this; - } - inline KeyPair& operator=(KeyPair&& from) noexcept { - if (this == &from) return *this; - if (GetOwningArena() == from.GetOwningArena() - #ifdef PROTOBUF_FORCE_COPY_IN_MOVE - && GetOwningArena() != nullptr - #endif // !PROTOBUF_FORCE_COPY_IN_MOVE - ) { - InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - - inline const std::string& unknown_fields() const { - return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); - } - inline std::string* mutable_unknown_fields() { - return _internal_metadata_.mutable_unknown_fields(); - } - - static const KeyPair& default_instance() { - return *internal_default_instance(); - } - static inline const KeyPair* internal_default_instance() { - return reinterpret_cast( - &_KeyPair_default_instance_); - } - static constexpr int kIndexInFileMessages = - 6; - - friend void swap(KeyPair& a, KeyPair& b) { - a.Swap(&b); - } - inline void Swap(KeyPair* other) { - if (other == this) return; - #ifdef PROTOBUF_FORCE_COPY_IN_SWAP - if (GetOwningArena() != nullptr && - GetOwningArena() == other->GetOwningArena()) { - #else // PROTOBUF_FORCE_COPY_IN_SWAP - if (GetOwningArena() == other->GetOwningArena()) { - #endif // !PROTOBUF_FORCE_COPY_IN_SWAP - InternalSwap(other); - } else { - ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); - } - } - void UnsafeArenaSwap(KeyPair* other) { - if (other == this) return; - GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); - InternalSwap(other); - } - - // implements Message ---------------------------------------------- - - KeyPair* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final { - return CreateMaybeMessage(arena); - } - void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; - void CopyFrom(const KeyPair& from); - void MergeFrom(const KeyPair& from); - PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; - bool IsInitialized() const final; - - size_t ByteSizeLong() const final; - const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; - uint8_t* _InternalSerialize( - uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; - int GetCachedSize() const final { return _impl_._cached_size_.Get(); } - - private: - void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned); - void SharedDtor(); - void SetCachedSize(int size) const; - void InternalSwap(KeyPair* other); - - private: - friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; - static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { - return "SessionProtos.KeyPair"; - } - protected: - explicit KeyPair(::PROTOBUF_NAMESPACE_ID::Arena* arena, - bool is_message_owned = false); - public: - - std::string GetTypeName() const final; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - enum : int { - kPublicKeyFieldNumber = 1, - kPrivateKeyFieldNumber = 2, - }; - // required bytes publicKey = 1; - bool has_publickey() const; - private: - bool _internal_has_publickey() const; - public: - void clear_publickey(); - const std::string& publickey() const; - template - void set_publickey(ArgT0&& arg0, ArgT... args); - std::string* mutable_publickey(); - PROTOBUF_NODISCARD std::string* release_publickey(); - void set_allocated_publickey(std::string* publickey); - private: - const std::string& _internal_publickey() const; - inline PROTOBUF_ALWAYS_INLINE void _internal_set_publickey(const std::string& value); - std::string* _internal_mutable_publickey(); - public: - - // required bytes privateKey = 2; - bool has_privatekey() const; - private: - bool _internal_has_privatekey() const; - public: - void clear_privatekey(); - const std::string& privatekey() const; - template - void set_privatekey(ArgT0&& arg0, ArgT... args); - std::string* mutable_privatekey(); - PROTOBUF_NODISCARD std::string* release_privatekey(); - void set_allocated_privatekey(std::string* privatekey); - private: - const std::string& _internal_privatekey() const; - inline PROTOBUF_ALWAYS_INLINE void _internal_set_privatekey(const std::string& value); - std::string* _internal_mutable_privatekey(); - public: - - // @@protoc_insertion_point(class_scope:SessionProtos.KeyPair) - private: - class _Internal; - - // helper for ByteSizeLong() - size_t RequiredFieldsByteSizeFallback() const; - - template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; - typedef void InternalArenaConstructable_; - typedef void DestructorSkippable_; - struct Impl_ { - ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; - mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; - ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr publickey_; - ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr privatekey_; - }; - union { Impl_ _impl_; }; - friend struct ::TableStruct_SessionProtos_2eproto; -}; -// ------------------------------------------------------------------- - class DataExtractionNotification final : public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:SessionProtos.DataExtractionNotification) */ { public: @@ -2119,7 +1975,7 @@ class DataExtractionNotification final : &_DataExtractionNotification_default_instance_); } static constexpr int kIndexInFileMessages = - 7; + 6; friend void swap(DataExtractionNotification& a, DataExtractionNotification& b) { a.Swap(&b); @@ -2210,6 +2066,9 @@ class DataExtractionNotification final : enum : int { kTimestampFieldNumber = 2, + kMsgTimestampFieldNumber = 3, + kMsgIdFieldNumber = 4, + kAttIndexFieldNumber = 5, kTypeFieldNumber = 1, }; // optional uint64 timestamp = 2; @@ -2225,6 +2084,45 @@ class DataExtractionNotification final : void _internal_set_timestamp(uint64_t value); public: + // optional uint64 msgTimestamp = 3; + bool has_msgtimestamp() const; + private: + bool _internal_has_msgtimestamp() const; + public: + void clear_msgtimestamp(); + uint64_t msgtimestamp() const; + void set_msgtimestamp(uint64_t value); + private: + uint64_t _internal_msgtimestamp() const; + void _internal_set_msgtimestamp(uint64_t value); + public: + + // optional sfixed64 msgId = 4; + bool has_msgid() const; + private: + bool _internal_has_msgid() const; + public: + void clear_msgid(); + int64_t msgid() const; + void set_msgid(int64_t value); + private: + int64_t _internal_msgid() const; + void _internal_set_msgid(int64_t value); + public: + + // optional sint32 attIndex = 5; + bool has_attindex() const; + private: + bool _internal_has_attindex() const; + public: + void clear_attindex(); + int32_t attindex() const; + void set_attindex(int32_t value); + private: + int32_t _internal_attindex() const; + void _internal_set_attindex(int32_t value); + public: + // required .SessionProtos.DataExtractionNotification.Type type = 1; bool has_type() const; private: @@ -2249,6 +2147,9 @@ class DataExtractionNotification final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; uint64_t timestamp_; + uint64_t msgtimestamp_; + int64_t msgid_; + int32_t attindex_; int type_; }; union { Impl_ _impl_; }; @@ -2302,7 +2203,7 @@ class LokiProfile final : &_LokiProfile_default_instance_); } static constexpr int kIndexInFileMessages = - 8; + 7; friend void swap(LokiProfile& a, LokiProfile& b) { a.Swap(&b); @@ -2484,7 +2385,7 @@ class DataMessage_Quote_QuotedAttachment final : &_DataMessage_Quote_QuotedAttachment_default_instance_); } static constexpr int kIndexInFileMessages = - 9; + 8; friend void swap(DataMessage_Quote_QuotedAttachment& a, DataMessage_Quote_QuotedAttachment& b) { a.Swap(&b); @@ -2710,7 +2611,7 @@ class DataMessage_Quote final : &_DataMessage_Quote_default_instance_); } static constexpr int kIndexInFileMessages = - 10; + 9; friend void swap(DataMessage_Quote& a, DataMessage_Quote& b) { a.Swap(&b); @@ -2779,7 +2680,8 @@ class DataMessage_Quote final : kAttachmentsFieldNumber = 4, kAuthorFieldNumber = 2, kTextFieldNumber = 3, - kIdFieldNumber = 1, + kMsgTimestampFieldNumber = 1, + kMsgIdFieldNumber = 5, }; // repeated .SessionProtos.DataMessage.Quote.QuotedAttachment attachments = 4; int attachments_size() const; @@ -2835,17 +2737,30 @@ class DataMessage_Quote final : std::string* _internal_mutable_text(); public: - // required uint64 id = 1; - bool has_id() const; + // required uint64 msgTimestamp = 1; + bool has_msgtimestamp() const; private: - bool _internal_has_id() const; + bool _internal_has_msgtimestamp() const; public: - void clear_id(); - uint64_t id() const; - void set_id(uint64_t value); + void clear_msgtimestamp(); + uint64_t msgtimestamp() const; + void set_msgtimestamp(uint64_t value); private: - uint64_t _internal_id() const; - void _internal_set_id(uint64_t value); + uint64_t _internal_msgtimestamp() const; + void _internal_set_msgtimestamp(uint64_t value); + public: + + // optional sfixed64 msgId = 5; + bool has_msgid() const; + private: + bool _internal_has_msgid() const; + public: + void clear_msgid(); + int64_t msgid() const; + void set_msgid(int64_t value); + private: + int64_t _internal_msgid() const; + void _internal_set_msgid(int64_t value); public: // @@protoc_insertion_point(class_scope:SessionProtos.DataMessage.Quote) @@ -2864,7 +2779,8 @@ class DataMessage_Quote final : ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::SessionProtos::DataMessage_Quote_QuotedAttachment > attachments_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr author_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr text_; - uint64_t id_; + uint64_t msgtimestamp_; + int64_t msgid_; }; union { Impl_ _impl_; }; friend struct ::TableStruct_SessionProtos_2eproto; @@ -2917,7 +2833,7 @@ class DataMessage_Preview final : &_DataMessage_Preview_default_instance_); } static constexpr int kIndexInFileMessages = - 11; + 10; friend void swap(DataMessage_Preview& a, DataMessage_Preview& b) { a.Swap(&b); @@ -3104,7 +3020,7 @@ class DataMessage_Reaction final : &_DataMessage_Reaction_default_instance_); } static constexpr int kIndexInFileMessages = - 12; + 11; friend void swap(DataMessage_Reaction& a, DataMessage_Reaction& b) { a.Swap(&b); @@ -3196,7 +3112,8 @@ class DataMessage_Reaction final : enum : int { kAuthorFieldNumber = 2, kEmojiFieldNumber = 3, - kIdFieldNumber = 1, + kMsgTimestampFieldNumber = 1, + kMsgIdFieldNumber = 5, kActionFieldNumber = 4, }; // required string author = 2; @@ -3235,17 +3152,30 @@ class DataMessage_Reaction final : std::string* _internal_mutable_emoji(); public: - // required uint64 id = 1; - bool has_id() const; + // required uint64 msgTimestamp = 1; + bool has_msgtimestamp() const; private: - bool _internal_has_id() const; + bool _internal_has_msgtimestamp() const; public: - void clear_id(); - uint64_t id() const; - void set_id(uint64_t value); + void clear_msgtimestamp(); + uint64_t msgtimestamp() const; + void set_msgtimestamp(uint64_t value); private: - uint64_t _internal_id() const; - void _internal_set_id(uint64_t value); + uint64_t _internal_msgtimestamp() const; + void _internal_set_msgtimestamp(uint64_t value); + public: + + // optional sfixed64 msgId = 5; + bool has_msgid() const; + private: + bool _internal_has_msgid() const; + public: + void clear_msgid(); + int64_t msgid() const; + void set_msgid(int64_t value); + private: + int64_t _internal_msgid() const; + void _internal_set_msgid(int64_t value); public: // required .SessionProtos.DataMessage.Reaction.Action action = 4; @@ -3276,7 +3206,8 @@ class DataMessage_Reaction final : mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr author_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr emoji_; - uint64_t id_; + uint64_t msgtimestamp_; + int64_t msgid_; int action_; }; union { Impl_ _impl_; }; @@ -3330,7 +3261,7 @@ class DataMessage_OpenGroupInvitation final : &_DataMessage_OpenGroupInvitation_default_instance_); } static constexpr int kIndexInFileMessages = - 13; + 12; friend void swap(DataMessage_OpenGroupInvitation& a, DataMessage_OpenGroupInvitation& b) { a.Swap(&b); @@ -3500,7 +3431,7 @@ class DataMessage final : &_DataMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 13; friend void swap(DataMessage& a, DataMessage& b) { a.Swap(&b); @@ -3901,7 +3832,7 @@ class ReceiptMessage final : &_ReceiptMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 14; friend void swap(ReceiptMessage& a, ReceiptMessage& b) { a.Swap(&b); @@ -3992,6 +3923,7 @@ class ReceiptMessage final : enum : int { kTimestampFieldNumber = 2, + kMsgIdFieldNumber = 3, kTypeFieldNumber = 1, }; // repeated uint64 timestamp = 2; @@ -4016,6 +3948,28 @@ class ReceiptMessage final : ::PROTOBUF_NAMESPACE_ID::RepeatedField< uint64_t >* mutable_timestamp(); + // repeated sfixed64 msgId = 3 [packed = true]; + int msgid_size() const; + private: + int _internal_msgid_size() const; + public: + void clear_msgid(); + private: + int64_t _internal_msgid(int index) const; + const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& + _internal_msgid() const; + void _internal_add_msgid(int64_t value); + ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* + _internal_mutable_msgid(); + public: + int64_t msgid(int index) const; + void set_msgid(int index, int64_t value); + void add_msgid(int64_t value); + const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& + msgid() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* + mutable_msgid(); + // required .SessionProtos.ReceiptMessage.Type type = 1; bool has_type() const; private: @@ -4040,6 +3994,7 @@ class ReceiptMessage final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< uint64_t > timestamp_; + ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t > msgid_; int type_; }; union { Impl_ _impl_; }; @@ -4093,7 +4048,7 @@ class AttachmentPointer final : &_AttachmentPointer_default_instance_); } static constexpr int kIndexInFileMessages = - 16; + 15; friend void swap(AttachmentPointer& a, AttachmentPointer& b) { a.Swap(&b); @@ -4459,7 +4414,7 @@ class SharedConfigMessage final : &_SharedConfigMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 17; + 16; friend void swap(SharedConfigMessage& a, SharedConfigMessage& b) { a.Swap(&b); @@ -4669,7 +4624,7 @@ class GroupUpdateMessage final : &_GroupUpdateMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 18; + 17; friend void swap(GroupUpdateMessage& a, GroupUpdateMessage& b) { a.Swap(&b); @@ -4956,7 +4911,7 @@ class GroupUpdateInviteMessage final : &_GroupUpdateInviteMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 19; + 18; friend void swap(GroupUpdateInviteMessage& a, GroupUpdateInviteMessage& b) { a.Swap(&b); @@ -5166,7 +5121,7 @@ class GroupUpdatePromoteMessage final : &_GroupUpdatePromoteMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 20; + 19; friend void swap(GroupUpdatePromoteMessage& a, GroupUpdatePromoteMessage& b) { a.Swap(&b); @@ -5336,7 +5291,7 @@ class GroupUpdateInfoChangeMessage final : &_GroupUpdateInfoChangeMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 21; + 20; friend void swap(GroupUpdateInfoChangeMessage& a, GroupUpdateInfoChangeMessage& b) { a.Swap(&b); @@ -5564,7 +5519,7 @@ class GroupUpdateMemberChangeMessage final : &_GroupUpdateMemberChangeMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 22; + 21; friend void swap(GroupUpdateMemberChangeMessage& a, GroupUpdateMemberChangeMessage& b) { a.Swap(&b); @@ -5798,7 +5753,7 @@ class GroupUpdateMemberLeftMessage final : &_GroupUpdateMemberLeftMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 23; + 22; friend void swap(GroupUpdateMemberLeftMessage& a, GroupUpdateMemberLeftMessage& b) { a.Swap(&b); @@ -5922,7 +5877,7 @@ class GroupUpdateMemberLeftNotificationMessage final : &_GroupUpdateMemberLeftNotificationMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 24; + 23; friend void swap(GroupUpdateMemberLeftNotificationMessage& a, GroupUpdateMemberLeftNotificationMessage& b) { a.Swap(&b); @@ -6046,7 +6001,7 @@ class GroupUpdateInviteResponseMessage final : &_GroupUpdateInviteResponseMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 25; + 24; friend void swap(GroupUpdateInviteResponseMessage& a, GroupUpdateInviteResponseMessage& b) { a.Swap(&b); @@ -6188,7 +6143,7 @@ class GroupUpdateDeleteMemberContentMessage final : &_GroupUpdateDeleteMemberContentMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 26; + 25; friend void swap(GroupUpdateDeleteMemberContentMessage& a, GroupUpdateDeleteMemberContentMessage& b) { a.Swap(&b); @@ -6387,7 +6342,7 @@ class ProProof final : &_ProProof_default_instance_); } static constexpr int kIndexInFileMessages = - 27; + 26; friend void swap(ProProof& a, ProProof& b) { a.Swap(&b); @@ -6589,7 +6544,7 @@ class ProMessage final : &_ProMessage_default_instance_); } static constexpr int kIndexInFileMessages = - 28; + 27; friend void swap(ProMessage& a, ProMessage& b) { a.Swap(&b); @@ -7111,32 +7066,32 @@ inline void TypingMessage::set_action(::SessionProtos::TypingMessage_Action valu // UnsendRequest -// required uint64 timestamp = 1; -inline bool UnsendRequest::_internal_has_timestamp() const { +// required uint64 msgTimestamp = 1; +inline bool UnsendRequest::_internal_has_msgtimestamp() const { bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; return value; } -inline bool UnsendRequest::has_timestamp() const { - return _internal_has_timestamp(); +inline bool UnsendRequest::has_msgtimestamp() const { + return _internal_has_msgtimestamp(); } -inline void UnsendRequest::clear_timestamp() { - _impl_.timestamp_ = uint64_t{0u}; +inline void UnsendRequest::clear_msgtimestamp() { + _impl_.msgtimestamp_ = uint64_t{0u}; _impl_._has_bits_[0] &= ~0x00000002u; } -inline uint64_t UnsendRequest::_internal_timestamp() const { - return _impl_.timestamp_; +inline uint64_t UnsendRequest::_internal_msgtimestamp() const { + return _impl_.msgtimestamp_; } -inline uint64_t UnsendRequest::timestamp() const { - // @@protoc_insertion_point(field_get:SessionProtos.UnsendRequest.timestamp) - return _internal_timestamp(); +inline uint64_t UnsendRequest::msgtimestamp() const { + // @@protoc_insertion_point(field_get:SessionProtos.UnsendRequest.msgTimestamp) + return _internal_msgtimestamp(); } -inline void UnsendRequest::_internal_set_timestamp(uint64_t value) { +inline void UnsendRequest::_internal_set_msgtimestamp(uint64_t value) { _impl_._has_bits_[0] |= 0x00000002u; - _impl_.timestamp_ = value; + _impl_.msgtimestamp_ = value; } -inline void UnsendRequest::set_timestamp(uint64_t value) { - _internal_set_timestamp(value); - // @@protoc_insertion_point(field_set:SessionProtos.UnsendRequest.timestamp) +inline void UnsendRequest::set_msgtimestamp(uint64_t value) { + _internal_set_msgtimestamp(value); + // @@protoc_insertion_point(field_set:SessionProtos.UnsendRequest.msgTimestamp) } // required string author = 2; @@ -7207,6 +7162,34 @@ inline void UnsendRequest::set_allocated_author(std::string* author) { // @@protoc_insertion_point(field_set_allocated:SessionProtos.UnsendRequest.author) } +// optional sfixed64 msgId = 3; +inline bool UnsendRequest::_internal_has_msgid() const { + bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; + return value; +} +inline bool UnsendRequest::has_msgid() const { + return _internal_has_msgid(); +} +inline void UnsendRequest::clear_msgid() { + _impl_.msgid_ = int64_t{0}; + _impl_._has_bits_[0] &= ~0x00000004u; +} +inline int64_t UnsendRequest::_internal_msgid() const { + return _impl_.msgid_; +} +inline int64_t UnsendRequest::msgid() const { + // @@protoc_insertion_point(field_get:SessionProtos.UnsendRequest.msgId) + return _internal_msgid(); +} +inline void UnsendRequest::_internal_set_msgid(int64_t value) { + _impl_._has_bits_[0] |= 0x00000004u; + _impl_.msgid_ = value; +} +inline void UnsendRequest::set_msgid(int64_t value) { + _internal_set_msgid(value); + // @@protoc_insertion_point(field_set:SessionProtos.UnsendRequest.msgId) +} + // ------------------------------------------------------------------- // MessageRequestResponse @@ -8364,6 +8347,34 @@ inline void Content::set_allocated_prosigforcommunitymessageonly(std::string* pr // @@protoc_insertion_point(field_set_allocated:SessionProtos.Content.proSigForCommunityMessageOnly) } +// optional sfixed64 msgId = 18; +inline bool Content::_internal_has_msgid() const { + bool value = (_impl_._has_bits_[0] & 0x00002000u) != 0; + return value; +} +inline bool Content::has_msgid() const { + return _internal_has_msgid(); +} +inline void Content::clear_msgid() { + _impl_.msgid_ = int64_t{0}; + _impl_._has_bits_[0] &= ~0x00002000u; +} +inline int64_t Content::_internal_msgid() const { + return _impl_.msgid_; +} +inline int64_t Content::msgid() const { + // @@protoc_insertion_point(field_get:SessionProtos.Content.msgId) + return _internal_msgid(); +} +inline void Content::_internal_set_msgid(int64_t value) { + _impl_._has_bits_[0] |= 0x00002000u; + _impl_.msgid_ = value; +} +inline void Content::set_msgid(int64_t value) { + _internal_set_msgid(value); + // @@protoc_insertion_point(field_set:SessionProtos.Content.msgId) +} + // ------------------------------------------------------------------- // CallMessage @@ -8664,151 +8675,11 @@ inline void CallMessage::set_allocated_uuid(std::string* uuid) { // ------------------------------------------------------------------- -// KeyPair - -// required bytes publicKey = 1; -inline bool KeyPair::_internal_has_publickey() const { - bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; - return value; -} -inline bool KeyPair::has_publickey() const { - return _internal_has_publickey(); -} -inline void KeyPair::clear_publickey() { - _impl_.publickey_.ClearToEmpty(); - _impl_._has_bits_[0] &= ~0x00000001u; -} -inline const std::string& KeyPair::publickey() const { - // @@protoc_insertion_point(field_get:SessionProtos.KeyPair.publicKey) - return _internal_publickey(); -} -template -inline PROTOBUF_ALWAYS_INLINE -void KeyPair::set_publickey(ArgT0&& arg0, ArgT... args) { - _impl_._has_bits_[0] |= 0x00000001u; - _impl_.publickey_.SetBytes(static_cast(arg0), args..., GetArenaForAllocation()); - // @@protoc_insertion_point(field_set:SessionProtos.KeyPair.publicKey) -} -inline std::string* KeyPair::mutable_publickey() { - std::string* _s = _internal_mutable_publickey(); - // @@protoc_insertion_point(field_mutable:SessionProtos.KeyPair.publicKey) - return _s; -} -inline const std::string& KeyPair::_internal_publickey() const { - return _impl_.publickey_.Get(); -} -inline void KeyPair::_internal_set_publickey(const std::string& value) { - _impl_._has_bits_[0] |= 0x00000001u; - _impl_.publickey_.Set(value, GetArenaForAllocation()); -} -inline std::string* KeyPair::_internal_mutable_publickey() { - _impl_._has_bits_[0] |= 0x00000001u; - return _impl_.publickey_.Mutable(GetArenaForAllocation()); -} -inline std::string* KeyPair::release_publickey() { - // @@protoc_insertion_point(field_release:SessionProtos.KeyPair.publicKey) - if (!_internal_has_publickey()) { - return nullptr; - } - _impl_._has_bits_[0] &= ~0x00000001u; - auto* p = _impl_.publickey_.Release(); -#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (_impl_.publickey_.IsDefault()) { - _impl_.publickey_.Set("", GetArenaForAllocation()); - } -#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - return p; -} -inline void KeyPair::set_allocated_publickey(std::string* publickey) { - if (publickey != nullptr) { - _impl_._has_bits_[0] |= 0x00000001u; - } else { - _impl_._has_bits_[0] &= ~0x00000001u; - } - _impl_.publickey_.SetAllocated(publickey, GetArenaForAllocation()); -#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (_impl_.publickey_.IsDefault()) { - _impl_.publickey_.Set("", GetArenaForAllocation()); - } -#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - // @@protoc_insertion_point(field_set_allocated:SessionProtos.KeyPair.publicKey) -} - -// required bytes privateKey = 2; -inline bool KeyPair::_internal_has_privatekey() const { - bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; - return value; -} -inline bool KeyPair::has_privatekey() const { - return _internal_has_privatekey(); -} -inline void KeyPair::clear_privatekey() { - _impl_.privatekey_.ClearToEmpty(); - _impl_._has_bits_[0] &= ~0x00000002u; -} -inline const std::string& KeyPair::privatekey() const { - // @@protoc_insertion_point(field_get:SessionProtos.KeyPair.privateKey) - return _internal_privatekey(); -} -template -inline PROTOBUF_ALWAYS_INLINE -void KeyPair::set_privatekey(ArgT0&& arg0, ArgT... args) { - _impl_._has_bits_[0] |= 0x00000002u; - _impl_.privatekey_.SetBytes(static_cast(arg0), args..., GetArenaForAllocation()); - // @@protoc_insertion_point(field_set:SessionProtos.KeyPair.privateKey) -} -inline std::string* KeyPair::mutable_privatekey() { - std::string* _s = _internal_mutable_privatekey(); - // @@protoc_insertion_point(field_mutable:SessionProtos.KeyPair.privateKey) - return _s; -} -inline const std::string& KeyPair::_internal_privatekey() const { - return _impl_.privatekey_.Get(); -} -inline void KeyPair::_internal_set_privatekey(const std::string& value) { - _impl_._has_bits_[0] |= 0x00000002u; - _impl_.privatekey_.Set(value, GetArenaForAllocation()); -} -inline std::string* KeyPair::_internal_mutable_privatekey() { - _impl_._has_bits_[0] |= 0x00000002u; - return _impl_.privatekey_.Mutable(GetArenaForAllocation()); -} -inline std::string* KeyPair::release_privatekey() { - // @@protoc_insertion_point(field_release:SessionProtos.KeyPair.privateKey) - if (!_internal_has_privatekey()) { - return nullptr; - } - _impl_._has_bits_[0] &= ~0x00000002u; - auto* p = _impl_.privatekey_.Release(); -#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (_impl_.privatekey_.IsDefault()) { - _impl_.privatekey_.Set("", GetArenaForAllocation()); - } -#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - return p; -} -inline void KeyPair::set_allocated_privatekey(std::string* privatekey) { - if (privatekey != nullptr) { - _impl_._has_bits_[0] |= 0x00000002u; - } else { - _impl_._has_bits_[0] &= ~0x00000002u; - } - _impl_.privatekey_.SetAllocated(privatekey, GetArenaForAllocation()); -#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING - if (_impl_.privatekey_.IsDefault()) { - _impl_.privatekey_.Set("", GetArenaForAllocation()); - } -#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING - // @@protoc_insertion_point(field_set_allocated:SessionProtos.KeyPair.privateKey) -} - -// ------------------------------------------------------------------- - // DataExtractionNotification // required .SessionProtos.DataExtractionNotification.Type type = 1; inline bool DataExtractionNotification::_internal_has_type() const { - bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000010u) != 0; return value; } inline bool DataExtractionNotification::has_type() const { @@ -8816,7 +8687,7 @@ inline bool DataExtractionNotification::has_type() const { } inline void DataExtractionNotification::clear_type() { _impl_.type_ = 1; - _impl_._has_bits_[0] &= ~0x00000002u; + _impl_._has_bits_[0] &= ~0x00000010u; } inline ::SessionProtos::DataExtractionNotification_Type DataExtractionNotification::_internal_type() const { return static_cast< ::SessionProtos::DataExtractionNotification_Type >(_impl_.type_); @@ -8827,7 +8698,7 @@ inline ::SessionProtos::DataExtractionNotification_Type DataExtractionNotificati } inline void DataExtractionNotification::_internal_set_type(::SessionProtos::DataExtractionNotification_Type value) { assert(::SessionProtos::DataExtractionNotification_Type_IsValid(value)); - _impl_._has_bits_[0] |= 0x00000002u; + _impl_._has_bits_[0] |= 0x00000010u; _impl_.type_ = value; } inline void DataExtractionNotification::set_type(::SessionProtos::DataExtractionNotification_Type value) { @@ -8863,6 +8734,90 @@ inline void DataExtractionNotification::set_timestamp(uint64_t value) { // @@protoc_insertion_point(field_set:SessionProtos.DataExtractionNotification.timestamp) } +// optional uint64 msgTimestamp = 3; +inline bool DataExtractionNotification::_internal_has_msgtimestamp() const { + bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; + return value; +} +inline bool DataExtractionNotification::has_msgtimestamp() const { + return _internal_has_msgtimestamp(); +} +inline void DataExtractionNotification::clear_msgtimestamp() { + _impl_.msgtimestamp_ = uint64_t{0u}; + _impl_._has_bits_[0] &= ~0x00000002u; +} +inline uint64_t DataExtractionNotification::_internal_msgtimestamp() const { + return _impl_.msgtimestamp_; +} +inline uint64_t DataExtractionNotification::msgtimestamp() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataExtractionNotification.msgTimestamp) + return _internal_msgtimestamp(); +} +inline void DataExtractionNotification::_internal_set_msgtimestamp(uint64_t value) { + _impl_._has_bits_[0] |= 0x00000002u; + _impl_.msgtimestamp_ = value; +} +inline void DataExtractionNotification::set_msgtimestamp(uint64_t value) { + _internal_set_msgtimestamp(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataExtractionNotification.msgTimestamp) +} + +// optional sfixed64 msgId = 4; +inline bool DataExtractionNotification::_internal_has_msgid() const { + bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; + return value; +} +inline bool DataExtractionNotification::has_msgid() const { + return _internal_has_msgid(); +} +inline void DataExtractionNotification::clear_msgid() { + _impl_.msgid_ = int64_t{0}; + _impl_._has_bits_[0] &= ~0x00000004u; +} +inline int64_t DataExtractionNotification::_internal_msgid() const { + return _impl_.msgid_; +} +inline int64_t DataExtractionNotification::msgid() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataExtractionNotification.msgId) + return _internal_msgid(); +} +inline void DataExtractionNotification::_internal_set_msgid(int64_t value) { + _impl_._has_bits_[0] |= 0x00000004u; + _impl_.msgid_ = value; +} +inline void DataExtractionNotification::set_msgid(int64_t value) { + _internal_set_msgid(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataExtractionNotification.msgId) +} + +// optional sint32 attIndex = 5; +inline bool DataExtractionNotification::_internal_has_attindex() const { + bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + return value; +} +inline bool DataExtractionNotification::has_attindex() const { + return _internal_has_attindex(); +} +inline void DataExtractionNotification::clear_attindex() { + _impl_.attindex_ = 0; + _impl_._has_bits_[0] &= ~0x00000008u; +} +inline int32_t DataExtractionNotification::_internal_attindex() const { + return _impl_.attindex_; +} +inline int32_t DataExtractionNotification::attindex() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataExtractionNotification.attIndex) + return _internal_attindex(); +} +inline void DataExtractionNotification::_internal_set_attindex(int32_t value) { + _impl_._has_bits_[0] |= 0x00000008u; + _impl_.attindex_ = value; +} +inline void DataExtractionNotification::set_attindex(int32_t value) { + _internal_set_attindex(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataExtractionNotification.attIndex) +} + // ------------------------------------------------------------------- // LokiProfile @@ -9293,32 +9248,32 @@ inline void DataMessage_Quote_QuotedAttachment::set_flags(uint32_t value) { // DataMessage_Quote -// required uint64 id = 1; -inline bool DataMessage_Quote::_internal_has_id() const { +// required uint64 msgTimestamp = 1; +inline bool DataMessage_Quote::_internal_has_msgtimestamp() const { bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; return value; } -inline bool DataMessage_Quote::has_id() const { - return _internal_has_id(); +inline bool DataMessage_Quote::has_msgtimestamp() const { + return _internal_has_msgtimestamp(); } -inline void DataMessage_Quote::clear_id() { - _impl_.id_ = uint64_t{0u}; +inline void DataMessage_Quote::clear_msgtimestamp() { + _impl_.msgtimestamp_ = uint64_t{0u}; _impl_._has_bits_[0] &= ~0x00000004u; } -inline uint64_t DataMessage_Quote::_internal_id() const { - return _impl_.id_; +inline uint64_t DataMessage_Quote::_internal_msgtimestamp() const { + return _impl_.msgtimestamp_; } -inline uint64_t DataMessage_Quote::id() const { - // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Quote.id) - return _internal_id(); +inline uint64_t DataMessage_Quote::msgtimestamp() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Quote.msgTimestamp) + return _internal_msgtimestamp(); } -inline void DataMessage_Quote::_internal_set_id(uint64_t value) { +inline void DataMessage_Quote::_internal_set_msgtimestamp(uint64_t value) { _impl_._has_bits_[0] |= 0x00000004u; - _impl_.id_ = value; + _impl_.msgtimestamp_ = value; } -inline void DataMessage_Quote::set_id(uint64_t value) { - _internal_set_id(value); - // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Quote.id) +inline void DataMessage_Quote::set_msgtimestamp(uint64_t value) { + _internal_set_msgtimestamp(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Quote.msgTimestamp) } // required string author = 2; @@ -9497,6 +9452,34 @@ DataMessage_Quote::attachments() const { return _impl_.attachments_; } +// optional sfixed64 msgId = 5; +inline bool DataMessage_Quote::_internal_has_msgid() const { + bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + return value; +} +inline bool DataMessage_Quote::has_msgid() const { + return _internal_has_msgid(); +} +inline void DataMessage_Quote::clear_msgid() { + _impl_.msgid_ = int64_t{0}; + _impl_._has_bits_[0] &= ~0x00000008u; +} +inline int64_t DataMessage_Quote::_internal_msgid() const { + return _impl_.msgid_; +} +inline int64_t DataMessage_Quote::msgid() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Quote.msgId) + return _internal_msgid(); +} +inline void DataMessage_Quote::_internal_set_msgid(int64_t value) { + _impl_._has_bits_[0] |= 0x00000008u; + _impl_.msgid_ = value; +} +inline void DataMessage_Quote::set_msgid(int64_t value) { + _internal_set_msgid(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Quote.msgId) +} + // ------------------------------------------------------------------- // DataMessage_Preview @@ -9731,32 +9714,32 @@ inline void DataMessage_Preview::set_allocated_image(::SessionProtos::Attachment // DataMessage_Reaction -// required uint64 id = 1; -inline bool DataMessage_Reaction::_internal_has_id() const { +// required uint64 msgTimestamp = 1; +inline bool DataMessage_Reaction::_internal_has_msgtimestamp() const { bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; return value; } -inline bool DataMessage_Reaction::has_id() const { - return _internal_has_id(); +inline bool DataMessage_Reaction::has_msgtimestamp() const { + return _internal_has_msgtimestamp(); } -inline void DataMessage_Reaction::clear_id() { - _impl_.id_ = uint64_t{0u}; +inline void DataMessage_Reaction::clear_msgtimestamp() { + _impl_.msgtimestamp_ = uint64_t{0u}; _impl_._has_bits_[0] &= ~0x00000004u; } -inline uint64_t DataMessage_Reaction::_internal_id() const { - return _impl_.id_; +inline uint64_t DataMessage_Reaction::_internal_msgtimestamp() const { + return _impl_.msgtimestamp_; } -inline uint64_t DataMessage_Reaction::id() const { - // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Reaction.id) - return _internal_id(); +inline uint64_t DataMessage_Reaction::msgtimestamp() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Reaction.msgTimestamp) + return _internal_msgtimestamp(); } -inline void DataMessage_Reaction::_internal_set_id(uint64_t value) { +inline void DataMessage_Reaction::_internal_set_msgtimestamp(uint64_t value) { _impl_._has_bits_[0] |= 0x00000004u; - _impl_.id_ = value; + _impl_.msgtimestamp_ = value; } -inline void DataMessage_Reaction::set_id(uint64_t value) { - _internal_set_id(value); - // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Reaction.id) +inline void DataMessage_Reaction::set_msgtimestamp(uint64_t value) { + _internal_set_msgtimestamp(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Reaction.msgTimestamp) } // required string author = 2; @@ -9897,7 +9880,7 @@ inline void DataMessage_Reaction::set_allocated_emoji(std::string* emoji) { // required .SessionProtos.DataMessage.Reaction.Action action = 4; inline bool DataMessage_Reaction::_internal_has_action() const { - bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000010u) != 0; return value; } inline bool DataMessage_Reaction::has_action() const { @@ -9905,7 +9888,7 @@ inline bool DataMessage_Reaction::has_action() const { } inline void DataMessage_Reaction::clear_action() { _impl_.action_ = 0; - _impl_._has_bits_[0] &= ~0x00000008u; + _impl_._has_bits_[0] &= ~0x00000010u; } inline ::SessionProtos::DataMessage_Reaction_Action DataMessage_Reaction::_internal_action() const { return static_cast< ::SessionProtos::DataMessage_Reaction_Action >(_impl_.action_); @@ -9916,7 +9899,7 @@ inline ::SessionProtos::DataMessage_Reaction_Action DataMessage_Reaction::action } inline void DataMessage_Reaction::_internal_set_action(::SessionProtos::DataMessage_Reaction_Action value) { assert(::SessionProtos::DataMessage_Reaction_Action_IsValid(value)); - _impl_._has_bits_[0] |= 0x00000008u; + _impl_._has_bits_[0] |= 0x00000010u; _impl_.action_ = value; } inline void DataMessage_Reaction::set_action(::SessionProtos::DataMessage_Reaction_Action value) { @@ -9924,6 +9907,34 @@ inline void DataMessage_Reaction::set_action(::SessionProtos::DataMessage_Reacti // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Reaction.action) } +// optional sfixed64 msgId = 5; +inline bool DataMessage_Reaction::_internal_has_msgid() const { + bool value = (_impl_._has_bits_[0] & 0x00000008u) != 0; + return value; +} +inline bool DataMessage_Reaction::has_msgid() const { + return _internal_has_msgid(); +} +inline void DataMessage_Reaction::clear_msgid() { + _impl_.msgid_ = int64_t{0}; + _impl_._has_bits_[0] &= ~0x00000008u; +} +inline int64_t DataMessage_Reaction::_internal_msgid() const { + return _impl_.msgid_; +} +inline int64_t DataMessage_Reaction::msgid() const { + // @@protoc_insertion_point(field_get:SessionProtos.DataMessage.Reaction.msgId) + return _internal_msgid(); +} +inline void DataMessage_Reaction::_internal_set_msgid(int64_t value) { + _impl_._has_bits_[0] |= 0x00000008u; + _impl_.msgid_ = value; +} +inline void DataMessage_Reaction::set_msgid(int64_t value) { + _internal_set_msgid(value); + // @@protoc_insertion_point(field_set:SessionProtos.DataMessage.Reaction.msgId) +} + // ------------------------------------------------------------------- // DataMessage_OpenGroupInvitation @@ -10966,6 +10977,53 @@ ReceiptMessage::mutable_timestamp() { return _internal_mutable_timestamp(); } +// repeated sfixed64 msgId = 3 [packed = true]; +inline int ReceiptMessage::_internal_msgid_size() const { + return _impl_.msgid_.size(); +} +inline int ReceiptMessage::msgid_size() const { + return _internal_msgid_size(); +} +inline void ReceiptMessage::clear_msgid() { + _impl_.msgid_.Clear(); +} +inline int64_t ReceiptMessage::_internal_msgid(int index) const { + return _impl_.msgid_.Get(index); +} +inline int64_t ReceiptMessage::msgid(int index) const { + // @@protoc_insertion_point(field_get:SessionProtos.ReceiptMessage.msgId) + return _internal_msgid(index); +} +inline void ReceiptMessage::set_msgid(int index, int64_t value) { + _impl_.msgid_.Set(index, value); + // @@protoc_insertion_point(field_set:SessionProtos.ReceiptMessage.msgId) +} +inline void ReceiptMessage::_internal_add_msgid(int64_t value) { + _impl_.msgid_.Add(value); +} +inline void ReceiptMessage::add_msgid(int64_t value) { + _internal_add_msgid(value); + // @@protoc_insertion_point(field_add:SessionProtos.ReceiptMessage.msgId) +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& +ReceiptMessage::_internal_msgid() const { + return _impl_.msgid_; +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >& +ReceiptMessage::msgid() const { + // @@protoc_insertion_point(field_list:SessionProtos.ReceiptMessage.msgId) + return _internal_msgid(); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* +ReceiptMessage::_internal_mutable_msgid() { + return &_impl_.msgid_; +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< int64_t >* +ReceiptMessage::mutable_msgid() { + // @@protoc_insertion_point(field_mutable_list:SessionProtos.ReceiptMessage.msgId) + return _internal_mutable_msgid(); +} + // ------------------------------------------------------------------- // AttachmentPointer @@ -13961,8 +14019,6 @@ inline void ProMessage::set_msgbitset(uint64_t value) { // ------------------------------------------------------------------- -// ------------------------------------------------------------------- - // @@protoc_insertion_point(namespace_scope) diff --git a/proto/SessionProtos.proto b/proto/SessionProtos.proto index fde14358a..7694c52f5 100644 --- a/proto/SessionProtos.proto +++ b/proto/SessionProtos.proto @@ -39,10 +39,18 @@ message TypingMessage { } message UnsendRequest { + // The message to delete; see Reaction.msgTimestamp, which this mirrors. + // + // Matching this on the timestamp alone is the most damaging instance of that problem: deleting + // the wrong one of several messages sent in the same millisecond destroys content the user did + // not ask to remove, and those are exactly the messages a user deletes -- someone pasting + // several lines at once and then thinking better of it. Prefer leaving a request unresolved + // over deleting a message you are not sure of. // @required - required uint64 timestamp = 1; + required uint64 msgTimestamp = 1; // @required - required string author = 2; + required string author = 2; + optional sfixed64 msgId = 3; } message MessageRequestResponse { @@ -133,6 +141,33 @@ message Content { // | v3 | Envelope | No | No | Yes | Yes | // +---------+----------------+-------------+------------------+-------------+-------------+ optional bytes proSigForCommunityMessageOnly = 17; + + // 8 random bytes distinguishing this message from another sent by the same account in the same + // millisecond, which `sigTimestamp` alone does not: a client sending several messages at once + // stamps them identically, and clients keying on the timestamp then treat them as one message and + // silently drop all but the first. + // + // Set once, before the message is copied for its recipient and for our own swarm, so every copy + // of a message carries the same value. That makes it the only identifier all parties agree on: + // the two copies differ in `syncTarget`, so they do not hash alike, and no party can derive one + // copy's hash from the other's. + // + // ALWAYS use this together with `sigTimestamp`, never alone. 8 bytes is sized to separate + // messages *within one sender-millisecond*, where a handful is a lot; as a standalone identifier + // over a whole conversation it would start colliding around a few billion messages. Anything + // referring to a message -- a reaction, a quote, a notification about an attachment -- must carry + // both, and match on both. + // + // Absent from messages sent by clients that predate it. A reference to one of those can only be + // matched by timestamp, and is therefore ambiguous exactly when several messages share it. + // + // sfixed64 rather than bytes or a varint type: it is 8 bytes on the wire with no length prefix + // and no varint expansion, which for uniformly random values is the smallest of the three. + // Signed rather than fixed64 only because a signed 64-bit integer is what every implementation + // can hold losslessly -- SQLite, for one, has no unsigned form -- and the two encode identically. + // Treat it as an opaque bit pattern, not a number: nothing about it is ordered or comparable + // beyond equality, and half of all valid values are negative. + optional sfixed64 msgId = 18; } message CallMessage { @@ -157,23 +192,43 @@ message CallMessage { required string uuid = 5; } -message KeyPair { - // @required - required bytes publicKey = 1; - // @required - required bytes privateKey = 2; -} - message DataExtractionNotification { enum Type { SCREENSHOT = 1; - MEDIA_SAVED = 2; // timestamp + MEDIA_SAVED = 2; } // @required required Type type = 1; - optional uint64 timestamp = 2; + + // DEPRECATED: do not write, do not read. Replaced by msgTimestamp and msgId below. + // + // This was meant to be the referenced message's timestamp, but nothing ever said so and the three + // clients disagreed: session-ios set the message's timestamp, session-android set the *current* + // time, and session-desktop set one of several things depending on the path -- the message's, the + // server's, the received-at time, or zero. A receiver had no way to tell which it had been + // given, so no value here could be interpreted, not even to test whether it matched a message + // already held. + optional uint64 timestamp = 2; + + // The message whose media this is about; see Reaction.msgTimestamp, which these mirror. Both are + // needed to resolve one: a timestamp alone is the ambiguity above, and msgId alone is far too + // small to identify a message on its own. + optional uint64 msgTimestamp = 3; + optional sfixed64 msgId = 4; + + // Which of that message's attachments this is about: + // + // >= 0 -- that one, by position in the message's `attachments` list + // -1 -- all of them, saved together, which is what saving from a gallery view does + // absent -- not stated. A SCREENSHOT is about the message rather than any file, and a sender + // predating this field says nothing either way; neither may be read as "the first + // one". + // + // sint32 rather than int32 because -1 is a common value here and proto2 sign-extends a negative + // int32 to ten bytes, where zigzag encodes it in one. + optional sint32 attIndex = 5; } message LokiProfile { @@ -202,12 +257,14 @@ message DataMessage { optional uint32 flags = 4; } + // The message being quoted; see Reaction.msgTimestamp, which this mirrors exactly. // @required - required uint64 id = 1; + required uint64 msgTimestamp = 1; // @required - required string author = 2; - optional string text = 3; - repeated QuotedAttachment attachments = 4; + required string author = 2; + optional string text = 3; + repeated QuotedAttachment attachments = 4; + optional sfixed64 msgId = 5; } message Preview { @@ -222,13 +279,20 @@ message DataMessage { REACT = 0; REMOVE = 1; } + // The message being reacted to: its sigTimestamp, and -- from senders that set one -- its + // msgId. Match on both where both are present; a timestamp alone cannot tell apart messages + // sent in the same millisecond, which is what msgId exists to fix. See Content.msgId. + // + // Named msgTimestamp rather than `id` because it is not one: it is the referenced message's + // time, and calling it `id` next to a field actually called msgId would mislead every reader. // @required - required uint64 id = 1; // Message timestamp + required uint64 msgTimestamp = 1; // @required - required string author = 2; - optional string emoji = 3; + required string author = 2; + optional string emoji = 3; // @required - required Action action = 4; + required Action action = 4; + optional sfixed64 msgId = 5; } message OpenGroupInvitation { @@ -269,7 +333,19 @@ message ReceiptMessage { // @required required Type type = 1; + + // The messages being acknowledged, by their timestamps -- which do not identify them: several + // sent in the same millisecond share one. See Reaction.msgTimestamp. repeated uint64 timestamp = 2; + + // Their msgIds, positionally aligned with `timestamp`: entry i belongs to timestamp i. A batch + // can mix messages that carry a msgId with ones that do not, so entries are never skipped -- 0 + // means "this one has none", which a real msgId is with probability 2^-64. + // + // A receiver that finds these two of different lengths must ignore this field entirely and match + // on timestamps alone, rather than pairing up the prefix: a sender that got the alignment wrong + // has no correct prefix to salvage. + repeated sfixed64 msgId = 3 [packed=true]; } message AttachmentPointer { diff --git a/proto/debug_print.cpp b/proto/debug_print.cpp new file mode 100644 index 000000000..1c5108e7f --- /dev/null +++ b/proto/debug_print.cpp @@ -0,0 +1,826 @@ +// Generated by gen_debug_print.py from the .proto files; do not edit. +// Regenerate with the `regen-protobuf` build target. + +#include "debug_print.hpp" + +#include + +namespace session::proto { + +namespace { + + // Printable ASCII through, everything else escaped: a dump of what is in the field is more use + // here than a guess at what it was meant to say. + std::string quote(std::string_view s) { + std::string out; + out.reserve(s.size() + 2); + out += '"'; + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c >= 0x20 && c < 0x7f) + out += static_cast(c); + else + out += fmt::format("\\x{:02x}", c); + } + } + out += '"'; + return out; + } + + // Summarised past this, because a dump is for reading and a thumbnail is not. + constexpr size_t BYTES_SHOWN = 32; + + std::string hex(std::string_view s) { + auto shown = s.substr(0, BYTES_SHOWN); + return fmt::format( + "({} bytes) {}{}", + s.size(), + oxenc::to_hex(shown.begin(), shown.end()), + s.size() > BYTES_SHOWN ? "..." : ""); + } + + // The field number goes in beside the name because this is a view of the wire, and the number + // is what the wire actually carries -- it is what you compare against a capture, another + // client's output, or the .proto itself. + void + line(std::string& out, int depth, std::string_view name, int number, std::string_view value) { + fmt::format_to( + std::back_inserter(out), "{:{}}{} [{}]: {}\n", "", depth * 2, name, number, value); + } + + // A nested message brackets its fields, the way protobuf's own text format does: the opening + // brace ends the field's line and the closing one sits back at the field's indent. No colon + // before it -- that is what distinguishes "here comes a block" from "here is a value". + void open_msg(std::string& out, int depth, std::string_view name, int number) { + fmt::format_to(std::back_inserter(out), "{:{}}{} [{}] {{\n", "", depth * 2, name, number); + } + + void close_msg(std::string& out, int depth) { + fmt::format_to(std::back_inserter(out), "{:{}}}}\n", "", depth * 2); + } + + // Anything this build does not model. Lite messages keep unknown fields as raw bytes rather + // than discarding them, so we can at least say that something newer arrived and how much of it + // there was + // -- naming it is precisely what the descriptors we do not have would have done. + void unknown(std::string& out, int depth, const std::string& raw) { + if (!raw.empty()) + fmt::format_to( + std::back_inserter(out), + "{:{}}: {}\n", + "", + depth * 2, + hex(raw)); + } + + void print(const SessionProtos::Envelope& m, std::string& out, int depth); + void print(const SessionProtos::TypingMessage& m, std::string& out, int depth); + void print(const SessionProtos::UnsendRequest& m, std::string& out, int depth); + void print(const SessionProtos::MessageRequestResponse& m, std::string& out, int depth); + void print(const SessionProtos::Content& m, std::string& out, int depth); + void print(const SessionProtos::CallMessage& m, std::string& out, int depth); + void print(const SessionProtos::DataExtractionNotification& m, std::string& out, int depth); + void print(const SessionProtos::LokiProfile& m, std::string& out, int depth); + void print(const SessionProtos::DataMessage& m, std::string& out, int depth); + void print(const SessionProtos::DataMessage_Quote& m, std::string& out, int depth); + void + print(const SessionProtos::DataMessage_Quote_QuotedAttachment& m, std::string& out, int depth); + void print(const SessionProtos::DataMessage_Preview& m, std::string& out, int depth); + void print(const SessionProtos::DataMessage_Reaction& m, std::string& out, int depth); + void print( + const SessionProtos::DataMessage_OpenGroupInvitation& m, std::string& out, int depth); + void print(const SessionProtos::ReceiptMessage& m, std::string& out, int depth); + void print(const SessionProtos::AttachmentPointer& m, std::string& out, int depth); + void print(const SessionProtos::SharedConfigMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdateMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdateInviteMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdatePromoteMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdateInfoChangeMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdateMemberChangeMessage& m, std::string& out, int depth); + void print(const SessionProtos::GroupUpdateMemberLeftMessage& m, std::string& out, int depth); + void print( + const SessionProtos::GroupUpdateMemberLeftNotificationMessage& m, + std::string& out, + int depth); + void print( + const SessionProtos::GroupUpdateInviteResponseMessage& m, std::string& out, int depth); + void print( + const SessionProtos::GroupUpdateDeleteMemberContentMessage& m, + std::string& out, + int depth); + void print(const SessionProtos::ProProof& m, std::string& out, int depth); + void print(const SessionProtos::ProMessage& m, std::string& out, int depth); + void print(const WebSocketProtos::WebSocketRequestMessage& m, std::string& out, int depth); + void print(const WebSocketProtos::WebSocketResponseMessage& m, std::string& out, int depth); + void print(const WebSocketProtos::WebSocketMessage& m, std::string& out, int depth); + + void print(const SessionProtos::Envelope& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, Envelope_Type_Name(m.type())); + if (m.has_source()) + line(out, depth, "source", 2, quote(m.source())); + if (m.has_sourcedevice()) + line(out, depth, "sourceDevice", 7, fmt::format("{}", m.sourcedevice())); + if (m.has_timestamp()) + line(out, depth, "timestamp", 5, fmt::format("{}", m.timestamp())); + if (m.has_content()) + line(out, depth, "content", 8, hex(m.content())); + if (m.has_servertimestamp()) + line(out, depth, "serverTimestamp", 10, fmt::format("{}", m.servertimestamp())); + if (m.has_prosig()) + line(out, depth, "proSig", 11, hex(m.prosig())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::TypingMessage& m, std::string& out, int depth) { + if (m.has_timestamp()) + line(out, depth, "timestamp", 1, fmt::format("{}", m.timestamp())); + if (m.has_action()) + line(out, depth, "action", 2, TypingMessage_Action_Name(m.action())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::UnsendRequest& m, std::string& out, int depth) { + if (m.has_msgtimestamp()) + line(out, depth, "msgTimestamp", 1, fmt::format("{}", m.msgtimestamp())); + if (m.has_author()) + line(out, depth, "author", 2, quote(m.author())); + if (m.has_msgid()) + line(out, depth, "msgId", 3, fmt::format("{}", m.msgid())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::MessageRequestResponse& m, std::string& out, int depth) { + if (m.has_isapproved()) + line(out, depth, "isApproved", 1, (m.isapproved() ? "true" : "false")); + if (m.has_profilekey()) + line(out, depth, "profileKey", 2, hex(m.profilekey())); + if (m.has_profile()) { + open_msg(out, depth, "profile", 3); + print(m.profile(), out, depth + 1); + close_msg(out, depth); + } + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::Content& m, std::string& out, int depth) { + if (m.has_datamessage()) { + open_msg(out, depth, "dataMessage", 1); + print(m.datamessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_callmessage()) { + open_msg(out, depth, "callMessage", 3); + print(m.callmessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_receiptmessage()) { + open_msg(out, depth, "receiptMessage", 5); + print(m.receiptmessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_typingmessage()) { + open_msg(out, depth, "typingMessage", 6); + print(m.typingmessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_dataextractionnotification()) { + open_msg(out, depth, "dataExtractionNotification", 8); + print(m.dataextractionnotification(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_unsendrequest()) { + open_msg(out, depth, "unsendRequest", 9); + print(m.unsendrequest(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_messagerequestresponse()) { + open_msg(out, depth, "messageRequestResponse", 10); + print(m.messagerequestresponse(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_sharedconfigmessage()) { + open_msg(out, depth, "sharedConfigMessage", 11); + print(m.sharedconfigmessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_expirationtype()) + line(out, depth, "expirationType", 12, Content_ExpirationType_Name(m.expirationtype())); + if (m.has_expirationtimer()) + line(out, depth, "expirationTimer", 13, fmt::format("{}", m.expirationtimer())); + if (m.has_sigtimestamp()) + line(out, depth, "sigTimestamp", 15, fmt::format("{}", m.sigtimestamp())); + if (m.has_promessage()) { + open_msg(out, depth, "proMessage", 16); + print(m.promessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_prosigforcommunitymessageonly()) + line(out, + depth, + "proSigForCommunityMessageOnly", + 17, + hex(m.prosigforcommunitymessageonly())); + if (m.has_msgid()) + line(out, depth, "msgId", 18, fmt::format("{}", m.msgid())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::CallMessage& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, CallMessage_Type_Name(m.type())); + for (const auto& v : m.sdps()) + line(out, depth, "sdps", 2, quote(v)); + for (const auto& v : m.sdpmlineindexes()) + line(out, depth, "sdpMLineIndexes", 3, fmt::format("{}", v)); + for (const auto& v : m.sdpmids()) + line(out, depth, "sdpMids", 4, quote(v)); + if (m.has_uuid()) + line(out, depth, "uuid", 5, quote(m.uuid())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::DataExtractionNotification& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, DataExtractionNotification_Type_Name(m.type())); + if (m.has_timestamp()) + line(out, depth, "timestamp", 2, fmt::format("{}", m.timestamp())); + if (m.has_msgtimestamp()) + line(out, depth, "msgTimestamp", 3, fmt::format("{}", m.msgtimestamp())); + if (m.has_msgid()) + line(out, depth, "msgId", 4, fmt::format("{}", m.msgid())); + if (m.has_attindex()) + line(out, depth, "attIndex", 5, fmt::format("{}", m.attindex())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::LokiProfile& m, std::string& out, int depth) { + if (m.has_displayname()) + line(out, depth, "displayName", 1, quote(m.displayname())); + if (m.has_profilepicture()) + line(out, depth, "profilePicture", 2, quote(m.profilepicture())); + if (m.has_lastupdateseconds()) + line(out, depth, "lastUpdateSeconds", 3, fmt::format("{}", m.lastupdateseconds())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::DataMessage& m, std::string& out, int depth) { + if (m.has_body()) + line(out, depth, "body", 1, quote(m.body())); + for (const auto& v : m.attachments()) { + open_msg(out, depth, "attachments", 2); + print(v, out, depth + 1); + close_msg(out, depth); + } + if (m.has_flags()) + line(out, depth, "flags", 4, fmt::format("{}", m.flags())); + if (m.has_profilekey()) + line(out, depth, "profileKey", 6, hex(m.profilekey())); + if (m.has_timestamp()) + line(out, depth, "timestamp", 7, fmt::format("{}", m.timestamp())); + if (m.has_quote()) { + open_msg(out, depth, "quote", 8); + print(m.quote(), out, depth + 1); + close_msg(out, depth); + } + for (const auto& v : m.preview()) { + open_msg(out, depth, "preview", 10); + print(v, out, depth + 1); + close_msg(out, depth); + } + if (m.has_reaction()) { + open_msg(out, depth, "reaction", 11); + print(m.reaction(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_profile()) { + open_msg(out, depth, "profile", 101); + print(m.profile(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_opengroupinvitation()) { + open_msg(out, depth, "openGroupInvitation", 102); + print(m.opengroupinvitation(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_synctarget()) + line(out, depth, "syncTarget", 105, quote(m.synctarget())); + if (m.has_blockscommunitymessagerequests()) + line(out, + depth, + "blocksCommunityMessageRequests", + 106, + (m.blockscommunitymessagerequests() ? "true" : "false")); + if (m.has_groupupdatemessage()) { + open_msg(out, depth, "groupUpdateMessage", 120); + print(m.groupupdatemessage(), out, depth + 1); + close_msg(out, depth); + } + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::DataMessage_Quote& m, std::string& out, int depth) { + if (m.has_msgtimestamp()) + line(out, depth, "msgTimestamp", 1, fmt::format("{}", m.msgtimestamp())); + if (m.has_author()) + line(out, depth, "author", 2, quote(m.author())); + if (m.has_text()) + line(out, depth, "text", 3, quote(m.text())); + for (const auto& v : m.attachments()) { + open_msg(out, depth, "attachments", 4); + print(v, out, depth + 1); + close_msg(out, depth); + } + if (m.has_msgid()) + line(out, depth, "msgId", 5, fmt::format("{}", m.msgid())); + unknown(out, depth, m.unknown_fields()); + } + + void + print(const SessionProtos::DataMessage_Quote_QuotedAttachment& m, std::string& out, int depth) { + if (m.has_contenttype()) + line(out, depth, "contentType", 1, quote(m.contenttype())); + if (m.has_filename()) + line(out, depth, "fileName", 2, quote(m.filename())); + if (m.has_thumbnail()) { + open_msg(out, depth, "thumbnail", 3); + print(m.thumbnail(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_flags()) + line(out, depth, "flags", 4, fmt::format("{}", m.flags())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::DataMessage_Preview& m, std::string& out, int depth) { + if (m.has_url()) + line(out, depth, "url", 1, quote(m.url())); + if (m.has_title()) + line(out, depth, "title", 2, quote(m.title())); + if (m.has_image()) { + open_msg(out, depth, "image", 3); + print(m.image(), out, depth + 1); + close_msg(out, depth); + } + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::DataMessage_Reaction& m, std::string& out, int depth) { + if (m.has_msgtimestamp()) + line(out, depth, "msgTimestamp", 1, fmt::format("{}", m.msgtimestamp())); + if (m.has_author()) + line(out, depth, "author", 2, quote(m.author())); + if (m.has_emoji()) + line(out, depth, "emoji", 3, quote(m.emoji())); + if (m.has_action()) + line(out, depth, "action", 4, DataMessage_Reaction_Action_Name(m.action())); + if (m.has_msgid()) + line(out, depth, "msgId", 5, fmt::format("{}", m.msgid())); + unknown(out, depth, m.unknown_fields()); + } + + void print( + const SessionProtos::DataMessage_OpenGroupInvitation& m, std::string& out, int depth) { + if (m.has_url()) + line(out, depth, "url", 1, quote(m.url())); + if (m.has_name()) + line(out, depth, "name", 3, quote(m.name())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::ReceiptMessage& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, ReceiptMessage_Type_Name(m.type())); + for (const auto& v : m.timestamp()) + line(out, depth, "timestamp", 2, fmt::format("{}", v)); + for (const auto& v : m.msgid()) + line(out, depth, "msgId", 3, fmt::format("{}", v)); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::AttachmentPointer& m, std::string& out, int depth) { + if (m.has_id()) + line(out, depth, "id", 1, fmt::format("{}", m.id())); + if (m.has_contenttype()) + line(out, depth, "contentType", 2, quote(m.contenttype())); + if (m.has_key()) + line(out, depth, "key", 3, hex(m.key())); + if (m.has_size()) + line(out, depth, "size", 4, fmt::format("{}", m.size())); + if (m.has_thumbnail()) + line(out, depth, "thumbnail", 5, hex(m.thumbnail())); + if (m.has_digest()) + line(out, depth, "digest", 6, hex(m.digest())); + if (m.has_filename()) + line(out, depth, "fileName", 7, quote(m.filename())); + if (m.has_flags()) + line(out, depth, "flags", 8, fmt::format("{}", m.flags())); + if (m.has_width()) + line(out, depth, "width", 9, fmt::format("{}", m.width())); + if (m.has_height()) + line(out, depth, "height", 10, fmt::format("{}", m.height())); + if (m.has_caption()) + line(out, depth, "caption", 11, quote(m.caption())); + if (m.has_url()) + line(out, depth, "url", 101, quote(m.url())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::SharedConfigMessage& m, std::string& out, int depth) { + if (m.has_kind()) + line(out, depth, "kind", 1, SharedConfigMessage_Kind_Name(m.kind())); + if (m.has_seqno()) + line(out, depth, "seqno", 2, fmt::format("{}", m.seqno())); + if (m.has_data()) + line(out, depth, "data", 3, hex(m.data())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::GroupUpdateMessage& m, std::string& out, int depth) { + if (m.has_invitemessage()) { + open_msg(out, depth, "inviteMessage", 1); + print(m.invitemessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_infochangemessage()) { + open_msg(out, depth, "infoChangeMessage", 2); + print(m.infochangemessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_memberchangemessage()) { + open_msg(out, depth, "memberChangeMessage", 3); + print(m.memberchangemessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_promotemessage()) { + open_msg(out, depth, "promoteMessage", 4); + print(m.promotemessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_memberleftmessage()) { + open_msg(out, depth, "memberLeftMessage", 5); + print(m.memberleftmessage(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_inviteresponse()) { + open_msg(out, depth, "inviteResponse", 6); + print(m.inviteresponse(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_deletemembercontent()) { + open_msg(out, depth, "deleteMemberContent", 7); + print(m.deletemembercontent(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_memberleftnotificationmessage()) { + open_msg(out, depth, "memberLeftNotificationMessage", 8); + print(m.memberleftnotificationmessage(), out, depth + 1); + close_msg(out, depth); + } + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::GroupUpdateInviteMessage& m, std::string& out, int depth) { + if (m.has_groupsessionid()) + line(out, depth, "groupSessionId", 1, quote(m.groupsessionid())); + if (m.has_name()) + line(out, depth, "name", 2, quote(m.name())); + if (m.has_memberauthdata()) + line(out, depth, "memberAuthData", 3, hex(m.memberauthdata())); + if (m.has_adminsignature()) + line(out, depth, "adminSignature", 4, hex(m.adminsignature())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::GroupUpdatePromoteMessage& m, std::string& out, int depth) { + if (m.has_groupidentityseed()) + line(out, depth, "groupIdentitySeed", 1, hex(m.groupidentityseed())); + if (m.has_name()) + line(out, depth, "name", 2, quote(m.name())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::GroupUpdateInfoChangeMessage& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, GroupUpdateInfoChangeMessage_Type_Name(m.type())); + if (m.has_updatedname()) + line(out, depth, "updatedName", 2, quote(m.updatedname())); + if (m.has_updatedexpiration()) + line(out, depth, "updatedExpiration", 3, fmt::format("{}", m.updatedexpiration())); + if (m.has_adminsignature()) + line(out, depth, "adminSignature", 4, hex(m.adminsignature())); + unknown(out, depth, m.unknown_fields()); + } + + void print( + const SessionProtos::GroupUpdateMemberChangeMessage& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, GroupUpdateMemberChangeMessage_Type_Name(m.type())); + for (const auto& v : m.membersessionids()) + line(out, depth, "memberSessionIds", 2, quote(v)); + if (m.has_historyshared()) + line(out, depth, "historyShared", 3, (m.historyshared() ? "true" : "false")); + if (m.has_adminsignature()) + line(out, depth, "adminSignature", 4, hex(m.adminsignature())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::GroupUpdateMemberLeftMessage& m, std::string& out, int depth) { + (void)m; + (void)depth; + unknown(out, depth, m.unknown_fields()); + } + + void print( + const SessionProtos::GroupUpdateMemberLeftNotificationMessage& m, + std::string& out, + int depth) { + (void)m; + (void)depth; + unknown(out, depth, m.unknown_fields()); + } + + void print( + const SessionProtos::GroupUpdateInviteResponseMessage& m, std::string& out, int depth) { + if (m.has_isapproved()) + line(out, depth, "isApproved", 1, (m.isapproved() ? "true" : "false")); + unknown(out, depth, m.unknown_fields()); + } + + void print( + const SessionProtos::GroupUpdateDeleteMemberContentMessage& m, + std::string& out, + int depth) { + for (const auto& v : m.membersessionids()) + line(out, depth, "memberSessionIds", 1, quote(v)); + for (const auto& v : m.messagehashes()) + line(out, depth, "messageHashes", 2, quote(v)); + if (m.has_adminsignature()) + line(out, depth, "adminSignature", 3, hex(m.adminsignature())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::ProProof& m, std::string& out, int depth) { + if (m.has_revocationtag()) + line(out, depth, "revocationTag", 2, hex(m.revocationtag())); + if (m.has_rotatingpublickey()) + line(out, depth, "rotatingPublicKey", 3, hex(m.rotatingpublickey())); + if (m.has_expiryunixts()) + line(out, depth, "expiryUnixTs", 4, fmt::format("{}", m.expiryunixts())); + if (m.has_sig()) + line(out, depth, "sig", 5, hex(m.sig())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const SessionProtos::ProMessage& m, std::string& out, int depth) { + if (m.has_proof()) { + open_msg(out, depth, "proof", 1); + print(m.proof(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_profilebitset()) + line(out, depth, "profileBitset", 2, fmt::format("{}", m.profilebitset())); + if (m.has_msgbitset()) + line(out, depth, "msgBitset", 3, fmt::format("{}", m.msgbitset())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const WebSocketProtos::WebSocketRequestMessage& m, std::string& out, int depth) { + if (m.has_verb()) + line(out, depth, "verb", 1, quote(m.verb())); + if (m.has_path()) + line(out, depth, "path", 2, quote(m.path())); + if (m.has_body()) + line(out, depth, "body", 3, hex(m.body())); + for (const auto& v : m.headers()) + line(out, depth, "headers", 5, quote(v)); + if (m.has_requestid()) + line(out, depth, "requestId", 4, fmt::format("{}", m.requestid())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const WebSocketProtos::WebSocketResponseMessage& m, std::string& out, int depth) { + if (m.has_requestid()) + line(out, depth, "requestId", 1, fmt::format("{}", m.requestid())); + if (m.has_status()) + line(out, depth, "status", 2, fmt::format("{}", m.status())); + if (m.has_message()) + line(out, depth, "message", 3, quote(m.message())); + for (const auto& v : m.headers()) + line(out, depth, "headers", 5, quote(v)); + if (m.has_body()) + line(out, depth, "body", 4, hex(m.body())); + unknown(out, depth, m.unknown_fields()); + } + + void print(const WebSocketProtos::WebSocketMessage& m, std::string& out, int depth) { + if (m.has_type()) + line(out, depth, "type", 1, WebSocketMessage_Type_Name(m.type())); + if (m.has_request()) { + open_msg(out, depth, "request", 2); + print(m.request(), out, depth + 1); + close_msg(out, depth); + } + if (m.has_response()) { + open_msg(out, depth, "response", 3); + print(m.response(), out, depth + 1); + close_msg(out, depth); + } + unknown(out, depth, m.unknown_fields()); + } + +} // namespace + +std::string debug_print(const SessionProtos::Envelope& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::TypingMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::UnsendRequest& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::MessageRequestResponse& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::Content& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::CallMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataExtractionNotification& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::LokiProfile& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage_Quote& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage_Quote_QuotedAttachment& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage_Preview& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage_Reaction& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::DataMessage_OpenGroupInvitation& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::ReceiptMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::AttachmentPointer& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::SharedConfigMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateInviteMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdatePromoteMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateInfoChangeMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateMemberChangeMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateMemberLeftMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateMemberLeftNotificationMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateInviteResponseMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::GroupUpdateDeleteMemberContentMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::ProProof& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const SessionProtos::ProMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const WebSocketProtos::WebSocketRequestMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const WebSocketProtos::WebSocketResponseMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +std::string debug_print(const WebSocketProtos::WebSocketMessage& m) { + std::string out; + print(m, out, 0); + return out; +} + +} // namespace session::proto diff --git a/proto/debug_print.hpp b/proto/debug_print.hpp new file mode 100644 index 000000000..68b6b9568 --- /dev/null +++ b/proto/debug_print.hpp @@ -0,0 +1,55 @@ +// Generated by gen_debug_print.py from the .proto files; do not edit. +// Regenerate with the `regen-protobuf` build target. + +#pragma once + +#include +#include + +#include + +namespace session::proto { + +/// Renders a protobuf message as indented text: one line per field that is set, named, with nested +/// messages indented beneath their field and enums by name rather than number. +/// +/// What `TextFormat` would give you if these schemas were not LITE_RUNTIME. They are, so their +/// generated C++ has no descriptors and no reflection to build this on; it is generated from the +/// same descriptors protoc already emits, and so cannot fall behind the schema. +/// +/// Fields this build does not model are not lost -- lite messages keep them as raw bytes -- but +/// they can only be reported as `` and a length, since naming them is exactly what +/// the missing descriptors would have done. +std::string debug_print(const SessionProtos::Envelope& m); +std::string debug_print(const SessionProtos::TypingMessage& m); +std::string debug_print(const SessionProtos::UnsendRequest& m); +std::string debug_print(const SessionProtos::MessageRequestResponse& m); +std::string debug_print(const SessionProtos::Content& m); +std::string debug_print(const SessionProtos::CallMessage& m); +std::string debug_print(const SessionProtos::DataExtractionNotification& m); +std::string debug_print(const SessionProtos::LokiProfile& m); +std::string debug_print(const SessionProtos::DataMessage& m); +std::string debug_print(const SessionProtos::DataMessage_Quote& m); +std::string debug_print(const SessionProtos::DataMessage_Quote_QuotedAttachment& m); +std::string debug_print(const SessionProtos::DataMessage_Preview& m); +std::string debug_print(const SessionProtos::DataMessage_Reaction& m); +std::string debug_print(const SessionProtos::DataMessage_OpenGroupInvitation& m); +std::string debug_print(const SessionProtos::ReceiptMessage& m); +std::string debug_print(const SessionProtos::AttachmentPointer& m); +std::string debug_print(const SessionProtos::SharedConfigMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateInviteMessage& m); +std::string debug_print(const SessionProtos::GroupUpdatePromoteMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateInfoChangeMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateMemberChangeMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateMemberLeftMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateMemberLeftNotificationMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateInviteResponseMessage& m); +std::string debug_print(const SessionProtos::GroupUpdateDeleteMemberContentMessage& m); +std::string debug_print(const SessionProtos::ProProof& m); +std::string debug_print(const SessionProtos::ProMessage& m); +std::string debug_print(const WebSocketProtos::WebSocketRequestMessage& m); +std::string debug_print(const WebSocketProtos::WebSocketResponseMessage& m); +std::string debug_print(const WebSocketProtos::WebSocketMessage& m); + +} // namespace session::proto diff --git a/proto/gen_debug_print.py b/proto/gen_debug_print.py new file mode 100755 index 000000000..9e56cd091 --- /dev/null +++ b/proto/gen_debug_print.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Generates a text dumper for the protobuf schemas, as calls to the generated accessors. + +Run from the `regen-protobuf` target, alongside regenerating the .pb.cc files; the output is checked +in beside them and nothing in a normal build runs this. + +The reason this is generated rather than written: SessionProtos.proto is LITE_RUNTIME, so its +generated C++ carries no descriptors, and `TextFormat` and a real `DebugString()` -- both of which +are built on reflection -- do not exist to call. Writing the dumper by hand would work and would +silently omit every field added afterwards, which is the one thing a protocol viewer must not do. +Generating it from the same descriptors protoc already produces means it cannot fall behind the +schema: regenerate one and you regenerate the other. + +Usage: protoc --descriptor_set_out=/dev/stdout foo.proto | gen_debug_print.py OUTDIR +""" + +import sys + +from google.protobuf import descriptor_pb2 + +F = descriptor_pb2.FieldDescriptorProto + +# How a field's value becomes text. `{}` is the accessor expression. +SCALAR = { + F.TYPE_STRING: "quote({})", + F.TYPE_BYTES: "hex({})", + F.TYPE_BOOL: '({} ? "true" : "false")', + F.TYPE_DOUBLE: "fmt::format(\"{{}}\", {})", + F.TYPE_FLOAT: "fmt::format(\"{{}}\", {})", +} + + +def cpp_name(type_name, package): + """".SessionProtos.DataMessage.Quote" -> "DataMessage_Quote" (nested types are flattened).""" + stripped = type_name.lstrip(".") + if package and stripped.startswith(package + "."): + stripped = stripped[len(package) + 1 :] + return stripped.replace(".", "_") + + +def collect(msg, prefix, out): + full = f"{prefix}_{msg.name}" if prefix else msg.name + out.append((full, msg)) + for nested in msg.nested_type: + collect(nested, full, out) + + +def value_expr(field, expr, package): + if field.type == F.TYPE_ENUM: + return f"{cpp_name(field.type_name, package)}_Name({expr})" + if field.type in SCALAR: + return SCALAR[field.type].format(expr) + return f'fmt::format("{{}}", {expr})' + + +def emit_field(w, field, package): + # protoc lowercases a field name to make the C++ accessor, so `sourceDevice` is reached as + # `sourcedevice()`. The name as declared is what gets printed; the lowercased one is only ever + # used to call the generated code. + name = field.name.lower() + shown = field.name + num = field.number + repeated = field.label == F.LABEL_REPEATED + is_msg = field.type == F.TYPE_MESSAGE + + if repeated: + if is_msg: + w(f" for (const auto& v : m.{name}()) {{\n") + w(f' open_msg(out, depth, "{shown}", {num});\n') + w(" print(v, out, depth + 1);\n") + w(" close_msg(out, depth);\n") + w(" }\n") + else: + w(f" for (const auto& v : m.{name}())\n") + w(f' line(out, depth, "{shown}", {num},{value_expr(field, "v", package)});\n') + return + + if is_msg: + w(f" if (m.has_{name}()) {{\n") + w(f' open_msg(out, depth, "{shown}", {num});\n') + w(f" print(m.{name}(), out, depth + 1);\n") + w(" close_msg(out, depth);\n") + w(" }\n") + else: + w(f" if (m.has_{name}())\n") + w( + f' line(out, depth, "{shown}", {num},' + f'{value_expr(field, f"m.{name}()", package)});\n' + ) + + +PREAMBLE = '''\ +// Generated by gen_debug_print.py from the .proto files; do not edit. +// Regenerate with the `regen-protobuf` build target. + +#include "debug_print.hpp" + +#include + +namespace session::proto { + +namespace { + +// Printable ASCII through, everything else escaped: a dump of what is in the field is more use here +// than a guess at what it was meant to say. +std::string quote(std::string_view s) { + std::string out; + out.reserve(s.size() + 2); + out += '"'; + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\\\\""; break; + case '\\\\': out += "\\\\\\\\"; break; + case '\\n': out += "\\\\n"; break; + case '\\r': out += "\\\\r"; break; + case '\\t': out += "\\\\t"; break; + default: + if (c >= 0x20 && c < 0x7f) + out += static_cast(c); + else + out += fmt::format("\\\\x{:02x}", c); + } + } + out += '"'; + return out; +} + +// Summarised past this, because a dump is for reading and a thumbnail is not. +constexpr size_t BYTES_SHOWN = 32; + +std::string hex(std::string_view s) { + auto shown = s.substr(0, BYTES_SHOWN); + return fmt::format( + "({} bytes) {}{}", + s.size(), + oxenc::to_hex(shown.begin(), shown.end()), + s.size() > BYTES_SHOWN ? "..." : ""); +} + +// The field number goes in beside the name because this is a view of the wire, and the number is +// what the wire actually carries -- it is what you compare against a capture, another client's +// output, or the .proto itself. +void line(std::string& out, int depth, std::string_view name, int number, std::string_view value) { + fmt::format_to( + std::back_inserter(out), "{:{}}{} [{}]: {}\\n", "", depth * 2, name, number, value); +} + +// A nested message brackets its fields, the way protobuf's own text format does: the opening brace +// ends the field's line and the closing one sits back at the field's indent. No colon before it -- +// that is what distinguishes "here comes a block" from "here is a value". +void open_msg(std::string& out, int depth, std::string_view name, int number) { + fmt::format_to(std::back_inserter(out), "{:{}}{} [{}] {{\\n", "", depth * 2, name, number); +} + +void close_msg(std::string& out, int depth) { + fmt::format_to(std::back_inserter(out), "{:{}}}}\\n", "", depth * 2); +} + +// Anything this build does not model. Lite messages keep unknown fields as raw bytes rather than +// discarding them, so we can at least say that something newer arrived and how much of it there was +// -- naming it is precisely what the descriptors we do not have would have done. +void unknown(std::string& out, int depth, const std::string& raw) { + if (!raw.empty()) + fmt::format_to( + std::back_inserter(out), + "{:{}}: {}\\n", + "", + depth * 2, + hex(raw)); +} + +''' + + +def main(): + outdir = sys.argv[1] if len(sys.argv) > 1 else "." + fds = descriptor_pb2.FileDescriptorSet.FromString(sys.stdin.buffer.read()) + + files = [] + for f in fds.file: + messages = [] + for m in f.message_type: + collect(m, "", messages) + files.append((f, messages)) + + with open(f"{outdir}/debug_print.cpp", "w") as fh: + w = fh.write + w(PREAMBLE) + + for f, messages in files: + for full, _ in messages: + w(f"void print(const {f.package}::{full}& m, std::string& out, int depth);\n") + w("\n") + + for f, messages in files: + for full, msg in messages: + w(f"void print(const {f.package}::{full}& m, std::string& out, int depth) {{\n") + if not msg.field: + w(" (void)m;\n (void)depth;\n") + for field in msg.field: + emit_field(w, field, f.package) + w(" unknown(out, depth, m.unknown_fields());\n") + w("}\n\n") + + w("} // namespace\n\n") + + for f, messages in files: + for full, _ in messages: + w(f"std::string debug_print(const {f.package}::{full}& m) {{\n") + w(" std::string out;\n") + w(" print(m, out, 0);\n") + w(" return out;\n") + w("}\n\n") + + w("} // namespace session::proto\n") + + with open(f"{outdir}/debug_print.hpp", "w") as fh: + w = fh.write + w("// Generated by gen_debug_print.py from the .proto files; do not edit.\n") + w("// Regenerate with the `regen-protobuf` build target.\n\n") + w("#pragma once\n\n") + for f, _ in files: + w(f'#include <{f.name.replace(".proto", ".pb.h")}>\n') + w("\n#include \n\n") + w("namespace session::proto {\n\n") + w("""\ +/// Renders a protobuf message as indented text: one line per field that is set, named, with nested +/// messages indented beneath their field and enums by name rather than number. +/// +/// What `TextFormat` would give you if these schemas were not LITE_RUNTIME. They are, so their +/// generated C++ has no descriptors and no reflection to build this on; it is generated from the +/// same descriptors protoc already emits, and so cannot fall behind the schema. +/// +/// Fields this build does not model are not lost -- lite messages keep them as raw bytes -- but +/// they can only be reported as `` and a length, since naming them is exactly what +/// the missing descriptors would have done. +""") + for f, messages in files: + for full, _ in messages: + w(f"std::string debug_print(const {f.package}::{full}& m);\n") + w("\n} // namespace session::proto\n") + + +if __name__ == "__main__": + main() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e6815bc3a..96c0bd85c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,15 @@ if(WARNINGS_AS_ERRORS) endif() endif() +if(FATAL_MISSING_DECLARATIONS) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(common INTERFACE -Werror=missing-declarations) + message(STATUS "Compiling with -Werror=missing-declarations") + else() + message(WARNING "FATAL_MISSING_DECLARATIONS is not supported by this compiler (${CMAKE_CXX_COMPILER_ID}); ignoring") + endif() +endif() + # -Wunused-parameter isn't in the default warning set (we don't pass -Wextra), so enable it # explicitly. WARNINGS_AS_ERRORS's -Werror (above) then promotes it to an error. if(WARN_UNUSED_PARAMETERS AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") @@ -51,8 +60,10 @@ add_libsession_util_library(util add_libsession_util_library(crypto attachments.cpp blinding.cpp + crypto/ed25519.cpp + crypto/mlkem768.cpp + crypto/x25519.cpp curve25519.cpp - ed25519.cpp hash.cpp multi_encrypt.cpp random.cpp @@ -60,6 +71,7 @@ add_libsession_util_library(crypto session_protocol.cpp sodium_array.cpp xed25519.cpp + xed25519-tweetnacl.cpp pro_backend.cpp types.cpp ) @@ -82,7 +94,6 @@ add_libsession_util_library(config config/pro.cpp config/user_groups.cpp config/user_profile.cpp - fields.cpp ) @@ -91,81 +102,153 @@ target_link_libraries(util PUBLIC common oxen::logging - libzstd::static - simdutf + PRIVATE + sessiondep::libzstd + sessiondep::simdutf ) target_link_libraries(crypto PUBLIC util + session::secure_buffer PRIVATE - libsodium::sodium-internal + sessiondep::libsodium + # AES-CBC for legacy attachment decryption, which libsodium has no equivalent of: it exposes + # AES only as GCM, and only where the hardware supports it. + sessiondep::nettle + sessiondep::simdutf + sessiondep::libutf8proc + mlkem_native::mlkem768 nlohmann_json::nlohmann_json libsession::protos ) +# libsession "Core" for maintaining persistent Session client state +add_libsession_util_library(core + core.cpp + core/component.cpp + core/configs.cpp + core/devices.cpp + core/globals.cpp + core/link_sas.cpp + core/pro.cpp +) +add_subdirectory(core/schema) +add_subdirectory(mnemonics) + +# Conversation-level data model built on top of core. The dependency is one-directional by +# design: core neither knows nor links this. +add_libsession_util_library(client + client/client.cpp + client/conversation.cpp + client/download_cache.cpp + client/conversation_id.cpp +) +add_subdirectory(client/schema) + +target_link_libraries( + client + PUBLIC + core + PRIVATE + libsession::protos +) + +target_link_libraries( + core + PUBLIC + crypto + # core.hpp holds a sqlite::Database by value and takes DatabaseOptions in its constructor, so + # consumers need the SQLiteCpp headers. + session::SQLite + # Core owns the account's config objects and hands them out by reference, so a consumer that + # reads one calls into config itself. + config + PRIVATE + libsession::protos + sessiondep::libsodium + nlohmann_json::nlohmann_json + mlkem_native::mlkem768 + oxen::quic +) + target_link_libraries(config PUBLIC crypto libsession::protos PRIVATE - libsodium::sodium-internal + sessiondep::libsodium +) + +session_dep(libevent_core 2.1) + +add_libsession_util_library(network + onionreq/builder.cpp + onionreq/hop_encryption.cpp + onionreq/parser.cpp + onionreq/response_parser.cpp + network/ip_country/lookup.cpp + network/key_types.cpp + network/network_config.cpp + network/request_queue.cpp + network/service_node.cpp + network/session_network_internal.cpp + network/session_network_types.cpp + network/session_network.cpp + network/snode_pool.cpp + network/swarm.cpp + network/backends/quic_file_client.cpp + network/backends/session_file_server.cpp + network/backends/session_open_group_server.cpp + network/transport/quic_transport.cpp + network/routing/direct_router.cpp + network/routing/onion_request_router.cpp +) + +target_link_libraries(network + PUBLIC + crypto + oxen::quic + PRIVATE + nlohmann_json::nlohmann_json + sessiondep::libsodium + sessiondep::nettle + date::date + sessiondep::libevent_core ) -if(ENABLE_NETWORKING) - # libevent - if(NOT TARGET libevent::core) - add_library(libevent_core INTERFACE) - pkg_check_modules(LIBEVENT_core libevent_core>=2.1 IMPORTED_TARGET REQUIRED) - target_link_libraries(libevent_core INTERFACE PkgConfig::LIBEVENT_core) - add_library(libevent::core ALIAS libevent_core) +# The two databases define the same accessors, so the lookup code is identical either way and only +# the table it searches differs; see src/network/ip_country/data.hpp. +if(WITH_IP_GEOLOCATION) + if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/network/ip_country/data.cpp") + # Generating it needs a ~4.5MB download, so it is neither committed nor fetched during a + # build: ask for it explicitly, once, and it stays until the next refresh. + message(FATAL_ERROR + "WITH_IP_GEOLOCATION is enabled but the database has not been generated yet. Run\n" + " ${PROJECT_SOURCE_DIR}/utils/update-ip-country-db.py\n" + "to download a DB-IP Lite release and generate it, then re-run cmake.") endif() + target_sources(network PRIVATE network/ip_country/data.cpp) +else() + target_sources(network PRIVATE network/ip_country/no_data.cpp) +endif() - add_libsession_util_library(network - onionreq/builder.cpp - onionreq/hop_encryption.cpp - onionreq/parser.cpp - onionreq/response_parser.cpp - network/key_types.cpp - network/network_config.cpp - network/request_queue.cpp - network/service_node.cpp - network/session_network_internal.cpp - network/session_network_types.cpp - network/session_network.cpp - network/snode_pool.cpp - network/swarm.cpp - network/backends/session_file_server.cpp - network/backends/session_open_group_server.cpp - network/transport/quic_transport.cpp - network/routing/direct_router.cpp - network/routing/onion_request_router.cpp - ) - - target_link_libraries(network - PUBLIC - crypto - quic +if(ENABLE_NETWORKING_SROUTER) + target_sources(network PRIVATE - nlohmann_json::nlohmann_json - libsodium::sodium-internal - nettle::nettle - date::date - libevent::core - ) - - if(ENABLE_NETWORKING_SROUTER) - target_sources(network - PRIVATE - network/routing/session_router_router.cpp) - target_link_libraries(network PUBLIC session-router::libsessionrouter) - endif() + network/routing/session_router_router.cpp) + target_link_libraries(network PUBLIC session-router::core) + target_compile_definitions(network PUBLIC ENABLE_NETWORKING_SROUTER) +endif() - if (BUILD_STATIC_DEPS) - target_include_directories(network PUBLIC ${CMAKE_BINARY_DIR}/static-deps/include) - endif() +if (BUILD_STATIC_DEPS) + target_include_directories(network PUBLIC ${CMAKE_BINARY_DIR}/static-deps/include) endif() +# core's public headers expose network::Network and it constructs network::Request, so it +# depends on the network target (defined after core above). +target_link_libraries(core PUBLIC network) + if(WARNINGS_AS_ERRORS AND NOT USE_LTO AND CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION MATCHES "^11\\.") # GCC 11 has an overzealous (and false) stringop-overread warning, but only when LTO is off. # Um, yeah. @@ -216,11 +299,6 @@ target_link_libraries(common INTERFACE version) foreach(tgt ${export_targets}) add_library("libsession::${tgt}" ALIAS "${tgt}") endforeach() -export( - TARGETS ${export_targets} common version - NAMESPACE libsession:: - APPEND FILE libsessionTargets.cmake -) list(APPEND libsession_export_targets ${export_targets}) set(libsession_export_targets "${libsession_export_targets}" PARENT_SCOPE) diff --git a/src/attachments.cpp b/src/attachments.cpp index 29e885be2..ea34c022e 100644 --- a/src/attachments.cpp +++ b/src/attachments.cpp @@ -1,8 +1,13 @@ #include "session/attachments.hpp" +#include +#include +#include +#include +#include +#include #include #include -#include #include #include @@ -12,11 +17,13 @@ #include #include #include +#include #include #include #include #include "internal-util.hpp" +#include "session/hash.hpp" namespace session::attachment { @@ -103,15 +110,14 @@ class decryption_failure : public std::runtime_error { // the randomness. static crypto_secretstream_xchacha20poly1305_state secretstream_xchacha20poly1305_init_push_with_nonce( - std::span header, - std::span key, - std::span nonce) { + std::span header, + std::span key, + std::span nonce) { crypto_secretstream_xchacha20poly1305_state st; std::memcpy(header.data(), nonce.data(), ENCRYPT_HEADER); - crypto_core_hchacha20( - st.k, header.data(), reinterpret_cast(key.data()), nullptr); + crypto_core_hchacha20(st.k, to_unsigned(header.data()), to_unsigned(key.data()), nullptr); static_assert(sizeof(st) == 52); std::memset(st.nonce, 0, 4 /*crypto_secretstream_xchacha20poly1305_COUNTERBYTES*/); st.nonce[0] = 1; @@ -124,262 +130,81 @@ secretstream_xchacha20poly1305_init_push_with_nonce( return st; } -// Encryption implementation function. `get_chunk(N)` returns a pair of [span, bool] of the next N (max ENCRYPT_CHUNK_SIZE) bytes, less than N only at the end of the -// input, where the bool is true if there is at least 1 byte more of data to be retrieved (i.e. -// false means the end of the data). It may not return an empty chunk except for the very first -// call. -template ReadData> -static void encrypt_impl( - std::span out, - size_t data_size, - std::span nonce_key, - ReadData get_chunk) { - size_t padding = encrypted_padding(data_size); - assert(padding >= 1); - size_t padded_size = data_size + padding; - - assert(out.size() == encrypted_size(data_size)); - out[0] = std::byte{'S'}; - - std::span uout{reinterpret_cast(out.data()), out.size()}; - - std::span header{uout.data() + 1, ENCRYPT_HEADER}; - - auto st = secretstream_xchacha20poly1305_init_push_with_nonce( - header, nonce_key.last(), nonce_key.first()); - - auto* outpos = uout.data() + 1 + ENCRYPT_HEADER; - auto* const outend = uout.data() + uout.size(); - - // Now we build a buffer containing padding, plus whatever initial actual data goes on the end - // of the last chunk of padding: - bool done = false; - { - std::vector buf; - buf.reserve(std::min(ENCRYPT_CHUNK_SIZE, padded_size)); - for (size_t padding_remaining = padding; padding_remaining;) { - unsigned char tag = 0; - if (padding_remaining > ENCRYPT_CHUNK_SIZE) { - // Full chunk of 0x00 padding (with more padding in the next chunk) - buf.resize(ENCRYPT_CHUNK_SIZE); - padding_remaining -= ENCRYPT_CHUNK_SIZE; - } else { - buf.resize(padding_remaining - 1); // 0x00 padding - buf.push_back(0x01); // padding terminator - auto [chunk, more] = get_chunk(ENCRYPT_CHUNK_SIZE - padding_remaining); - assert(chunk.size() == ENCRYPT_CHUNK_SIZE - padding_remaining || !more); - if (!chunk.empty()) - buf.insert(buf.end(), chunk.begin(), chunk.end()); - padding_remaining = 0; - if (!more) { - tag = crypto_secretstream_xchacha20poly1305_TAG_FINAL; - done = true; - } - } - - assert(outpos + buf.size() + crypto_secretstream_xchacha20poly1305_ABYTES <= outend); - - unsigned long long out_len; - crypto_secretstream_xchacha20poly1305_push( - &st, outpos, &out_len, buf.data(), buf.size(), nullptr, 0, tag); - assert(out_len == buf.size() + crypto_secretstream_xchacha20poly1305_ABYTES); - outpos += out_len; - } - } - - // Now we're through the initial padding (and probably some initial data): now all we need to do - // is push the rest of the data - - while (!done) { - auto [chunk, more] = get_chunk(ENCRYPT_CHUNK_SIZE); - assert(!chunk.empty()); - assert(chunk.size() == ENCRYPT_CHUNK_SIZE || !more); - assert(outpos + chunk.size() + crypto_secretstream_xchacha20poly1305_ABYTES <= outend); - - unsigned char tag = more ? 0 : crypto_secretstream_xchacha20poly1305_TAG_FINAL; - - unsigned long long out_len; - crypto_secretstream_xchacha20poly1305_push( - &st, outpos, &out_len, chunk.data(), chunk.size(), nullptr, 0, tag); - assert(out_len == chunk.size() + crypto_secretstream_xchacha20poly1305_ABYTES); - outpos += out_len; - if (!more) - done = true; - } -} - -static std::tuple< - std::array, - std::array, - const unsigned char*, - const unsigned char*> -encrypt_buffer_init( +// Helper: creates an Encryptor from in-memory data, encrypts, and writes output into a span. +static std::pair encrypt_to_span( std::span seed, std::span data, Domain domain, + std::span out, bool allow_large) { - std::tuple< - std::array, - std::array, - const unsigned char*, - const unsigned char*> - result; - auto& [nonce_key, key, inpos, inend] = result; - - if (seed.size() < 32) - throw std::invalid_argument{"attachment::encrypt requires a 32-byte uploader seed"}; - - if (data.size() > MAX_REGULAR_SIZE && !allow_large) - throw std::invalid_argument{"data to encrypt is too large"}; - - std::span udata{ - reinterpret_cast(data.data()), data.size()}; - - crypto_generichash_blake2b_state b_st; - const auto domain_byte = static_cast(domain); - crypto_generichash_blake2b_init(&b_st, &domain_byte, 1, nonce_key.size()); - crypto_generichash_blake2b_update( - &b_st, reinterpret_cast(seed.data()), 32); - crypto_generichash_blake2b_update(&b_st, udata.data(), udata.size()); - crypto_generichash_blake2b_final(&b_st, nonce_key.data(), nonce_key.size()); - std::memcpy(key.data(), nonce_key.data() + ENCRYPT_HEADER, ENCRYPT_KEY_SIZE); + Encryptor enc{seed, domain}; + enc.update_key(data); + + size_t pos = 0; + auto key = enc.start_encryption( + [&](std::span buf) -> size_t { + size_t avail = std::min(buf.size(), data.size() - pos); + std::memcpy(buf.data(), data.data() + pos, avail); + pos += avail; + return avail; + }, + allow_large); - inpos = udata.data(); - inend = inpos + udata.size(); + size_t written = 0; + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) { + assert(written + chunk.size() <= out.size()); + std::memcpy(out.data() + written, chunk.data(), chunk.size()); + written += chunk.size(); + } - return result; + return {std::move(key), written}; } -std::array encrypt( +cleared_b32 encrypt( std::span seed, std::span data, Domain domain, std::span out, bool allow_large) { - - auto [nonce_key, key, inpos, inend] = encrypt_buffer_init(seed, data, domain, allow_large); - - encrypt_impl( - out, - data.size(), - nonce_key, - [&inpos, &inend](size_t size) -> std::pair, bool> { - auto* start = inpos; - auto* end = std::min(inpos + size, inend); - inpos = end; - return {{start, end}, inpos != inend}; - }); - - return key; + return encrypt_to_span(seed, data, domain, out, allow_large).first; } -std::pair, std::array> encrypt( +std::pair, cleared_b32> encrypt( std::span seed, std::span data, Domain domain, bool allow_large) { - - if (seed.size() < 32) - throw std::invalid_argument{"attachment::encrypt requires a 32-byte uploader seed"}; - - if (data.size() > MAX_REGULAR_SIZE && !allow_large) - throw std::invalid_argument{"data to encrypt is too large"}; - - std::pair, std::array> result; - auto& [out, key] = result; - - out.resize(encrypted_size(data.size())); - - key = encrypt(seed, data, domain, out, allow_large); - - return result; + std::vector out(encrypted_size(data.size())); + auto key = encrypt_to_span(seed, data, domain, out, allow_large).first; + return {std::move(out), std::move(key)}; } -std::array encrypt( +cleared_b32 encrypt( std::span seed, const std::filesystem::path& file, Domain domain, std::function(size_t enc_size)> make_buffer, bool allow_large) { + auto [enc, key] = Encryptor::from_file(seed, domain, file, allow_large); - if (seed.size() < 32) - throw std::invalid_argument{"attachment::encrypt requires a 32-byte uploader seed"}; + auto out = make_buffer(encrypted_size(enc.data_size())); - std::ifstream in; - in.exceptions(std::ios::badbit); - in.open(file, std::ios::binary | std::ios::ate); - size_t size = in.tellg(); - in.seekg(0, std::ios::beg); - - if (size > MAX_REGULAR_SIZE && !allow_large) - throw std::invalid_argument{"data to encrypt is too large"}; - - size = encrypted_size(size); - - std::array nonce_key; - - crypto_generichash_blake2b_state b_st; - const auto domain_byte = static_cast(domain); - crypto_generichash_blake2b_init(&b_st, &domain_byte, 1, nonce_key.size()); - crypto_generichash_blake2b_update( - &b_st, reinterpret_cast(seed.data()), 32); - - size_t in_size = 0; - std::array chunk; - while (in.read(reinterpret_cast(chunk.data()), chunk.size())) { - crypto_generichash_blake2b_update( - &b_st, reinterpret_cast(chunk.data()), chunk.size()); - in_size += chunk.size(); - } - if (in.gcount() > 0) { - crypto_generichash_blake2b_update( - &b_st, reinterpret_cast(chunk.data()), in.gcount()); - in_size += in.gcount(); + size_t written = 0; + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) { + assert(written + chunk.size() <= out.size()); + std::memcpy(out.data() + written, chunk.data(), chunk.size()); + written += chunk.size(); } - crypto_generichash_blake2b_final(&b_st, nonce_key.data(), nonce_key.size()); - - std::array key; - std::memcpy(key.data(), nonce_key.data() + ENCRYPT_HEADER, ENCRYPT_KEY_SIZE); - - in.clear(); - in.exceptions(std::ios::badbit | std::ios::failbit); - in.seekg(0, std::ios::beg); - - auto encrypted = make_buffer(size); - if (encrypted.size() != size) - throw std::logic_error{ - "make_buffer returned span of invalid size: expected {}, got {}"_format( - size, encrypted.size())}; - - std::array buf; - encrypt_impl( - encrypted, - in_size, - nonce_key, - [&in, &in_size, &buf](size_t size) -> std::pair, bool> { - size_t consumed = in.tellg(); - if (consumed + size > in_size) - size = in_size - consumed; - - if (size > 0) - in.read(reinterpret_cast(buf.data()), size); - - in.peek(); - return {std::span{buf}.first(size), !in.eof()}; - }); - return key; } -std::pair, std::array> encrypt( +std::pair, cleared_b32> encrypt( std::span seed, const std::filesystem::path& file, Domain domain, bool allow_large) { - - std::pair, std::array> result; + std::pair, cleared_b32> result; auto& [encrypted, key] = result; key = encrypt( @@ -395,82 +220,33 @@ std::pair, std::array> encry return result; } -std::array encrypt( +cleared_b32 encrypt( std::span seed, std::span data, Domain domain, const std::filesystem::path& file, bool allow_large) { - - auto [nonce_key, key, inpos, inend] = encrypt_buffer_init(seed, data, domain, allow_large); - - size_t padding = encrypted_padding(data.size()); - assert(padding >= 1); - size_t padded_size = data.size() + padding; + Encryptor enc{seed, domain}; + enc.update_key(data); + + size_t pos = 0; + auto key = enc.start_encryption( + [&](std::span buf) -> size_t { + size_t avail = std::min(buf.size(), data.size() - pos); + std::memcpy(buf.data(), data.data() + pos, avail); + pos += avail; + return avail; + }, + allow_large); try { std::ofstream out; out.exceptions(std::ios::failbit | std::ios::badbit); out.open(file, std::ios::binary | std::ios::trunc); - out.write("S", 1); - - std::array cbuf; - std::span ubuf{reinterpret_cast(cbuf.data()), cbuf.size()}; - - auto st = secretstream_xchacha20poly1305_init_push_with_nonce( - ubuf.first(), - std::span{nonce_key}.last(), - std::span{nonce_key}.first()); - - out.write(cbuf.data(), ENCRYPT_HEADER); - - // Now we build a buffer containing padding, plus whatever initial actual data goes on the - // end of the last chunk of padding, and write those encrypted padding chunks to the file: - { - std::vector buf; - buf.reserve(std::min(ENCRYPT_CHUNK_SIZE, padded_size)); - for (size_t padding_remaining = padding; padding_remaining;) { - if (padding_remaining > ENCRYPT_CHUNK_SIZE) { - // Full chunk of 0x00 padding (with more padding in the next chunk) - buf.resize(ENCRYPT_CHUNK_SIZE); - padding_remaining -= ENCRYPT_CHUNK_SIZE; - } else { - buf.resize(padding_remaining - 1); // 0x00 padding - buf.push_back(0x01); // padding terminator - if (size_t first_data = - std::min(ENCRYPT_CHUNK_SIZE - padding_remaining, data.size())) { - buf.insert(buf.end(), inpos, inpos + first_data); - inpos += first_data; - } - padding_remaining = 0; - } - - unsigned char tag = - inpos < inend ? 0 : crypto_secretstream_xchacha20poly1305_TAG_FINAL; - - unsigned long long out_len; - crypto_secretstream_xchacha20poly1305_push( - &st, ubuf.data(), &out_len, buf.data(), buf.size(), nullptr, 0, tag); - assert(out_len == buf.size() + crypto_secretstream_xchacha20poly1305_ABYTES); - out.write(cbuf.data(), out_len); - } - } - - // Now we're through the initial padding (and probably some initial data): now all we need - // to do is write the rest of the data in chunks - while (inpos < inend) { - auto* chunk_start = inpos; - inpos = std::min(chunk_start + ENCRYPT_CHUNK_SIZE, inend); - unsigned char tag = inpos < inend ? 0 : crypto_secretstream_xchacha20poly1305_TAG_FINAL; - unsigned long long out_len; - crypto_secretstream_xchacha20poly1305_push( - &st, ubuf.data(), &out_len, chunk_start, inpos - chunk_start, nullptr, 0, tag); - assert(out_len == inpos - chunk_start + crypto_secretstream_xchacha20poly1305_ABYTES); - - out.write(cbuf.data(), out_len); - } - } catch (const std::exception& e) { + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) + out.write(reinterpret_cast(chunk.data()), chunk.size()); + } catch (const std::exception&) { std::error_code ec; std::filesystem::remove(file, ec); throw; @@ -497,12 +273,11 @@ size_t decrypt( throw std::logic_error{ "Attachment decryption failed: output buffer too small to decrypt contents"}; - std::span uenc{ - reinterpret_cast(encrypted.data()), encrypted.size()}; + auto uenc = encrypted; crypto_secretstream_xchacha20poly1305_state st; crypto_secretstream_xchacha20poly1305_init_pull( - &st, uenc.data() + 1, reinterpret_cast(key.data())); + &st, to_unsigned(uenc.data() + 1), to_unsigned(key.data())); auto* inpos = uenc.data() + 1 + ENCRYPT_HEADER; auto* const inend = uenc.data() + uenc.size(); @@ -532,7 +307,7 @@ size_t decrypt( reinterpret_cast(padbuf.data()), nullptr, &tag, - inpos, + to_unsigned(inpos), chunk_size + ENCRYPT_CHUNK_OVERHEAD, nullptr, 0) != 0) @@ -591,7 +366,7 @@ size_t decrypt( reinterpret_cast(decrypted), nullptr, &tag, - inpos, + to_unsigned(inpos), chunk_size + ENCRYPT_CHUNK_OVERHEAD, nullptr, 0) != 0) @@ -635,6 +410,160 @@ std::vector decrypt( return result; } +std::vector legacy_display_pic_decrypt( + std::span encrypted, + std::span key) { + + if (encrypted.size() <= LEGACY_DISPLAY_PIC_NONCE_SIZE + LEGACY_DISPLAY_PIC_TAG_SIZE) + throw std::runtime_error{ + "Display picture decryption failed: {} bytes cannot hold a nonce, a tag and any " + "data"_format(encrypted.size())}; + + struct gcm_aes256_ctx ctx; + gcm_aes256_set_key(&ctx, to_unsigned(key.data())); + gcm_aes256_set_iv(&ctx, LEGACY_DISPLAY_PIC_NONCE_SIZE, to_unsigned(encrypted.data())); + + auto body = encrypted.subspan(LEGACY_DISPLAY_PIC_NONCE_SIZE); + auto tag_in = body.subspan(body.size() - LEGACY_DISPLAY_PIC_TAG_SIZE); + body = body.subspan(0, body.size() - LEGACY_DISPLAY_PIC_TAG_SIZE); + + std::vector plaintext(body.size()); + gcm_aes256_decrypt(&ctx, body.size(), to_unsigned(plaintext.data()), to_unsigned(body.data())); + + std::array tag_out; + gcm_aes256_digest(&ctx, tag_out.size(), tag_out.data()); + + // Constant time, as everywhere else a MAC is compared: leaking where two tags first differ is + // what lets an attacker find a valid one a byte at a time. + if (sodium_memcmp(tag_out.data(), tag_in.data(), LEGACY_DISPLAY_PIC_TAG_SIZE) != 0) + throw std::runtime_error{"Display picture decryption failed: bad tag"}; + + return plaintext; +} + +std::vector legacy_decrypt( + std::span encrypted, + std::span key, + // Still taken, and still required to be the right length by callers, because every other + // client needs one sent; see below for why nothing here reads it. + [[maybe_unused]] std::span digest, + size_t unpadded_size) { + + // Bounded by what the file server will actually store, so a caller cannot be talked into + // holding an arbitrary amount by anything a sender claims. This has to be all in memory: the + // authenticators below cover the whole ciphertext, and nothing may be decrypted until they + // pass. + if (encrypted.size() > LEGACY_MAX_ENCRYPTED_SIZE) + throw std::runtime_error{ + "Legacy attachment decryption failed: {}B exceeds the {}B maximum"_format( + encrypted.size(), LEGACY_MAX_ENCRYPTED_SIZE)}; + + if (encrypted.size() < LEGACY_IV_SIZE + AES_BLOCK_SIZE + LEGACY_MAC_SIZE) + throw std::runtime_error{"Legacy attachment decryption failed: encrypted data too short"}; + + auto body_size = encrypted.size() - LEGACY_IV_SIZE - LEGACY_MAC_SIZE; + if (body_size % AES_BLOCK_SIZE != 0) + throw std::runtime_error{ + "Legacy attachment decryption failed: ciphertext is not a whole number of blocks"}; + + auto iv = encrypted.first(); + auto body = encrypted.subspan(LEGACY_IV_SIZE, body_size); + auto mac = encrypted.last(); + // What both authenticators are computed over: everything but the trailing MAC itself. + auto authenticated = encrypted.first(LEGACY_IV_SIZE + body_size); + + std::array expected_mac; + { + hmac_sha256_ctx ctx; + hmac_sha256_set_key(&ctx, 32, to_unsigned(key.data()) + 32); + hmac_sha256_update(&ctx, authenticated.size(), to_unsigned(authenticated.data())); + hmac_sha256_digest(&ctx, expected_mac.size(), to_unsigned(expected_mac.data())); + } + if (!memeql_sec(expected_mac.data(), mac.data(), LEGACY_MAC_SIZE)) + throw std::runtime_error{"Legacy attachment decryption failed: HMAC mismatch"}; + + // Not verified, deliberately. The pointer also carries a SHA-256 over the whole blob -- IV, + // ciphertext and the MAC above -- which every other Session client requires and checks, and + // which is worth nothing here. + // + // The HMAC just verified covers the same bytes under a key that only the sender has. The + // digest is keyless and travels in the same pointer as that key, so it is not a second opinion + // from a second party: anyone able to forge one could forge both, and any alteration that would + // fail it -- substituted file, tampered ciphertext, truncation, a rewritten MAC -- fails the + // HMAC first and never reaches this. It catches nothing the line above did not already catch, + // at the cost of hashing the entire attachment a second time. + // + // It exists because Session inherited Signal's attachment pointer whole, where a digest served + // purposes Session does not have. We still *send* one, and still require the field to be + // present and the right length, because the other clients fail without it -- and iOS does worse + // than fail, writing the ciphertext to disk as though it were the file. + // + // Left here rather than deleted so that the reasoning is visible next to what it is about, and + // so restoring it is a matter of removing comment markers if that reasoning ever turns out to + // be wrong. + // + // std::array expected_digest; + // { + // sha256_ctx ctx; + // sha256_init(&ctx); + // sha256_update(&ctx, encrypted.size(), to_unsigned(encrypted.data())); + // sha256_digest(&ctx, expected_digest.size(), to_unsigned(expected_digest.data())); + // } + // if (!memeql_sec(expected_digest.data(), digest.data(), LEGACY_DIGEST_SIZE)) + // throw std::runtime_error{"Legacy attachment decryption failed: digest mismatch"}; + + std::vector plaintext; + plaintext.resize(body_size); + { + aes256_ctx ctx; + aes256_set_decrypt_key(&ctx, to_unsigned(key.data())); + // cbc_decrypt consumes the IV in place, so it gets a copy rather than the input span. + std::array iv_copy; + std::memcpy(iv_copy.data(), iv.data(), iv_copy.size()); + cbc_decrypt( + &ctx, + reinterpret_cast(aes256_decrypt), + AES_BLOCK_SIZE, + iv_copy.data(), + body_size, + to_unsigned(plaintext.data()), + to_unsigned(body.data())); + } + + // PKCS#7: the last byte is how many padding bytes there are, and every one of them must say so. + // Nettle leaves this to us, unlike the platform APIs the other clients decrypt with. + auto pad = static_cast(plaintext.back()); + if (pad == 0 || pad > AES_BLOCK_SIZE || pad > plaintext.size()) + throw std::runtime_error{"Legacy attachment decryption failed: bad PKCS#7 padding"}; + for (auto it = plaintext.end() - pad; it != plaintext.end(); ++it) + if (static_cast(*it) != pad) + throw std::runtime_error{"Legacy attachment decryption failed: bad PKCS#7 padding"}; + plaintext.resize(plaintext.size() - pad); + + // Session's own zero padding, on top of the block padding, is what `unpadded_size` describes. + // Zero means the sender never said -- only clients older than the field do that -- and the + // padding stays rather than being guessed at. + if (unpadded_size > 0) { + if (unpadded_size > plaintext.size()) + throw std::runtime_error{ + "Legacy attachment decryption failed: claimed size {}B exceeds the {}B decrypted"_format( + unpadded_size, plaintext.size())}; + + // What follows the claimed length is *not* checked, and cannot be. Our own encryptor pads + // with zeroes, but session-android's PaddingInputStream does not: its bulk read reports + // having produced `length` padding bytes without writing anything into the buffer, so the + // padding is whatever the caller's buffer already held. Only its single-byte read() emits + // 0x00, and nothing reads a file a byte at a time. + // + // So an attachment from Android is padded with arbitrary bytes, and requiring zeroes here + // would reject every one of them. Under-reporting therefore still truncates silently; the + // legacy format gives us nothing to catch it with. + plaintext.resize(unpadded_size); + } + + return plaintext; +} + Decryptor::Decryptor( std::span key_, std::function decrypted)> output_) : @@ -911,6 +840,251 @@ void decrypt( } } +// -- Encryptor -- + +static_assert( + sizeof(crypto_generichash_blake2b_state) == 384 && + alignof(crypto_generichash_blake2b_state) == 64, + "blake2b state size/alignment changed; update Encryptor::hash_st_data"); + +static_assert( + sizeof(crypto_secretstream_xchacha20poly1305_state) == 52, + "secretstream state size changed; update Encryptor::ss_st_data"); + +namespace { + using namespace session::literals; + + constexpr auto PERS_ATTACHMENT = "SessionAttachmnt"_b2b_pers; + constexpr auto PERS_PROFILE_PIC = "Session_Prof_Pic"_b2b_pers; + + const auto& domain_pers(Domain domain) { + switch (domain) { + case Domain::ATTACHMENT: return PERS_ATTACHMENT; + case Domain::PROFILE_PIC: return PERS_PROFILE_PIC; + } + throw std::invalid_argument{"Invalid encryption domain"}; + } + + auto& b2b_st(std::byte (&data)[384]) { + return *reinterpret_cast(data); + } + auto& ss_st(std::byte (&data)[52]) { + return *reinterpret_cast(data); + } + auto* uc(std::byte* p) { + return reinterpret_cast(p); + } + auto* uc(const std::byte* p) { + return reinterpret_cast(p); + } +} // namespace + +Encryptor::Encryptor(std::span seed, Domain domain) { + if (seed.size() < 32) + throw std::invalid_argument{"attachment::Encryptor requires a 32-byte uploader seed"}; + + const auto& pers = domain_pers(domain); + crypto_generichash_blake2b_init_salt_personal( + &b2b_st(hash_st_data), uc(seed.data()), 32, nonce_key.size(), nullptr, uc(pers.data())); +} + +Encryptor::Encryptor(std::span key) : key_given{true} { + // The layout the seed-based path derives: nonce first, then key. Here the key is given and the + // nonce is random, because one key encrypts everything we cache and a repeated nonce under a + // repeated key repeats the keystream. + random::fill(std::span{nonce_key.data(), ENCRYPT_HEADER}); + std::memcpy(nonce_key.data() + ENCRYPT_HEADER, key.data(), ENCRYPT_KEY_SIZE); +} + +void Encryptor::update_key(std::span data) { + if (key_given) + throw std::logic_error{"Encryptor::update_key() called on a fixed-key encryptor"}; + if (phase1_done) + throw std::logic_error{"Encryptor::update() called after start_encryption()"}; + + crypto_generichash_blake2b_update(&b2b_st(hash_st_data), uc(data.data()), data.size()); + hashed_size += data.size(); +} + +cleared_b32 Encryptor::start_encryption( + std::function buffer)> src, + bool allow_large, + std::optional enc_size) { + if (phase1_done) + throw std::logic_error{"start_encryption() called twice"}; + phase1_done = true; + + // With a key of our own there is no hash to finalize; nonce_key was filled at construction. + if (!key_given) + crypto_generichash_blake2b_final( + &b2b_st(hash_st_data), uc(nonce_key.data()), nonce_key.size()); + else if (!enc_size) + throw std::invalid_argument{ + "start_encryption() on a fixed-key encryptor requires encrypt_size: nothing " + "hashed the data to know how much is coming"}; + + cleared_b32 key; + std::memcpy(key.data(), nonce_key.data() + ENCRYPT_HEADER, ENCRYPT_KEY_SIZE); + + encrypt_size = enc_size.value_or(hashed_size); + + if (encrypt_size > MAX_REGULAR_SIZE && !allow_large) + throw std::invalid_argument{"data to encrypt is too large"}; + + source = std::move(src); + padding = encrypted_padding(encrypt_size); + padding_remaining = padding; + + // Write 'S' prefix + header into out_buf; initialize secretstream + out_buf[0] = std::byte{'S'}; + ss_st(ss_st_data) = secretstream_xchacha20poly1305_init_push_with_nonce( + std::span{out_buf.data() + 1, ENCRYPT_HEADER}, + std::span{ + nonce_key.data() + ENCRYPT_HEADER, ENCRYPT_KEY_SIZE}, + std::span{nonce_key.data(), ENCRYPT_HEADER}); + out_size = 1 + ENCRYPT_HEADER; + + plaintext_buf.reserve(ENCRYPT_CHUNK_SIZE); + + return key; +} + +bool Encryptor::produce_next() { + if (done) + return false; + if (!source) + throw std::logic_error{"Encryptor::next() requires a data source"}; + + plaintext_buf.clear(); + + size_t need = ENCRYPT_CHUNK_SIZE; + + // Fill with padding first + if (padding_remaining > 0) { + if (padding_remaining > ENCRYPT_CHUNK_SIZE) { + plaintext_buf.resize(ENCRYPT_CHUNK_SIZE, std::byte{0}); + padding_remaining -= ENCRYPT_CHUNK_SIZE; + need = 0; + } else { + plaintext_buf.resize(padding_remaining - 1, std::byte{0}); + plaintext_buf.push_back(std::byte{0x01}); + need = ENCRYPT_CHUNK_SIZE - padding_remaining; + padding_remaining = 0; + } + } + + // Fill the rest from the data source + if (need > 0) { + size_t before = plaintext_buf.size(); + plaintext_buf.resize(before + need); + size_t got = source(std::span{plaintext_buf}.subspan(before, need)); + plaintext_buf.resize(before + got); + encrypted_so_far += got; + + bool source_done = got < need; + + if (encrypted_so_far > encrypt_size) + throw std::runtime_error{ + "Encryptor data source provided too much data: expected {} bytes, got at least {}"_format( + encrypt_size, encrypted_so_far)}; + + if (source_done && encrypted_so_far < encrypt_size) + throw std::runtime_error{ + "Encryptor data source ended prematurely: expected {} bytes, got {}"_format( + encrypt_size, encrypted_so_far)}; + } + + if (plaintext_buf.empty()) { + done = true; + return false; + } + + bool is_final = encrypted_so_far >= encrypt_size && padding_remaining == 0; + unsigned char tag = is_final ? crypto_secretstream_xchacha20poly1305_TAG_FINAL : 0; + + unsigned long long enc_len; + crypto_secretstream_xchacha20poly1305_push( + &ss_st(ss_st_data), + uc(out_buf.data()), + &enc_len, + uc(plaintext_buf.data()), + plaintext_buf.size(), + nullptr, + 0, + tag); + out_size = static_cast(enc_len); + + if (is_final) + done = true; + + return true; +} + +std::span Encryptor::next() { + if (!phase1_done) + throw std::logic_error{"Encryptor::next() called before start_encryption()"}; + + // First call returns the 'S' prefix + header + if (!header_emitted) { + header_emitted = true; + return {out_buf.data(), out_size}; + } + + if (!produce_next()) + return {}; + + return {out_buf.data(), out_size}; +} + +cleared_b32 Encryptor::load_key_from_file( + const std::filesystem::path& file, + bool allow_large, + std::function progress) { + auto in = std::make_shared(); + in->exceptions(std::ios::badbit); + in->open(file, std::ios::binary | std::ios::ate); + int64_t total = in->tellg(); + in->seekg(0, std::ios::beg); + + // Phase 1: hash the file + constexpr size_t READ_SIZE = 65536; + std::vector chunk(READ_SIZE); + int64_t read_so_far = 0; + while (in->read(reinterpret_cast(chunk.data()), chunk.size())) { + update_key(chunk); + read_so_far += chunk.size(); + if (progress) + progress(read_so_far, total); + } + if (in->gcount() > 0) { + update_key(std::span{chunk}.first(in->gcount())); + read_so_far += in->gcount(); + if (progress) + progress(read_so_far, total); + } + + // Seek back for phase 2 + in->clear(); + in->seekg(0, std::ios::beg); + + return start_encryption( + [in](std::span buffer) -> size_t { + in->read(reinterpret_cast(buffer.data()), buffer.size()); + return in->gcount(); + }, + allow_large); +} + +std::pair Encryptor::from_file( + std::span seed, + Domain domain, + const std::filesystem::path& file, + bool allow_large) { + Encryptor enc{seed, domain}; + auto key = enc.load_key_from_file(file, allow_large); + return {std::move(enc), std::move(key)}; +} + } // namespace session::attachment extern "C" { @@ -948,7 +1122,8 @@ LIBSESSION_C_API bool session_attachment_encrypt( sodium_zero_buffer(key.data(), key.size()); return true; } catch (const std::exception& e) { - return set_error(error, e); + copy_c_str(error, 256, e.what()); + return false; } } @@ -972,7 +1147,8 @@ LIBSESSION_C_API bool session_attachment_decrypt( std::span{reinterpret_cast(out), *max_size}); return true; } catch (const std::exception& e) { - return set_error(error, e); + copy_c_str(error, 256, e.what()); + return false; } } @@ -1000,7 +1176,8 @@ LIBSESSION_C_API bool session_attachment_decrypt_alloc( } catch (const std::exception& e) { if (decrypted) std::free(decrypted); - return set_error(error, e); + copy_c_str(error, 256, e.what()); + return false; } } @@ -1034,7 +1211,7 @@ LIBSESSION_C_API size_t session_attachment_encrypt_file( sodium_zero_buffer(key.data(), key.size()); return enc_size; } catch (const std::exception& e) { - set_error(error, e); + copy_c_str(error, 256, e.what()); return 0; } } @@ -1059,7 +1236,7 @@ LIBSESSION_C_API size_t session_attachment_decrypt_file( return std::span{reinterpret_cast(buf), s}; }); } catch (const std::exception& e) { - set_error(error, e); + copy_c_str(error, 256, e.what()); return std::numeric_limits::max(); } } @@ -1079,7 +1256,8 @@ LIBSESSION_C_API bool session_attachment_decrypt_to_file( std::filesystem::path{file_out}); return true; } catch (const std::exception& e) { - return set_error(error, e); + copy_c_str(error, 256, e.what()); + return false; } } @@ -1094,7 +1272,8 @@ LIBSESSION_C_API bool session_attachment_decrypt_file_to_file( std::filesystem::path{file_out}); return true; } catch (const std::exception& e) { - return set_error(error, e); + copy_c_str(error, 256, e.what()); + return false; } } } diff --git a/src/blinding.cpp b/src/blinding.cpp index 1f5d91d85..5c44dbd05 100644 --- a/src/blinding.cpp +++ b/src/blinding.cpp @@ -1,16 +1,17 @@ #include "session/blinding.hpp" +#include #include -#include -#include -#include -#include #include +#include +#include #include -#include "session/ed25519.hpp" +#include "session/blinding.h" +#include "session/crypto/ed25519.hpp" #include "session/export.h" +#include "session/hash.hpp" #include "session/platform.h" #include "session/platform.hpp" #include "session/xed25519.hpp" @@ -18,91 +19,110 @@ namespace session { using namespace std::literals; +using namespace oxen::log::literals; -using uc32 = std::array; -using uc33 = std::array; -using uc64 = std::array; +b32 blind15_factor(std::span server_pk) { + auto blind_hash = hash::blake2b<64>(server_pk); -std::array blind15_factor(std::span server_pk) { - assert(server_pk.size() == 32); - - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, nullptr, 0, 64); - crypto_generichash_blake2b_update(&st, server_pk.data(), server_pk.size()); - uc64 blind_hash; - crypto_generichash_blake2b_final(&st, blind_hash.data(), blind_hash.size()); - - uc32 k; - crypto_core_ed25519_scalar_reduce(k.data(), blind_hash.data()); + b32 k; + ed25519::scalar_reduce(k, blind_hash); return k; } -std::array blind25_factor( - std::span session_id, std::span server_pk) { - assert(session_id.size() == 32 || session_id.size() == 33); - assert(server_pk.size() == 32); +b32 blind25_factor( + std::span session_id, std::span server_pk) { - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, nullptr, 0, 64); - if (session_id.size() == 32) { - constexpr unsigned char prefix = 0x05; - crypto_generichash_blake2b_update(&st, &prefix, 1); - } - crypto_generichash_blake2b_update(&st, session_id.data(), session_id.size()); - crypto_generichash_blake2b_update(&st, server_pk.data(), server_pk.size()); - uc64 blind_hash; - crypto_generichash_blake2b_final(&st, blind_hash.data(), blind_hash.size()); + b64 blind_hash; + if (session_id.size() == 32) + hash::blake2b(blind_hash, "05"_hex_b, session_id, server_pk); + else + hash::blake2b(blind_hash, session_id, server_pk); - uc32 k; - crypto_core_ed25519_scalar_reduce(k.data(), blind_hash.data()); + b32 k; + ed25519::scalar_reduce(k, blind_hash); return k; } namespace { - void blind15_id_impl( - std::span session_id, - std::span server_pk, - unsigned char* out) { - auto k = blind15_factor(server_pk); + void blind_id_impl( + std::span session_id, + std::span blind_factor, + std::span out, + std::byte prefix) { if (session_id.size() == 33) session_id = session_id.subspan(1); - auto ed_pk = xed25519::pubkey(session_id.first<32>()); - if (0 != crypto_scalarmult_ed25519_noclamp(out + 1, k.data(), ed_pk.data())) - throw std::runtime_error{"Cannot blind: invalid session_id (not on main subgroup)"}; - out[0] = 0x15; + if (session_id.size() != 32) + throw std::invalid_argument{"Invalid session id"}; + + ed25519::scalarmult_noclamp( + out.last<32>(), blind_factor, xed25519::pubkey(session_id.first<32>())); + out[0] = prefix; + } + + void blind15_id_impl( + std::span session_id, + std::span server_pk, + std::span out) { + blind_id_impl(session_id, blind15_factor(server_pk), out, std::byte{0x15}); } void blind25_id_impl( - std::span session_id, - std::span server_pk, - unsigned char* out) { - auto k = blind25_factor(session_id, server_pk); - if (session_id.size() == 33) - session_id = session_id.subspan(1); - auto ed_pk = xed25519::pubkey(session_id.first<32>()); - if (0 != crypto_scalarmult_ed25519_noclamp(out + 1, k.data(), ed_pk.data())) - throw std::runtime_error{"Cannot blind: invalid session_id (not on main subgroup)"}; - out[0] = 0x25; + std::span session_id, + std::span server_pk, + std::span out) { + blind_id_impl(session_id, blind25_factor(session_id, server_pk), out, std::byte{0x25}); + } + + // Parses server_pk from either 32 raw bytes or 64 hex digits. + b32 parse_server_pk(std::string_view server_pk_in, std::string_view func_name) { + b32 server_pk; + if (server_pk_in.size() == 32) + std::memcpy(server_pk.data(), server_pk_in.data(), 32); + else if (server_pk_in.size() == 64 && oxenc::is_hex(server_pk_in)) + oxenc::from_hex(server_pk_in.begin(), server_pk_in.end(), server_pk.begin()); + else + throw std::invalid_argument{ + "{}: Invalid server_pk: expected 32 bytes or 64 hex"_format(func_name)}; + return server_pk; + } + + // Common final portion of blind15/blind25 signing: given blinded pubkey A, blinded scalar a, + // nonce r, and message, computes and returns the 64-byte signature. + b64 blinded_sign_finish( + std::span A, + std::span a, + std::span r, + std::span message) { + b64 result; + auto sig_R = std::span{result}.first<32>(); + auto sig_S = std::span{result}.last<32>(); + + ed25519::scalarmult_base_noclamp(sig_R, r); + + b64 hram; + hash::sha512(hram, sig_R, A, message); + + ed25519::scalar_reduce(sig_S, hram); // S = H(R||A||M) + ed25519::scalar_mul(sig_S, sig_S, a); // S = H(R||A||M) a + ed25519::scalar_add(sig_S, sig_S, r); // S = r + H(R||A||M) a + + return result; } } // namespace -std::vector blind15_id( - std::span session_id, std::span server_pk) { +b33 blind15_id(std::span session_id, std::span server_pk) { if (session_id.size() == 33) { - if (session_id[0] != 0x05) + if (session_id[0] != std::byte{0x05}) throw std::invalid_argument{"blind15_id: session_id must start with 0x05"}; session_id = session_id.subspan(1); } else if (session_id.size() != 32) { throw std::invalid_argument{"blind15_id: session_id must be 32 or 33 bytes"}; } - if (server_pk.size() != 32) - throw std::invalid_argument{"blind15_id: server_pk must be 32 bytes"}; - std::vector result; - result.resize(33); - blind15_id_impl(session_id, server_pk, result.data()); + b33 result; + blind15_id_impl(session_id, server_pk, result); return result; } @@ -114,34 +134,30 @@ std::array blind15_id(std::string_view session_id, std::string_v if (server_pk.size() != 64 || !oxenc::is_hex(server_pk)) throw std::invalid_argument{"blind15_id: server_pk must be hex (64 digits)"}; - uc33 raw_sid; + b33 raw_sid; oxenc::from_hex(session_id.begin(), session_id.end(), raw_sid.begin()); - uc32 raw_server_pk; + b32 raw_server_pk; oxenc::from_hex(server_pk.begin(), server_pk.end(), raw_server_pk.begin()); - uc33 blinded; - blind15_id_impl(to_span(raw_sid), to_span(raw_server_pk), blinded.data()); + b33 blinded; + blind15_id_impl(raw_sid, raw_server_pk, blinded); std::array result; - result[0] = oxenc::to_hex(blinded.begin(), blinded.end()); - blinded.back() ^= 0x80; - result[1] = oxenc::to_hex(blinded.begin(), blinded.end()); + result[0] = oxenc::to_hex(blinded); + blinded.back() ^= std::byte{0x80}; + result[1] = oxenc::to_hex(blinded); return result; } -std::vector blind25_id( - std::span session_id, std::span server_pk) { +b33 blind25_id(std::span session_id, std::span server_pk) { if (session_id.size() == 33) { - if (session_id[0] != 0x05) + if (session_id[0] != std::byte{0x05}) throw std::invalid_argument{"blind25_id: session_id must start with 0x05"}; } else if (session_id.size() != 32) { throw std::invalid_argument{"blind25_id: session_id must be 32 or 33 bytes"}; } - if (server_pk.size() != 32) - throw std::invalid_argument{"blind25_id: server_pk must be 32 bytes"}; - std::vector result; - result.resize(33); - blind25_id_impl(session_id, server_pk, result.data()); + b33 result; + blind25_id_impl(session_id, server_pk, result); return result; } @@ -153,378 +169,210 @@ std::string blind25_id(std::string_view session_id, std::string_view server_pk) if (server_pk.size() != 64 || !oxenc::is_hex(server_pk)) throw std::invalid_argument{"blind25_id: server_pk must be hex (64 digits)"}; - uc33 raw_sid; + b33 raw_sid; oxenc::from_hex(session_id.begin(), session_id.end(), raw_sid.begin()); - uc32 raw_server_pk; + b32 raw_server_pk; oxenc::from_hex(server_pk.begin(), server_pk.end(), raw_server_pk.begin()); - uc33 blinded; - blind25_id_impl(to_span(raw_sid), to_span(raw_server_pk), blinded.data()); - return oxenc::to_hex(blinded.begin(), blinded.end()); + b33 blinded; + blind25_id_impl(raw_sid, raw_server_pk, blinded); + return oxenc::to_hex(blinded); } -std::vector blinded15_id_from_ed( - std::span ed_pubkey, - std::span server_pk, - std::vector* session_id) { - if (ed_pubkey.size() != 32) - throw std::invalid_argument{"blind15_id_from_ed: ed_pubkey must be 32 bytes"}; - if (server_pk.size() != 32) - throw std::invalid_argument{"blind15_id_from_ed: server_pk must be 32 bytes"}; - if (session_id && !session_id->empty()) - throw std::invalid_argument{ - "blind15_id_from_ed: session_id pointer must be an empty string"}; - - if (session_id) { - session_id->resize(33); - session_id->front() = 0x05; - if (0 != crypto_sign_ed25519_pk_to_curve25519(session_id->data() + 1, ed_pubkey.data())) - throw std::runtime_error{"ed25519 pubkey to x25519 pubkey conversion failed"}; - } +b33 blinded15_id_from_ed( + std::span ed_pubkey, + std::span server_pk, + std::optional* session_id) { + if (session_id && !session_id->has_value()) + session_id->emplace(ed25519::pk_to_session_id(ed_pubkey)); - std::vector result; - result.resize(33); + b33 result; auto k = blind15_factor(server_pk); - if (0 != crypto_scalarmult_ed25519_noclamp(result.data() + 1, k.data(), ed_pubkey.data())) - throw std::runtime_error{"Cannot blind: invalid session_id (not on main subgroup)"}; - result[0] = 0x15; + ed25519::scalarmult_noclamp(std::span{result.data() + 1, 32}, k, ed_pubkey); + result[0] = std::byte{0x15}; return result; } -std::vector blinded25_id_from_ed( - std::span ed_pubkey, - std::span server_pk, - std::vector* session_id) { - if (ed_pubkey.size() != 32) - throw std::invalid_argument{"blind25_id_from_ed: ed_pubkey must be 32 bytes"}; - if (server_pk.size() != 32) - throw std::invalid_argument{"blind25_id_from_ed: server_pk must be 32 bytes"}; - if (session_id && session_id->size() != 0 && session_id->size() != 33) - throw std::invalid_argument{"blind25_id_from_ed: session_id pointer must be 0 or 33 bytes"}; - - std::vector tmp_session_id; +b33 blinded25_id_from_ed( + std::span ed_pubkey, + std::span server_pk, + std::optional* session_id) { + std::optional tmp_session_id; if (!session_id) session_id = &tmp_session_id; - if (session_id->size() == 0) { - session_id->resize(33); - session_id->front() = 0x05; - if (0 != crypto_sign_ed25519_pk_to_curve25519(session_id->data() + 1, ed_pubkey.data())) - throw std::runtime_error{"ed25519 pubkey to x25519 pubkey conversion failed"}; - } + if (!session_id->has_value()) + session_id->emplace(ed25519::pk_to_session_id(ed_pubkey)); - auto k = blind25_factor(*session_id, server_pk); + auto k = blind25_factor(**session_id, server_pk); - std::vector result; - result.resize(33); + b33 result; // Blinded25 ids are always constructed using the absolute value of the ed pubkey, so if // negative we need to clear the sign bit to make it positive before computing the blinded // pubkey. - uc32 pos_ed_pubkey; - std::memcpy(pos_ed_pubkey.data(), ed_pubkey.data(), 32); - pos_ed_pubkey[31] &= 0x7f; + b32 pos_ed_pubkey; + std::ranges::copy(ed_pubkey, pos_ed_pubkey.begin()); + pos_ed_pubkey[31] &= std::byte{0x7f}; - if (0 != crypto_scalarmult_ed25519_noclamp(result.data() + 1, k.data(), pos_ed_pubkey.data())) - throw std::runtime_error{"Cannot blind: invalid session_id (not on main subgroup)"}; - result[0] = 0x25; + ed25519::scalarmult_noclamp(std::span{result.data() + 1, 32}, k, pos_ed_pubkey); + result[0] = std::byte{0x25}; return result; } -std::pair blind15_key_pair( - std::span ed25519_sk, - std::span server_pk, - uc32* k) { - std::array ed_sk_tmp; - if (ed25519_sk.size() == 32) { - std::array pk_ignore; - crypto_sign_ed25519_seed_keypair(pk_ignore.data(), ed_sk_tmp.data(), ed25519_sk.data()); - ed25519_sk = {ed_sk_tmp.data(), 64}; - } - if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "blind15_key_pair: Invalid ed25519_sk is not the expected 32- or 64-byte value"}; - - if (server_pk.size() != 32) - throw std::invalid_argument{"blind15_key_pair: server_pk must be 32 bytes"}; - - std::pair result; +std::pair blind15_key_pair( + const ed25519::PrivKeySpan& ed25519_sk, std::span server_pk, b32* k) { + std::pair result; auto& [A, a] = result; /// Generate the blinding factor (storing into `*k`, if a pointer was provided) - uc32 k_tmp; + b32 k_tmp; if (!k) k = &k_tmp; *k = blind15_factor(server_pk); - /// Generate a scalar for the private key - if (0 != crypto_sign_ed25519_sk_to_curve25519(a.data(), ed25519_sk.data())) - throw std::runtime_error{ - "blind15_key_pair: Invalid ed25519_sk; conversion to curve25519 seckey failed"}; + // Calculate the private scalar `a` + ed25519::sk_to_private(a, ed25519_sk.seed()); // Turn a, A into their blinded versions - crypto_core_ed25519_scalar_mul(a.data(), k->data(), a.data()); - crypto_scalarmult_ed25519_base_noclamp(A.data(), a.data()); + ed25519::scalar_mul(a, *k, a); + ed25519::scalarmult_base_noclamp(A, a); return result; } -std::pair blind25_key_pair( - std::span ed25519_sk, - std::span server_pk, - uc32* k_prime) { - std::array ed_sk_tmp; - if (ed25519_sk.size() == 32) { - std::array pk_ignore; - crypto_sign_ed25519_seed_keypair(pk_ignore.data(), ed_sk_tmp.data(), ed25519_sk.data()); - ed25519_sk = {ed_sk_tmp.data(), 64}; - } - if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "blind15_key_pair: Invalid ed25519_sk is not the expected 32- or 64-byte value"}; - - if (server_pk.size() != 32) - throw std::invalid_argument{"blind15_key_pair: server_pk must be 32 bytes"}; +std::pair blind25_key_pair( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + b32* k_prime) { + b33 session_id; + session_id[0] = std::byte{0x05}; + ed25519::pk_to_x25519(std::span{session_id}.last<32>(), ed25519_sk.pubkey()); - uc33 session_id; - session_id[0] = 0x05; - if (0 != crypto_sign_ed25519_pk_to_curve25519(session_id.data() + 1, ed25519_sk.data() + 32)) - throw std::runtime_error{ - "blind25_key_pair: Invalid ed25519_sk; conversion to curve25519 pubkey failed"}; - - std::span X{session_id.data() + 1, 32}; + auto X = std::span{session_id}.last<32>(); /// Generate the blinding factor (storing into `*k`, if a pointer was provided) - uc32 k_tmp; + b32 k_tmp; if (!k_prime) k_prime = &k_tmp; - *k_prime = blind25_factor(X, {server_pk.data(), server_pk.size()}); + *k_prime = blind25_factor(X, server_pk); // For a negative pubkey we use k' = -k so that k'A == kA when A is positive, and k'A = -kA = // k|A| when A is negative. - if (*(ed25519_sk.data() + 63) & 0x80) - crypto_core_ed25519_scalar_negate(k_prime->data(), k_prime->data()); + if ((ed25519_sk.pubkey()[31] & std::byte{0x80}) != std::byte{}) + ed25519::scalar_negate(*k_prime, *k_prime); - std::pair result; + std::pair result; auto& [A, a] = result; // Generate the private key (scalar), a; (the sodium function naming here is misleading; this // call actually has nothing to do with conversion to X25519, it just so happens that the // conversion method is the easiest way to get `a` out of libsodium). - if (0 != crypto_sign_ed25519_sk_to_curve25519(a.data(), ed25519_sk.data())) - throw std::runtime_error{ - "blind25_key_pair: Invalid ed25519_sk; conversion to curve25519 seckey failed"}; + a = ed25519::sk_to_x25519(ed25519_sk); // Turn a, A into their blinded versions - crypto_core_ed25519_scalar_mul(a.data(), k_prime->data(), a.data()); - crypto_scalarmult_ed25519_base_noclamp(A.data(), a.data()); + ed25519::scalar_mul(a, *k_prime, a); + ed25519::scalarmult_base_noclamp(A, a); return result; } -static const auto version_blinding_hash_key_sig = to_span("VersionCheckKey_sig"); - -std::pair blind_version_key_pair(std::span ed25519_sk) { - if (ed25519_sk.size() != 32 && ed25519_sk.size() != 64) - throw std::invalid_argument{ - "blind_version_key_pair: Invalid ed25519_sk is not the expected 32- or 64-byte " - "value"}; - - std::pair result; - cleared_uc32 blind_seed; - auto& [pk, sk] = result; - crypto_generichash_blake2b( - blind_seed.data(), - 32, - ed25519_sk.data(), - 32, - version_blinding_hash_key_sig.data(), - version_blinding_hash_key_sig.size()); - - // Reuse `sk` to avoid needing extra secure erasing: - if (0 != crypto_sign_ed25519_seed_keypair(pk.data(), sk.data(), blind_seed.data())) - throw std::runtime_error{"blind_version_key_pair: ed25519 generation from seed failed"}; +static constexpr auto version_blinding_hash_key_sig = "VersionCheckKey_sig"_bytes; - return result; +std::pair blind_version_key_pair(const ed25519::PrivKeySpan& ed25519_sk) { + cleared_b32 blind_seed; + hash::blake2b_key(blind_seed, version_blinding_hash_key_sig, ed25519_sk.seed()); + return ed25519::keypair(blind_seed); } -static const auto hash_key_seed = to_span("SessCommBlind25_seed"); -static const auto hash_key_sig = to_span("SessCommBlind25_sig"); +static constexpr auto hash_key_seed = "SessCommBlind25_seed"_bytes; +static constexpr auto hash_key_sig = "SessCommBlind25_sig"_bytes; -std::vector blind25_sign( - std::span ed25519_sk, - std::string_view server_pk_in, - std::span message) { - std::array ed_sk_tmp; - if (ed25519_sk.size() == 32) { - std::array pk_ignore; - crypto_sign_ed25519_seed_keypair(pk_ignore.data(), ed_sk_tmp.data(), ed25519_sk.data()); - ed25519_sk = {ed_sk_tmp.data(), 64}; - } - if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "blind25_sign: Invalid ed25519_sk is not the expected 32- or 64-byte value"}; - uc32 server_pk; - if (server_pk_in.size() == 32) - std::memcpy(server_pk.data(), server_pk_in.data(), 32); - else if (server_pk_in.size() == 64 && oxenc::is_hex(server_pk_in)) - oxenc::from_hex(server_pk_in.begin(), server_pk_in.end(), server_pk.begin()); - else - throw std::invalid_argument{"blind25_sign: Invalid server_pk: expected 32 bytes or 64 hex"}; - - auto [A, a] = blind25_key_pair(ed25519_sk, to_span(server_pk)); - - uc32 seedhash; - crypto_generichash_blake2b( - seedhash.data(), - seedhash.size(), - ed25519_sk.data(), - 32, - hash_key_seed.data(), - hash_key_seed.size()); - - uc64 r_hash; - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, hash_key_sig.data(), hash_key_sig.size(), r_hash.size()); - crypto_generichash_blake2b_update(&st, seedhash.data(), seedhash.size()); - crypto_generichash_blake2b_update(&st, A.data(), A.size()); - crypto_generichash_blake2b_update(&st, message.data(), message.size()); - crypto_generichash_blake2b_final(&st, r_hash.data(), r_hash.size()); - - uc32 r; - crypto_core_ed25519_scalar_reduce(r.data(), r_hash.data()); - - std::vector result; - result.resize(64); - auto* sig_R = result.data(); - auto* sig_S = result.data() + 32; - crypto_scalarmult_ed25519_base_noclamp(sig_R, r.data()); - - crypto_hash_sha512_state st2; - crypto_hash_sha512_init(&st2); - crypto_hash_sha512_update(&st2, sig_R, 32); - crypto_hash_sha512_update(&st2, A.data(), A.size()); - crypto_hash_sha512_update(&st2, message.data(), message.size()); - uc64 hram; - crypto_hash_sha512_final(&st2, hram.data()); - - crypto_core_ed25519_scalar_reduce(sig_S, hram.data()); // S = H(R||A||M) - - crypto_core_ed25519_scalar_mul(sig_S, sig_S, a.data()); // S = H(R||A||M) a - crypto_core_ed25519_scalar_add(sig_S, sig_S, r.data()); // S = r + H(R||A||M) a +b64 blind25_sign( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + std::span message) { + auto [A, a] = blind25_key_pair(ed25519_sk, server_pk); - return result; + b32 seedhash; + hash::blake2b_key(seedhash, hash_key_seed, ed25519_sk.seed()); + + b64 r_hash; + hash::blake2b_key(r_hash, hash_key_sig, seedhash, A, message); + + b32 r; + ed25519::scalar_reduce(r, r_hash); + + return blinded_sign_finish(A, a, r, message); } -std::vector blind15_sign( - std::span ed25519_sk, +b64 blind25_sign( + const ed25519::PrivKeySpan& ed25519_sk, std::string_view server_pk_in, - std::span message) { - std::array ed_sk_tmp; - if (ed25519_sk.size() == 32) { - std::array pk_ignore; - crypto_sign_ed25519_seed_keypair(pk_ignore.data(), ed_sk_tmp.data(), ed25519_sk.data()); - ed25519_sk = {ed_sk_tmp.data(), 64}; - } - if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "blind15_sign: Invalid ed25519_sk is not the expected 32- or 64-byte value"}; - - uc32 server_pk; - if (server_pk_in.size() == 32) - std::memcpy(server_pk.data(), server_pk_in.data(), 32); - else if (server_pk_in.size() == 64 && oxenc::is_hex(server_pk_in)) - oxenc::from_hex(server_pk_in.begin(), server_pk_in.end(), server_pk.begin()); - else - throw std::invalid_argument{"blind15_sign: Invalid server_pk: expected 32 bytes or 64 hex"}; + std::span message) { + return blind25_sign(ed25519_sk, parse_server_pk(server_pk_in, "blind25_sign"), message); +} - auto [blind_15_pk, blind_15_sk] = blind15_key_pair(ed25519_sk, {server_pk.data(), 32}); +b64 blind15_sign( + const ed25519::PrivKeySpan& ed25519_sk, + std::span server_pk, + std::span message) { + auto [blind_15_pk, blind_15_sk] = blind15_key_pair(ed25519_sk, server_pk); // H_rh = sha512(s.encode()).digest()[32:] - uc64 hrh; - crypto_hash_sha512_state st1; - crypto_hash_sha512_init(&st1); - crypto_hash_sha512_update(&st1, ed25519_sk.data(), 64); - crypto_hash_sha512_final(&st1, hrh.data()); + b64 hrh; + hash::sha512(hrh, ed25519_sk); // r = salt.crypto_core_ed25519_scalar_reduce(sha512_multipart(H_rh, kA, message_parts)) - auto hrh_suffix = hrh.data() + 32; - uc32 r; - uc64 r_hash; - crypto_hash_sha512_state st2; - crypto_hash_sha512_init(&st2); - crypto_hash_sha512_update(&st2, hrh_suffix, 32); - crypto_hash_sha512_update(&st2, blind_15_pk.data(), blind_15_pk.size()); - crypto_hash_sha512_update(&st2, message.data(), message.size()); - crypto_hash_sha512_final(&st2, r_hash.data()); - crypto_core_ed25519_scalar_reduce(r.data(), r_hash.data()); - - // sig_R = salt.crypto_scalarmult_ed25519_base_noclamp(r) - std::vector result; - result.resize(64); - auto* sig_R = result.data(); - auto* sig_S = result.data() + 32; - crypto_scalarmult_ed25519_base_noclamp(sig_R, r.data()); - - // HRAM = salt.crypto_core_ed25519_scalar_reduce(sha512_multipart(sig_R, kA, message_parts)) - uc64 hram; - crypto_hash_sha512_state st3; - crypto_hash_sha512_init(&st3); - crypto_hash_sha512_update(&st3, sig_R, 32); - crypto_hash_sha512_update(&st3, blind_15_pk.data(), blind_15_pk.size()); - crypto_hash_sha512_update(&st3, message.data(), message.size()); - crypto_hash_sha512_final(&st3, hram.data()); - - // sig_s = salt.crypto_core_ed25519_scalar_add(r, salt.crypto_core_ed25519_scalar_mul(HRAM, ka)) - crypto_core_ed25519_scalar_reduce(sig_S, hram.data()); // S = H(R||A||M) - crypto_core_ed25519_scalar_mul(sig_S, sig_S, blind_15_sk.data()); // S = H(R||A||M) a - crypto_core_ed25519_scalar_add(sig_S, sig_S, r.data()); // S = r + H(R||A||M) a + b64 r_hash; + hash::sha512(r_hash, std::span{hrh}.last<32>(), blind_15_pk, message); - return result; + b32 r; + ed25519::scalar_reduce(r, r_hash); + + return blinded_sign_finish(blind_15_pk, blind_15_sk, r, message); +} + +b64 blind15_sign( + const ed25519::PrivKeySpan& ed25519_sk, + std::string_view server_pk_in, + std::span message) { + return blind15_sign(ed25519_sk, parse_server_pk(server_pk_in, "blind15_sign"), message); } -std::vector blind_version_sign_request( - std::span ed25519_sk, +b64 blind_version_sign_request( + const ed25519::PrivKeySpan& ed25519_sk, uint64_t timestamp, std::string_view method, std::string_view path, - std::optional> body) { + std::optional> body) { auto [pk, sk] = blind_version_key_pair(ed25519_sk); // Signature should be on `TIMESTAMP || METHOD || PATH || BODY` - std::vector ts = to_vector(std::to_string(timestamp)); - std::vector buf; - buf.reserve(10 /* timestamp */ + method.size() + path.size() + (body ? body->size() : 0)); - buf.insert(buf.end(), ts.begin(), ts.end()); - buf.insert(buf.end(), method.begin(), method.end()); - buf.insert(buf.end(), path.begin(), path.end()); - + auto ts = "{}"_format(timestamp); + std::vector buf; + buf.reserve(ts.size() + method.size() + path.size() + (body ? body->size() : 0)); + auto app = [&](std::string_view sv) { + auto s = to_span(sv); + buf.insert(buf.end(), s.begin(), s.end()); + }; + app(ts); + app(method); + app(path); if (body) buf.insert(buf.end(), body->begin(), body->end()); - return ed25519::sign({sk.data(), sk.size()}, buf); + return ed25519::sign(sk, buf); } -std::vector blind_version_sign( - std::span ed25519_sk, Platform platform, uint64_t timestamp) { - auto [pk, sk] = blind_version_key_pair(ed25519_sk); - - // Signature should be on `TIMESTAMP || METHOD || PATH` - std::vector ts = to_vector(std::to_string(timestamp)); - std::vector method = to_vector("GET"); - std::vector buf; - buf.reserve(10 + 6 + 33); - buf.insert(buf.end(), ts.begin(), ts.end()); - buf.insert(buf.end(), method.begin(), method.end()); - - std::vector url; +b64 blind_version_sign( + const ed25519::PrivKeySpan& ed25519_sk, Platform platform, uint64_t timestamp) { + std::string_view url; switch (platform) { - case Platform::android: url = to_vector("/session_version?platform=android"); break; - case Platform::desktop: url = to_vector("/session_version?platform=desktop"); break; - case Platform::ios: url = to_vector("/session_version?platform=ios"); break; - default: url = to_vector("/session_version?platform=desktop"); break; + case Platform::android: url = "/session_version?platform=android"; break; + case Platform::ios: url = "/session_version?platform=ios"; break; + case Platform::desktop: + default: url = "/session_version?platform=desktop"; break; } - buf.insert(buf.end(), url.begin(), url.end()); - - return ed25519::sign({sk.data(), sk.size()}, buf); + return blind_version_sign_request(ed25519_sk, timestamp, "GET", url, std::nullopt); } bool session_id_matches_blinded_id( @@ -545,7 +393,7 @@ bool session_id_matches_blinded_id( "session_id_matches_blinded_id: server_pk must be hex (64 digits)"}; std::string converted_blind_id1, converted_blind_id2; - std::vector converted_blind_id1_raw; + std::vector converted_blind_id1_raw; switch (blinded_id[0]) { case '1': { @@ -569,7 +417,8 @@ LIBSESSION_C_API bool session_blind15_key_pair( unsigned char* blinded_pk_out, unsigned char* blinded_sk_out) { try { - auto [b_pk, b_sk] = session::blind15_key_pair({ed25519_seckey, 64}, {server_pk, 32}); + auto [b_pk, b_sk] = + session::blind15_key_pair({ed25519_seckey, 64}, to_byte_span<32>(server_pk)); std::memcpy(blinded_pk_out, b_pk.data(), b_pk.size()); std::memcpy(blinded_sk_out, b_sk.data(), b_sk.size()); return true; @@ -584,7 +433,8 @@ LIBSESSION_C_API bool session_blind25_key_pair( unsigned char* blinded_pk_out, unsigned char* blinded_sk_out) { try { - auto [b_pk, b_sk] = session::blind25_key_pair({ed25519_seckey, 64}, {server_pk, 32}); + auto [b_pk, b_sk] = + session::blind25_key_pair({ed25519_seckey, 64}, to_byte_span<32>(server_pk)); std::memcpy(blinded_pk_out, b_pk.data(), b_pk.size()); std::memcpy(blinded_sk_out, b_sk.data(), b_sk.size()); return true; @@ -617,7 +467,7 @@ LIBSESSION_C_API bool session_blind15_sign( auto sig = session::blind15_sign( {ed25519_seckey, 64}, {reinterpret_cast(server_pk), 32}, - {msg, msg_len}); + to_byte_span(msg, msg_len)); std::memcpy(blinded_sig_out, sig.data(), sig.size()); return true; } catch (...) { @@ -635,7 +485,7 @@ LIBSESSION_C_API bool session_blind25_sign( auto sig = session::blind25_sign( {ed25519_seckey, 64}, {reinterpret_cast(server_pk), 32}, - {msg, msg_len}); + to_byte_span(msg, msg_len)); std::memcpy(blinded_sig_out, sig.data(), sig.size()); return true; } catch (...) { @@ -645,7 +495,7 @@ LIBSESSION_C_API bool session_blind25_sign( LIBSESSION_C_API bool session_blind_version_sign_request( const unsigned char* ed25519_seckey, - size_t timestamp, + uint64_t timestamp, const char* method, const char* path, const unsigned char* body, @@ -654,9 +504,9 @@ LIBSESSION_C_API bool session_blind_version_sign_request( std::string_view method_sv{method}; std::string_view path_sv{path}; - std::optional> body_sv{std::nullopt}; + std::optional> body_sv{std::nullopt}; if (body) - body_sv = std::span{body, body_len}; + body_sv = to_byte_span(body, body_len); try { auto sig = session::blind_version_sign_request( @@ -671,7 +521,7 @@ LIBSESSION_C_API bool session_blind_version_sign_request( LIBSESSION_C_API bool session_blind_version_sign( const unsigned char* ed25519_seckey, CLIENT_PLATFORM platform, - size_t timestamp, + uint64_t timestamp, unsigned char* blinded_sig_out) { try { auto sig = session::blind_version_sign( diff --git a/src/client/client.cpp b/src/client/client.cpp new file mode 100644 index 000000000..2dfd30b59 --- /dev/null +++ b/src/client/client.cpp @@ -0,0 +1,5200 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "download_cache.hpp" + +namespace session::client { + +namespace log = oxen::log; +static auto cat = log::Cat("client"); + +using namespace std::literals; + +/// A message identifier: an opaque 64-bit pattern, compared for equality and never ordered or +/// counted. Signed all the way through -- the protobuf declares it sfixed64 for exactly that +/// reason -- so no conversion is needed anywhere between the wire and the database. +using MsgId = int64_t; + +/// A new message's Content.msgId. Must be generated before the message is copied for its recipient +/// and for our own swarm, so that both copies carry it: it is the only identifier every party +/// agrees on, the two copies differing in syncTarget and so not hashing alike. +static MsgId new_msgid() { + return static_cast(csrng()); +} + +/// Fills in what every outgoing message carries, and returns its DataMessage for whatever else the +/// caller has to add. +/// +/// Three places build one: a plain send, the placeholder stored while attachments upload, and the +/// rewrite that finally goes out once they have. They agree on these fields by construction here, +/// rather than by three copies of them staying in step. +static SessionProtos::DataMessage* fill_outgoing_content( + SessionProtos::Content& content, + sys_ms timestamp, + std::optional msgid, + std::string_view body) { + content.set_sigtimestamp(static_cast(epoch_ms(timestamp))); + if (msgid) + content.set_msgid(*msgid); + auto* data = content.mutable_datamessage(); + data->set_body(std::string{body}); + data->set_timestamp(static_cast(epoch_ms(timestamp))); + return data; +} + +/// Reads a message's identifier out of arriving content. Absent from anything sent by a client +/// that predates the field, which then has no identity beyond its timestamp. +static std::optional msgid_of(const SessionProtos::Content& content) { + if (!content.has_msgid()) + return std::nullopt; + return content.msgid(); +} + +// AttachmentPointer.Flags.VOICE_MESSAGE. Mirrored rather than taken from the generated header so +// that the column's meaning is legible where it is written. +constexpr int ATTACHMENT_FLAG_VOICE_MESSAGE = 1; + +// An attachment is a whole file rather than a swarm request, so it gets its own, longer allowances: +// the per-request one covers a stalled transfer, and the overall one bounds the upload entire. +constexpr auto ATTACHMENT_REQUEST_TIMEOUT = 60s; +constexpr auto ATTACHMENT_OVERALL_TIMEOUT = 10min; + +// Rate limits one stream of updates, so that a producer reporting faster than a consumer can +// usefully act on cannot flood it. `allow()` is true at most once per interval. +// +// One of these belongs to each thing being reported on, never to the reporter: a single instance +// shared between several streams would let whichever of them happened to be first in each window +// squelch the others indefinitely, so a transfer could appear to have stalled while it was in fact +// progressing. +class update_throttle { + std::chrono::milliseconds _interval; + std::optional _emitted; + + public: + explicit update_throttle(std::chrono::milliseconds interval) : _interval{interval} {} + + bool allow() { + if (_interval <= 0ms) + return true; + auto now = std::chrono::steady_clock::now(); + if (_emitted && now - *_emitted < _interval) + return false; + _emitted = now; + return true; + } +}; + +static SendState state_for(core::MessageSendStatus status) { + switch (status) { + case core::MessageSendStatus::awaiting_keys: return SendState::pending; + case core::MessageSendStatus::sending: + case core::MessageSendStatus::retrying: return SendState::sending; + case core::MessageSendStatus::success: return SendState::sent; + case core::MessageSendStatus::network_error: + case core::MessageSendStatus::no_network: + case core::MessageSendStatus::encrypt_failed: return SendState::failed; + } + return SendState::failed; +} + +static std::optional opt_view(const std::optional& s) { + if (s) + return *s; + return std::nullopt; +} + +static bool is_terminal(core::MessageSendStatus status) { + switch (status) { + case core::MessageSendStatus::success: + case core::MessageSendStatus::network_error: + case core::MessageSendStatus::no_network: + case core::MessageSendStatus::encrypt_failed: return true; + default: return false; + } +} + +// Parses a session ID as a protobuf carries one -- hex, in a syncTarget or a quote's author -- or +// nullopt if the string is not one. +// +// Both the length and the hex-ness have to be checked before converting: `from_hex` on a string +// that is not hex writes garbage rather than reporting anything, so an unchecked call turns a +// malformed field into a plausible-looking account. +static std::optional parse_session_id(std::string_view hex) { + if (hex.size() != 66 || !oxenc::is_hex(hex)) + return std::nullopt; + b33 out; + oxenc::from_hex(hex.begin(), hex.end(), reinterpret_cast(out.data())); + return out; +} + +// Returns the accounts row id for a session ID, creating it if this is the first time we have seen +// the account. Must be called inside the caller's transaction. +static int64_t account_id(sqlite::Connection& c, std::span session_id) { + c.prepared_exec("INSERT OR IGNORE INTO accounts (session_id) VALUES (?)", session_id); + return c.prepared_get("SELECT id FROM accounts WHERE session_id = ?", session_id); +} + +// The conversations column a given kind occupies, plus the queries that resolve or create the row +// in the table that column references. +struct ConvoKind { + std::string_view column; + std::string_view find; + std::string_view create; +}; + +static ConvoKind kind_of(ConversationId::Type type) { + switch (type) { + case ConversationId::Type::dm: + return {"dm", + "SELECT id FROM accounts WHERE session_id = ?", + "INSERT OR IGNORE INTO accounts (session_id) VALUES (?)"}; + case ConversationId::Type::group: + return {"closed_group", + "SELECT id FROM groups WHERE group_id = ?", + "INSERT OR IGNORE INTO groups (group_id) VALUES (?)"}; + case ConversationId::Type::community: + return {"community", + "SELECT id FROM communities WHERE base_url = ? AND room = ?", + "INSERT OR IGNORE INTO communities (base_url, room) VALUES (?, ?)"}; + } + throw std::logic_error{"unhandled conversation kind"}; +} + +// Invokes `run` with a ConvoKind query and whichever bind parameters that kind's identity takes: +// one blob for a DM or group, two strings for a community. +template +static R with_identity_binds(const ConversationId& id, std::string_view query, auto&& run) { + if (id.type() == ConversationId::Type::community) { + auto [url, room] = id.community(); + return run(std::string{query}, url, room); + } + auto raw = id.type() == ConversationId::Type::dm ? id.session_id() : id.group_id(); + return run(std::string{query}, raw); +} + +// Returns the identity row id for what a conversation is with, creating it if absent. Must be +// called inside the caller's transaction. +static int64_t identity_id(sqlite::Connection& c, const ConversationId& id) { + auto kind = kind_of(id.type()); + with_identity_binds(id, kind.create, [&](const std::string& q, const auto&... b) { + return c.prepared_exec(q, b...); + }); + return with_identity_binds(id, kind.find, [&](const std::string& q, const auto&... b) { + return c.prepared_get(q, b...); + }); +} + +// Rebuilds a ConversationId from whichever identity a conversation row joined to. Exactly one is +// set -- that is what the table's CHECK constraint enforces -- so the throw is unreachable unless +// the database has been corrupted or written behind our back. +static ConversationId subject_to_id( + int64_t convo, + const std::optional>& sid, + const std::optional>& gid, + const std::optional& url, + const std::optional& room) { + if (sid) + return ConversationId::dm(*sid); + if (gid) + return ConversationId::group(*gid); + if (url && room) + return ConversationId::community(*url, *room); + throw std::runtime_error{"conversation {} has no subject"_format(convo)}; +} + +static constexpr auto SUBJECT_JOIN = R"( + FROM conversations c + LEFT JOIN accounts a ON a.id = c.dm + LEFT JOIN contacts ct ON ct.account = a.id + LEFT JOIN groups g ON g.id = c.closed_group + LEFT JOIN communities m ON m.id = c.community +)"sv; + +static ConversationId conversation_id_at(sqlite::Connection& c, int64_t convo) { + auto [sid, gid, url, room] = c.prepared_get< + std::optional>, + std::optional>, + std::optional, + std::optional>( + "SELECT a.session_id, g.group_id, m.base_url, m.room {} WHERE c.id = ?"_format( + SUBJECT_JOIN), + convo); + return subject_to_id(convo, sid, gid, url, room); +} + +struct ConvoRow { + int64_t id; + bool created; +}; + +// Creates the conversation row if it is missing and moves last_activity forward to `activity` if +// that is newer, returning its row id. Must be called inside the caller's transaction along with +// whatever change prompted it. +static ConvoRow ensure_conversation( + sqlite::Connection& c, const ConversationId& id, sys_ms activity) { + auto kind = kind_of(id.type()); + auto subject = identity_id(c, id); + auto ms = epoch_ms(activity); + + bool created = c.prepared_exec( + R"( + INSERT OR IGNORE INTO conversations ({}, created, last_activity) VALUES (?, ?, ?) + )"_format(kind.column), + subject, + epoch_seconds(activity), + ms) > 0; + if (!created) + c.prepared_exec( + R"( + UPDATE conversations SET last_activity = ?2 + WHERE {} = ?1 AND last_activity < ?2 + )"_format(kind.column), + subject, + ms); + + return {c.prepared_get( + "SELECT id FROM conversations WHERE {} = ?"_format(kind.column), subject), + created}; +} + +// Makes an account a contact if it is not one already, and says whether that changed anything. +// Must be called inside the caller's transaction. +// +// Anything that records a fact about a relationship needs this first: the fact belongs in the +// Contacts config, and a row here is what an entry there is made from, so there is nowhere to put +// it otherwise. +static bool ensure_contact(sqlite::Connection& c, int64_t account, bool approved) { + return c.prepared_exec( + "INSERT OR IGNORE INTO contacts (account, approved) VALUES (?, ?)", + account, + approved ? 1 : 0) > 0; +} + +// Records that we have approved whoever an outgoing message is addressed to, and says whether that +// changed anything. Must be called inside the caller's transaction. +// +// Writing to someone is what approving them is -- there is no separate accept -- so answering a +// message request is what takes it out of the requests list. Never for note to self, which is not +// a contact and cannot be a request. +static bool approve_recipient(sqlite::Connection& c, const ConversationId& id, Client& client) { + if (client.is_me(id.session_id())) + return false; + auto account = identity_id(c, id); + auto made = ensure_contact(c, account, true); + auto flagged = c.prepared_exec( + "UPDATE contacts SET approved = 1 WHERE account = ? AND NOT approved", + account) > 0; + return made || flagged; +} + +// What counts as unread, as a fragment the three places that recompute the count share. The schema +// says why this is not in a trigger: it is policy rather than structure, and it grows. It has more +// than one clause now, which is the point at which three copies of it start drifting. +// +// Unqualified on purpose, so it reads the same inside a correlated subquery as in a plain one. +static constexpr auto UNREAD = "outgoing = 0 AND deleted IS NULL"sv; + +// Carries out a delete-before instruction, and says whether anything went. Must be called inside +// the caller's transaction. +// +// The unread count is recomputed rather than decremented by what was deleted: what counts as unread +// is the application's policy and is about to grow (mutes, requests, tombstones), so a second place +// applying it is a second place to get it wrong. +static bool delete_messages_before(sqlite::Connection& c, int64_t convo, sys_ms before) { + if (c.prepared_exec( + "DELETE FROM messages WHERE conversation = ?1 AND timestamp < ?2", + convo, + epoch_ms(before)) == 0) + return false; + + c.prepared_exec( + R"( + UPDATE conversations SET unread_count = ( + SELECT COUNT(*) FROM messages + WHERE messages.conversation = conversations.id AND {} + AND messages.timestamp > conversations.last_read) + WHERE id = ? + )"_format(UNREAD), + convo); + return true; +} + +// As above for a delete-attachments-before instruction, which takes the files but leaves the +// messages that carried them. Must be called inside the caller's transaction. +static bool delete_attachments_before(sqlite::Connection& c, int64_t convo, sys_ms before) { + return c.prepared_exec( + R"( + DELETE FROM message_attachments WHERE message IN + (SELECT id FROM messages WHERE conversation = ?1 AND timestamp < ?2) + )", + convo, + epoch_ms(before)) > 0; +} + +// Returns the conversation row id, or nullopt if we have no such conversation. +static std::optional find_conversation(sqlite::Connection& c, const ConversationId& id) { + auto kind = kind_of(id.type()); + auto subject = with_identity_binds>( + id, kind.find, [&](const std::string& q, const auto&... b) { + return c.prepared_maybe_get(q, b...); + }); + if (!subject) + return std::nullopt; + return c.prepared_maybe_get( + "SELECT id FROM conversations WHERE {} = ?"_format(kind.column), *subject); +} + +core::callbacks Client::_core_callbacks() { + // Capturing `this` here is safe despite running in Core's member-init list: every callback we + // install can only fire from receive_messages(), send_dm(), or a config merge, none of which + // Core calls during its own construction. + // + // These are Client's own wiring, and an application cannot supply any of its own: what it is + // promised is `client::callbacks`, which is reported through the dispatcher and carries whole + // state. Anything it needs that only Core knows is reported by handling it here and + // re-reporting it there. + core::callbacks cb; + + // Persist first, then notify: a throwing callback is a bug Core can only log, so Client must + // never rely on an exception to reject a batch. + cb.message_received = [this](core::ReceivedMessage&& msg) { + _on_message_received(std::move(msg)); + }; + + cb.message_send_status = [this](int64_t id, + core::MessageSendStatus status, + std::optional swarm_hash) { + _on_send_status(id, status, swarm_hash); + }; + + cb.configs_changed = [this](std::span changed) { + _on_configs_changed(changed); + }; + + return cb; +} + +void Client::_init() { + // Core's send queue is in-memory, so anything still mid-flight when the last run ended is not + // resumed and its outcome is unknowable. Say so rather than guessing either way. + auto c = core.database().conn(); + c.prepared_exec( + R"( + UPDATE messages SET send_state = ? WHERE send_state IN (?, ?) + )", + static_cast(SendState::interrupted), + static_cast(SendState::pending), + static_cast(SendState::sending)); + c.prepared_exec( + R"( + UPDATE messages SET sync_send_state = ? WHERE sync_send_state IN (?, ?) + )", + static_cast(SendState::interrupted), + static_cast(SendState::pending), + static_cast(SendState::sending)); + + // A message whose attachments were still uploading is not in that same doubt: nothing can have + // reached a swarm, because the message could not be built until the uploads finished. So it + // failed outright rather than unknowably, and it is squarely retryable -- its attachment rows + // record which files did get up, so resuming re-uploads only the rest. + c.prepared_exec( + "UPDATE messages SET send_state = ? WHERE send_state = ?", + static_cast(SendState::failed), + static_cast(SendState::uploading)); + c.prepared_exec( + "UPDATE messages SET sync_send_state = ? WHERE sync_send_state = ?", + static_cast(SendState::failed), + static_cast(SendState::uploading)); + + // Guarded because a Core opened with defer_account has no account yet, and the configs are + // encrypted to its key. Nothing is missed by skipping it: an account that does not exist has + // no configs to have fallen behind, and whatever arrives once it does comes through a merge, + // which reports itself. + // + // On the loop, because reconciling dirties conversations and `_dirty` is guarded by nothing but + // that every writer is the loop thread. We are the constructing thread here, and the loop is + // already running by the time a Client is built: the first `_touch` schedules `_flush_pending`, + // which steals `_dirty` out from under the reconcile still filling it. + if (core.globals.have_account()) + loop.call_get([this] { _reconcile_all(); }); +} + +// -- Change notification ---------------------------------------------------------------------- + +void Client::_emit(std::function invoke) { + _dispatch_out([cbs = _cbs, invoke = std::move(invoke)] { invoke(*cbs); }); +} + +void Client::set_dispatcher(dispatcher d) { + loop.call([this, d = std::move(d)]() mutable { _dispatcher = std::move(d); }); +} + +void Client::set_high_freq_dispatch_interval(std::chrono::milliseconds interval) { + loop.call([this, interval] { _high_freq_dispatch_interval = interval; }); +} + +void Client::_dispatch_out(std::function job) { + auto guarded = [job = std::move(job)] { + try { + job(); + } catch (const std::exception& e) { + log::error(cat, "client handler threw: {}", e.what()); + } + }; + + if (_dispatcher) + _dispatcher(std::move(guarded)); + else + guarded(); +} + +void Client::_emit_conversation_added(const ConversationId& id) { + auto convo = _conversation(id); + if (!convo) + return; + // Mutable so the value moves out: each _emit job runs once, and the handler owns what it gets. + _emit([convo = std::move(*convo)](const callbacks& cbs) mutable { + if (cbs.conversation_added) + cbs.conversation_added(std::move(convo)); + }); +} + +void Client::_emit_conversation_removed(const ConversationId& id) { + // `id = id` rather than `id`: a copy-capture of a const lvalue is itself const, which `mutable` + // does not undo, and the handler is given the id outright. + _emit([id = id](const callbacks& cbs) mutable { + if (cbs.conversation_removed) + cbs.conversation_removed(std::move(id)); + }); +} + +void Client::_emit_history_replaced(const ConversationId& id) { + _emit([id = id](const callbacks& cbs) mutable { + if (cbs.history_replaced) + cbs.history_replaced(std::move(id)); + }); +} + +void Client::_emit_message_alone(bool added, const ConversationId& id, int64_t message_id) { + auto msg = _message(message_id); + if (!msg) + return; + _emit([added, id = id, msg = std::move(*msg)](const callbacks& cbs) mutable { + const auto& h = added ? cbs.message_added : cbs.message_updated; + if (h) + h(std::move(id), std::move(msg)); + }); +} + +void Client::_emit_message(bool added, const ConversationId& id, int64_t message_id) { + _emit_message_alone(added, id, message_id); + + // Matched the way the reference was written rather than the way it resolves: a quote with no + // msgid whose target is ambiguous resolves to only one of the candidates, but which one is not + // worth computing here. Reporting a message whose displayed reply did not actually change + // costs a redraw; failing to report one that did leaves the display wrong. + // + // `reply_timestamp IS NOT NULL` is redundant against the equality but is what lets the planner + // use `messages_reply_target`, which is a partial index over exactly that condition. + auto c = core.database().conn(); + for (auto replier : c.prepared_results( + R"( + SELECT r.id FROM messages r JOIN messages t ON t.id = ?1 + WHERE r.conversation = t.conversation + AND r.reply_timestamp IS NOT NULL + AND r.reply_author = t.sender + AND r.reply_timestamp = t.timestamp + AND (r.reply_msgid IS NULL OR r.reply_msgid = t.msgid) + AND r.id != t.id + )", + message_id)) + _emit_message_alone(false, id, replier); +} + +void Client::_touch(const ConversationId& id) { + if (std::ranges::find(_dirty, id) == _dirty.end()) + _dirty.push_back(id); + + // Deferred to the end of the loop's current turn rather than reported here: everything that + // dirties a conversation runs on the loop, so by the time this fires a whole received batch has + // been stored and the conversation has one settled state to report instead of fifty. + if (!_flush_scheduled) { + _flush_scheduled = true; + _jq.call_soon([this] { _flush_pending(); }); + } +} + +void Client::_flush_pending() { + _flush_scheduled = false; + auto dirty = std::move(_dirty); + _dirty.clear(); + + for (const auto& id : dirty) { + auto convo = _conversation(id); + if (!convo) + continue; + _emit([convo = std::move(*convo)](const callbacks& cbs) mutable { + if (cbs.conversation_updated) + cbs.conversation_updated(std::move(convo)); + }); + } +} + +// -- Asynchronous interface --------------------------------------------------------------------- + +// Checked on the calling thread so that an unreadable file throws where the mistake was made, +// rather than failing a message that has already been stored and shown. +void Client::_require_readable(const std::vector& attachments) { + for (const auto& a : attachments) { + std::error_code ec; + if (!std::filesystem::is_regular_file(a.path, ec) || ec) + throw std::invalid_argument{ + "send_message: attachment {} is not a readable file"_format(a.path.string())}; + if (std::filesystem::file_size(a.path, ec) == 0 || ec) + throw std::invalid_argument{ + "send_message: attachment {} is empty"_format(a.path.string())}; + } +} + +void Client::_require_sendable( + std::string_view op, const ConversationId& id, const OutgoingMessage& msg) { + _require_dm(op, id); + _require_readable(msg.attachments); + + if (!msg.reply_to) + return; + + // Reading the database from the calling thread, which nothing else here does -- but the + // alternative is to accept the send, discover inside the loop that the target is not there, and + // have only a callback to say so. A caller naming a message that does not exist has made a + // mistake at the call site, and that is where it should be reported. + auto found = loop.call_get([this, id, target = *msg.reply_to] { + auto c = core.database().conn(); + return c.prepared_maybe_get( + R"( + SELECT m.id FROM messages m + JOIN conversations c ON c.id = m.conversation + JOIN accounts a ON a.id = c.dm + WHERE m.id = ? AND a.session_id = ? + )", + target, + id.session_id()); + }); + if (!found) + throw std::invalid_argument{ + "{}: reply_to message {} does not exist in this conversation"_format( + op, *msg.reply_to)}; +} + +void Client::_require_dm(std::string_view op, const ConversationId& id) { + // Checked on the calling thread so caller error surfaces at the call site rather than inside + // the loop, where the callback form would only be able to log it. + if (id.type() != ConversationId::Type::dm) + throw std::invalid_argument{ + "{}: only DM conversations are supported so far (got type {})"_format( + op, static_cast(id.type()))}; +} + +void Client::_require_contact(std::string_view op, const ConversationId& id) { + _require_dm(op, id); + // Our own account is not a contact of ours -- there is no entry to block or to remove, and + // what note to self does have lives in UserProfile. + if (is_note_to_self(id)) + throw std::invalid_argument{"{}: not applicable to your own account"_format(op)}; +} + +void Client::_require_page(std::string_view op, int limit) { + // The value reaches SQLite as `LIMIT ?`, where a negative is no limit at all and zero is the + // end of the history -- so an unchecked one loads the whole conversation instead of failing. + if (limit <= 0) + throw std::invalid_argument{ + "{}: limit must be a positive page size (got {})"_format(op, limit)}; +} + +void Client::log_operation_failure(const std::exception& e) { + log::error(cat, "Client operation failed: {}", e.what()); +} + +// Not dispatched onto the loop: reads nothing but the session ID, which cannot change underneath +// it. See the declaration. +bool Client::is_me(std::span session_id) { + return std::ranges::equal(session_id, _self_or_none()); +} + +bool Client::is_note_to_self(const ConversationId& id) { + return id.type() == ConversationId::Type::dm && is_me(id.session_id()); +} + +void Client::retry_send( + int64_t message_id, + std::function)> on_upload, + std::function, bool)> cb) { + _async( + [this, message_id, on_upload = std::move(on_upload)] { + return _retry_send(message_id, on_upload); + }, + std::move(cb)); +} + +bool Client::retry_send(int64_t message_id, Conversation::upload_progress on_upload, await_t) { + return loop.call_get([this, message_id, on_upload = std::move(on_upload)] { + return _retry_send(message_id, on_upload); + }); +} + +bool Client::retry_send(int64_t message_id, await_t) { + return retry_send(message_id, nullptr, await); +} + +void Client::message_debug( + int64_t message_id, failable_function)> cb) { + _async([this, message_id] { return _message_debug(message_id); }, std::move(cb)); +} + +std::optional Client::message_debug(int64_t message_id, await_t) { + return loop.call_get([this, message_id] { return _message_debug(message_id); }); +} + +void Client::delete_message(int64_t message_id, failable_function cb) { + _async([this, message_id] { return _delete_message(message_id, Deletion::here); }, + std::move(cb)); +} + +bool Client::delete_message(int64_t message_id, await_t) { + return loop.call_get( + [this, message_id] { return _delete_message(message_id, Deletion::here); }); +} + +void Client::set_cache_dir(std::filesystem::path dir) { + _cache_dir = std::move(dir); + _sweep_cache(); +} + +Client::~Client() { + // Before the loop it hands its listing back to, and before the members that listing is about. + if (_sweeper.joinable()) + _sweeper.join(); +} + +void Client::_sweep_cache() { + if (_cache_dir.empty()) + return; + + if (_sweeper.joinable()) + _sweeper.join(); + + // Split so that the walk -- the slow half, which reads nothing but the directory -- stays off + // the loop, and the deciding -- the quick half, which reads the database -- stays on it. That + // split is also the whole of the concurrency argument: a file is written and its row inserted + // by one loop job, so a reconcile running on the loop cannot land between the two and take a + // freshly cached file for an orphan. Files that appear after the listing are simply not in it. + _sweeper = std::thread{[this, dir = _cache_dir] { + auto attachments = cache::list(dir, cache::ATTACHMENT_DIR); + auto pictures = cache::list(dir, cache::PROFILE_DIR); + + // `call_get`, not `call`: the destructor joins this thread to know the sweep is over, and a + // thread that had only posted the job would finish while the job was still queued. + loop.call_get([this, &attachments, &pictures] { + try { + _reconcile_cache(std::move(attachments), std::move(pictures)); + } catch (const std::exception& e) { + log::warning(cat, "Cache sweep failed: {}", e.what()); + } + return 0; + }); + }}; +} + +void Client::_reconcile_cache( + std::vector attachments, std::vector pictures) { + auto c = core.database().conn(); + + // An attachment file without a row cannot be found by a lookup or counted by eviction, so it is + // not a cache entry at all -- it is a file taking up room under a name nobody can resolve. + size_t orphans = 0; + for (const auto& name : attachments) + if (!c.prepared_get( + "SELECT EXISTS(SELECT 1 FROM attachment_cache WHERE name = ?)", name) && + cache::remove(_cache_dir, cache::ATTACHMENT_DIR, name)) + orphans++; + + // The other direction, which is not cosmetic: eviction totals `size` over the rows, so a row + // naming a file that is gone makes the cache look fuller than it is and evicts live files to + // get back under a limit it was never over. + // + // Rows the listing covers are fine by definition. The rest are checked against the disk rather + // than assumed missing, because a row inserted after the listing was taken is legitimately + // absent from it and dropping it would strand the file it names. Collected before deleting + // any, rather than deleted as they are found: this is a read of the table it would modify. + std::set listed{attachments.begin(), attachments.end()}; + std::vector stale; + for (auto name : c.prepared_results("SELECT name FROM attachment_cache")) { + std::error_code ec; + if (!listed.contains(name) && + !std::filesystem::exists(_cache_dir / cache::ATTACHMENT_DIR / name, ec)) + stale.push_back(std::move(name)); + } + for (const auto& name : stale) + c.prepared_exec("DELETE FROM attachment_cache WHERE name = ?", name); + + // A picture is referenced by an account naming its url and by nothing else, so the referenced + // set is that column. Recomputed here rather than passed in, so that an account that appeared + // while the walk was running counts as referencing what it names. + std::set keep; + for (auto url : c.prepared_results( + "SELECT profile_pic_url FROM accounts WHERE profile_pic_url IS NOT NULL")) + keep.insert(cache::path_for(_cache_dir, cache::PROFILE_DIR, url).filename().string()); + + size_t unreferenced = 0; + for (const auto& name : pictures) + if (!keep.contains(name) && cache::remove(_cache_dir, cache::PROFILE_DIR, name)) + unreferenced++; + + if (orphans || !stale.empty() || unreferenced) + log::info( + cat, + "Cache sweep: dropped {} untracked attachment(s), {} row(s) for missing files, {} " + "unreferenced picture(s)", + orphans, + stale.size(), + unreferenced); +} + +const b32& Client::_cache_encryption_key() { + if (!_cache_key) { + auto& key = _cache_key.emplace(); + if (!core.globals.get_blob_to("client:cache_key", key)) { + // Generated once and kept: every cached file is encrypted under it, so losing it would + // orphan the whole cache -- which is survivable (everything in it can be fetched again) + // but silently wasteful, since nothing would ever read those files or delete them. + random::fill(key); + core.globals.set("client:cache_key", std::span{key}); + } + } + return *_cache_key; +} + +void Client::profile_picture( + const ConversationId& id, + std::function)> on_progress, + failable_function>)> cb) { + loop.call([this, id, on_progress = std::move(on_progress), cb = std::move(cb)]() mutable { + try { + _profile_picture(id, std::move(on_progress), std::move(cb)); + } catch (const std::exception& e) { + _report(cb, std::optional{std::string{e.what()}}, std::nullopt); + } + }); +} + +void Client::profile_picture( + const ConversationId& id, + failable_function>)> cb) { + profile_picture(id, nullptr, std::move(cb)); +} + +// No waiting form, deliberately: this is a network fetch of an unbounded file, and the one caller +// that wants to block on it is a caller that has not thought about a slow file server. +void Client::_profile_picture( + const ConversationId& id, + std::function)> on_progress, + failable_function>)> cb) { + + auto convo = _conversation(id); + if (!convo || convo->picture().url.empty()) { + // Nobody has told us of one, or it is a kind whose picture is not wired up yet. Not an + // error: there is simply nothing to show. + _report(cb, std::optional{}, std::nullopt); + return; + } + + // No check on the key here: a missing one is not malformed, it means the file is stored in the + // clear, which is how a community's image is kept. A key of the wrong length *is* malformed, + // and the download reports it as the error it is rather than as an absent picture. + + auto pic = convo->picture(); + + // A caller of this deals in an optional, because an absent picture is not an error -- but that + // case was answered above, so from here anything that is not an error is a picture. + failable_function)> bytes; + if (cb) + bytes = [cb = std::move(cb)]( + std::optional error, std::vector data) { + if (error) + cb(std::move(error), std::nullopt); + else + cb(std::nullopt, std::move(data)); + }; + + _fetch_cached( + {pic.url, pic.key, {}, std::nullopt, DownloadKind::display_pic, cache::PROFILE_DIR}, + on_progress ? _dispatch_progress(std::move(on_progress)) : nullptr, + std::move(bytes), + // Nothing on a hit: a picture is not indexed, so there is no use to record, and no + // progress to report either -- a bar that flashes for a local read is worse than none. + nullptr, + _store_picture(pic.url)); +} + +// Writes a fetched picture into the cache, or nothing at all when there is nowhere to put it. +std::function)> Client::_store_picture(std::string url) { + if (_cache_dir.empty()) + return nullptr; + + return [this, url = std::move(url), key = _cache_encryption_key()]( + std::span data) { + try { + cache::write(cache::path_for(_cache_dir, cache::PROFILE_DIR, url), key, data); + } catch (const std::exception& e) { + // A cache that cannot be written is a cache that misses next time, which is not worth + // failing the caller's fetch over. + log::warning(cat, "Could not cache a profile picture: {}", e.what()); + } + }; +} + +namespace { + // Device-local, so `globals` rather than a config: how much disk to spend, and what is worth + // fetching unasked, are properties of this machine and not of the account. + constexpr auto CACHE_LIMIT_KEY = "client:attachment_cache_limit"; + constexpr auto AUTO_DL_MAX_KEY = "client:auto_download_max_size"; +} // namespace + +// Absent rather than sentinel: "no limit" is the key not being there, so nothing has to reserve a +// magic value or decide whether 0 means unlimited or refuse-everything. +static void set_limit(core::Globals& g, std::string_view key, std::optional bytes) { + if (bytes) + g.set(key, *bytes); + else + g.erase(key); +} + +void Client::set_attachment_cache_limit( + std::optional bytes, failable_function cb) { + _async([this, bytes] { set_limit(core.globals, CACHE_LIMIT_KEY, bytes); }, std::move(cb)); +} +void Client::set_attachment_cache_limit(std::optional bytes, await_t) { + loop.call_get([this, bytes] { set_limit(core.globals, CACHE_LIMIT_KEY, bytes); }); +} +void Client::attachment_cache_limit(failable_function)> cb) { + _async([this] { return core.globals.get_integer(CACHE_LIMIT_KEY); }, std::move(cb)); +} +std::optional Client::attachment_cache_limit(await_t) { + return loop.call_get([this] { return core.globals.get_integer(CACHE_LIMIT_KEY); }); +} + +void Client::set_auto_download_max_size( + std::optional bytes, failable_function cb) { + _async([this, bytes] { set_limit(core.globals, AUTO_DL_MAX_KEY, bytes); }, std::move(cb)); +} +void Client::set_auto_download_max_size(std::optional bytes, await_t) { + loop.call_get([this, bytes] { set_limit(core.globals, AUTO_DL_MAX_KEY, bytes); }); +} +void Client::auto_download_max_size(failable_function)> cb) { + _async([this] { return core.globals.get_integer(AUTO_DL_MAX_KEY); }, std::move(cb)); +} +std::optional Client::auto_download_max_size(await_t) { + return loop.call_get([this] { return core.globals.get_integer(AUTO_DL_MAX_KEY); }); +} + +void Client::display_name(failable_function cb) { + _async([this] { return std::string{core.configs.user_profile().get_name().value_or("")}; }, + std::move(cb)); +} + +std::string Client::display_name(await_t) { + return loop.call_get( + [this] { return std::string{core.configs.user_profile().get_name().value_or("")}; }); +} + +void Client::set_display_name(std::string_view name, failable_function cb) { + _async([this, name = std::string{name}] { core.configs.user_profile().set_name(name); }, + std::move(cb)); +} + +void Client::set_display_name(std::string_view name, await_t) { + loop.call_get([this, name] { core.configs.user_profile().set_name(name); }); +} + +void Client::notify_media_saved(failable_function cb) { + _async([this] { return core.configs.user_profile().get_notify_media_saved(); }, std::move(cb)); +} + +bool Client::notify_media_saved(await_t) { + return loop.call_get([this] { return core.configs.user_profile().get_notify_media_saved(); }); +} + +void Client::set_notify_media_saved(bool notify, failable_function cb) { + _async([this, notify] { core.configs.user_profile().set_notify_media_saved(notify); }, + std::move(cb)); +} + +void Client::set_notify_media_saved(bool notify, await_t) { + loop.call_get([this, notify] { core.configs.user_profile().set_notify_media_saved(notify); }); +} + +void Client::delete_message_everywhere(int64_t message_id, failable_function cb) { + _async([this, message_id] { return _delete_message_everywhere(message_id); }, std::move(cb)); +} + +bool Client::delete_message_everywhere(int64_t message_id, await_t) { + return loop.call_get([this, message_id] { return _delete_message_everywhere(message_id); }); +} + +void Client::attachment_data( + int64_t message_id, + size_t index, + std::function on_progress, + failable_function)> cb) { + loop.call([this, message_id, index, on_progress = std::move(on_progress), cb]() mutable { + try { + _attachment_data(message_id, index, std::move(on_progress), cb); + } catch (const std::exception& e) { + log_operation_failure(e); + _report(cb, std::optional{std::string{e.what()}}, std::vector{}); + } + }); +} + +void Client::_attachment_data( + int64_t message_id, + size_t index, + std::function on_progress, + failable_function)> cb) { + + auto [url, key, digest, claimed_size] = _attachment_pointer(message_id, index); + + // The caller's own progress reporting, identified and hopped out to their thread. + std::function)> progress; + if (on_progress) + progress = _dispatch_progress([on_progress = std::move(on_progress), message_id, index]( + int64_t done, int64_t total, std::optional r) { + on_progress(AttachmentProgress{message_id, index, done, total, r}); + }); + + std::function)> store; + if (!_cache_dir.empty()) + store = [this, url, k = _cache_encryption_key()](std::span data) { + _cache_attachment(url, k, data); + }; + + _fetch_cached( + {url, + std::move(key), + std::move(digest), + claimed_size, + DownloadKind::attachment, + cache::ATTACHMENT_DIR}, + std::move(progress), + std::move(cb), + [this](const std::string& name) { _touch_cached(name); }, + std::move(store)); +} + +void Client::_fetch_cached( + FetchTarget target, + std::function)> progress, + failable_function)> cb, + std::function on_hit, + std::function)> store) { + + auto name = cache::path_for(_cache_dir, target.dir, target.url).filename().string(); + + if (!_cache_dir.empty()) { + auto file = cache::path_for(_cache_dir, target.dir, target.url); + if (auto cached = cache::read(file, _cache_encryption_key())) { + // Nothing to report: there is no transfer, and a progress bar for a local read is a + // flicker that means nothing. The caller gets the bytes. + if (on_hit) + on_hit(name); + _report(cb, std::optional{}, std::move(*cached)); + return; + } + } + + // Already being fetched: wait on that rather than asking for the same bytes again. + if (auto found = _in_flight.find(name); found != _in_flight.end()) { + if (progress) { + // Told where it has got to before anything else happens, so a display that arrives + // halfway through starts from halfway rather than from nothing. + progress(found->second.done, found->second.total, std::nullopt); + found->second.progress.push_back(std::move(progress)); + } + if (cb) + found->second.waiting.push_back(std::move(cb)); + return; + } + + auto& entry = _in_flight[name]; + entry.plain = std::make_shared>(); + if (progress) + entry.progress.push_back(std::move(progress)); + if (cb) + entry.waiting.push_back(std::move(cb)); + auto plain = entry.plain; + + _download_decrypted( + target.url, + target.kind, + std::move(target.key), + std::move(target.digest), + target.claimed_size, + [plain](std::span chunk) { + plain->insert(plain->end(), chunk.begin(), chunk.end()); + }, + // Onto the loop before touching the registry -- this arrives on the network thread, and + // `_in_flight` is ours. + [this, name](int64_t done, int64_t total, std::optional r) { + loop.call([this, name, done, total, r] { + auto found = _in_flight.find(name); + if (found == _in_flight.end()) + return; + found->second.done = done; + found->second.total = total; + for (const auto& p : found->second.progress) + p(done, total, r); + }); + }, + [this, name, store = std::move(store)](std::optional error) { + loop.call([this, name, store, error = std::move(error)]() mutable { + auto found = _in_flight.find(name); + if (found == _in_flight.end()) + return; + + // Stored before anyone is told, since a waiter may go straight back to the + // cache -- and only on success, because what a failed download produced is not + // the file. + if (!error && store) + store(*found->second.plain); + + // Lifted out before the callbacks run: one of them may ask for this same file + // again, and it must find a finished transfer rather than joining one that is + // about to be erased. + auto entry = std::move(found->second); + _in_flight.erase(found); + + for (const auto& w : entry.waiting) + _report(w, error, error ? std::vector{} : *entry.plain); + }); + }); +} + +void Client::set_gallery(int64_t message_id, bool gallery, failable_function cb) { + _async([this, message_id, gallery] { return _set_gallery(message_id, gallery); }, + std::move(cb)); +} + +bool Client::set_gallery(int64_t message_id, bool gallery, await_t) { + return loop.call_get([this, message_id, gallery] { return _set_gallery(message_id, gallery); }); +} + +void Client::purge_deleted_message(int64_t message_id, failable_function cb) { + _async([this, message_id] { return _purge_deleted_message(message_id); }, std::move(cb)); +} + +bool Client::purge_deleted_message(int64_t message_id, await_t) { + return loop.call_get([this, message_id] { return _purge_deleted_message(message_id); }); +} + +void Client::send_message( + const ConversationId& id, + OutgoingMessage msg, + std::function)> on_upload, + std::function, int64_t)> cb) { + _require_sendable("send_message", id, msg); + + _async( + [this, id, msg = std::move(msg), on_upload = std::move(on_upload)] { + return _send_message(id, msg, on_upload); + }, + std::move(cb)); +} + +void Client::send_message( + const ConversationId& id, + OutgoingMessage msg, + std::function, int64_t)> cb) { + send_message(id, std::move(msg), nullptr, std::move(cb)); +} + +// Callback forms: dispatch and return, delivering the result on the loop thread. + +void Client::conversations( + std::function, std::vector)> cb) { + _async([this] { return _conversations(); }, std::move(cb)); +} + +void Client::message_requests( + std::function, std::vector)> cb) { + _async([this] { return _message_requests(); }, std::move(cb)); +} + +void Client::set_blocked( + const ConversationId& id, + bool blocked, + std::function)> cb) { + _require_contact("set_blocked", id); + _async([this, id, blocked] { _set_blocked(id, blocked); }, std::move(cb)); +} + +void Client::set_blocked(const ConversationId& id, bool blocked, await_t) { + _require_contact("set_blocked", id); + loop.call_get([this, id, blocked] { _set_blocked(id, blocked); }); +} + +std::vector Client::conversations(await_t) { + return loop.call_get([this] { return _conversations(); }); +} + +std::vector Client::message_requests(await_t) { + return loop.call_get([this] { return _message_requests(); }); +} + +std::optional Client::conversation(const ConversationId& id, await_t) { + return loop.call_get([this, id] { return _conversation(id); }); +} + +std::optional Client::message(int64_t id, await_t) { + return loop.call_get([this, id] { return _message(id); }); +} + +int64_t Client::send_message(const ConversationId& id, OutgoingMessage msg, await_t) { + return send_message(id, std::move(msg), nullptr, await); +} + +int64_t Client::send_message( + const ConversationId& id, + OutgoingMessage msg, + Conversation::upload_progress on_upload, + await_t) { + _require_sendable("send_message", id, msg); + return loop.call_get([&] { return _send_message(id, msg, std::move(on_upload)); }); +} + +void Client::conversation( + const ConversationId& id, + std::function, std::optional)> cb) { + _async([this, id] { return _conversation(id); }, std::move(cb)); +} + +// A DM asked for by kind: the same lookup, then narrowed. Nullopt covers both "no such +// conversation" and, since _require_dm has already refused a group or community id, nothing else. +static std::optional as_dm(std::optional convo) { + if (!convo) + return std::nullopt; + if (auto* dm = convo->dm()) + return *dm; + return std::nullopt; +} + +void Client::dm( + const ConversationId& id, + std::function, std::optional)> cb) { + _require_dm("dm", id); + _async([this, id] { return as_dm(_conversation(id)); }, std::move(cb)); +} + +std::optional Client::dm(const ConversationId& id, await_t) { + _require_dm("dm", id); + return loop.call_get([this, id] { return as_dm(_conversation(id)); }); +} + +void Client::open_dm( + const ConversationId& id, + std::function, std::optional)> cb) { + _require_dm("open_dm", id); + _async([this, id] { return as_dm(std::optional{_create_conversation(id)}); }, + std::move(cb)); +} + +DM Client::open_dm(const ConversationId& id, await_t) { + _require_dm("open_dm", id); + return loop.call_get([this, id] { + return *as_dm(std::optional{_create_conversation(id)}); + }); +} + +void Client::message( + int64_t id, std::function, std::optional)> cb) { + _async([this, id] { return _message(id); }, std::move(cb)); +} + +void Client::save_attachment( + int64_t message_id, + size_t index, + std::filesystem::path dest, + std::function on_progress, + failable_function cb, + bool notify_sender, + bool replace) { + + // Checked on the calling thread so a caller's own mistake surfaces at the call site, where they + // still have a stack to make sense of it. + if (std::filesystem::is_directory(dest)) + throw std::invalid_argument{"save_attachment: {} is a directory"_format(dest.string())}; + if (auto dir = dest.parent_path(); !dir.empty() && !std::filesystem::is_directory(dir)) + throw std::invalid_argument{"save_attachment: {} does not exist"_format(dir.string())}; + + // Not _async: what that reports is the *start* of the transfer, and the answer a caller wants + // is whether the file arrived, which is minutes away. So the callback is carried down to the + // download's own completion, and only the failures that happen before it starts come back here. + loop.call([this, + message_id, + index, + dest = std::move(dest), + on_progress = std::move(on_progress), + cb, + notify_sender, + replace]() mutable { + try { + _save_attachment( + message_id, + index, + std::move(dest), + std::move(on_progress), + cb, + notify_sender, + replace); + } catch (const std::exception& e) { + log_operation_failure(e); + _report(cb, std::optional{std::string{e.what()}}, std::filesystem::path{}); + } + }); +} + +// -- Conversations ---------------------------------------------------------------------------- + +// Only a DM has a name source so far; groups and communities gain one with the features. The same +// goes for the picture: a group's and a community's live elsewhere and are not wired up yet, so +// those columns come back null and the conversation reports no picture. +static const auto CONVO_COLUMNS = R"( + SELECT c.id, a.session_id, g.group_id, m.base_url, m.room, + -- A nickname is ours for them and wins over the name they chose for themselves; falling + -- back means an account we have seen but never made a contact of still has a name. + coalesce(ct.nickname, a.name), c.last_activity, + -- The most recent message that still says something, for `last_preview`. `lm.id` is + -- NULL exactly when there is nothing to preview, which is what leaves the optional + -- unset; the attachment side of the preview is filled in afterwards, in one query for + -- the whole list. + lm.id, lm.body, lm.outgoing, + c.unread_count, c.priority, coalesce(ct.approved, 0), coalesce(ct.approved_me, 0), + c.marked_unread, coalesce(ct.blocked, 0), a.name, ct.nickname, + c.notifications, c.mute_until, c.exp_mode, c.exp_timer, + a.profile_pic_url, a.profile_pic_key, c.auto_download + {} + -- Joined rather than subqueried per column so that one index seek on messages_history serves + -- every field of the preview. A conversation with nothing to preview simply misses. + LEFT JOIN messages lm ON lm.id = ( + SELECT b.id FROM messages b + WHERE b.conversation = c.id AND b.deleted IS NULL + ORDER BY b.timestamp DESC, b.id DESC LIMIT 1) +)"_format(SUBJECT_JOIN); + +// Which conversations are message requests, as a fragment both list queries need: a DM with someone +// we have never written to. No entry at all counts, which is the usual case -- a stranger's first +// message creates the row that says they have approved us and nothing that says we approved them. +// +// Note to self is exempt because it cannot be a request; we are not our own contact, and the entry +// that would carry the approval is one that has no business existing. +static constexpr auto IS_REQUEST = + "(c.dm IS NOT NULL AND coalesce(ct.approved, 0) = 0 AND a.session_id IS NOT ?1)"sv; + +// Fills in the attachment side of the `last_preview` of every conversation that has one. +// `previews` pairs the previewed message with the index of the conversation it belongs to. +// +// One query for the whole list rather than one per row: message_attachments is keyed on +// (message, idx), so `message IN (...)` is one index range scan per message, and that key is also +// what makes `ORDER BY message, idx` free -- which matters, because the names have to come back in +// the order the sender listed them. Same reasoning, and the same prepared-statement-per-list-size +// caveat, as `load_attachments`. +// +// Messages with no attachments simply do not come back, and need not: a default-constructed +// preview already says a message has none. +static void load_preview_attachments( + sqlite::Connection& c, + std::vector& convos, + const std::vector>& previews) { + if (previews.empty()) + return; + + std::unordered_map at; + std::vector ids; + ids.reserve(previews.size()); + for (const auto& [msg, i] : previews) { + at.emplace(msg, i); + ids.push_back(msg); + } + + // A row per attachment rather than an aggregate, because the names are wanted individually; the + // three summary fields are then folded from the same rows instead of being asked for again. + // Only the three columns a preview uses, so a list does not carry the sizes, captions and urls + // that a message view reads. + // + // `substr(...) = 'image/'` rather than `LIKE 'image/%'` because LIKE is ASCII-case-insensitive + // in SQLite while `gallery_viewable`'s `starts_with` is not, and the two deciding differently + // about `image/PNG` is exactly the sort of disagreement nobody would think to look for. + // Every attachment starts an all-images run that its own answer then confirms or ends, so the + // flag means "at least one, and all of them" without a separate count to compare against. + for (auto&& [msg, filename, is_image, flags] : + c.prepared_results, int, int>( + R"( + SELECT message, filename, + content_type IS NOT NULL AND substr(content_type, 1, 6) = 'image/', + flags + FROM message_attachments WHERE message IN ({}) ORDER BY message, idx + )"_format(sqlite::placeholders(ids.size())), + sqlite::bind_each{ids})) { + auto found = at.find(msg); + if (found == at.end()) + continue; + auto& preview = convos[found->second].base().last_preview; + if (!preview) + continue; + + bool first = preview->filenames.empty(); + preview->filenames.push_back(std::move(filename).value_or("")); + preview->all_images = (first || preview->all_images) && is_image != 0; + if (flags & ATTACHMENT_FLAG_VOICE_MESSAGE) + preview->voice_message = true; + } +} + +template +static std::vector query_conversations( + Client& client, sqlite::Connection& c, const std::string& query, const Bind&... bind) { + std::vector out; + // Message id and the index of the conversation it previews; see load_preview_attachments. + std::vector> preview_msgs; + for (auto [convo, + sid, + gid, + url, + room, + display_name, + activity, + prev_id, + prev_body, + prev_outgoing, + unread, + priority, + approved, + approved_me, + marked_unread, + blocked, + name, + nickname, + notifications, + mute_until, + exp_mode, + exp_timer, + pic_url, + pic_key, + auto_download] : + c.prepared_results< + int64_t, + std::optional>, + std::optional>, + std::optional, + std::optional, + std::optional, + int64_t, + std::optional, + std::optional, + std::optional, + int, + int, + int, + int, + int, + int, + std::optional, + std::optional, + int, + int64_t, + int, + int64_t, + std::optional, + std::optional, + std::optional>(query, bind...)) { + Conversation base{client, subject_to_id(convo, sid, gid, url, room)}; + if (auto_download) + base.auto_download = static_cast(*auto_download); + if (pic_url && pic_key) { + base.picture.url = *pic_url; + base.picture.key.assign(pic_key->begin(), pic_key->end()); + } + base.display_name = display_name.value_or(""); + if (prev_id) { + // Attachment fields are left at their defaults here and filled in below; a message with + // no attachments is already correct as it stands. + base.last_preview = MessagePreview{ + .body = std::move(prev_body).value_or(""), + .outgoing = prev_outgoing.value_or(0) != 0}; + // The index this conversation is about to occupy: exactly one is appended per row. + preview_msgs.emplace_back(*prev_id, out.size()); + } + base.last_activity = from_epoch_ms(activity); + base.unread = unread; + base.marked_unread = marked_unread != 0; + base.priority = priority; + base.notifications = static_cast(notifications); + base.mute_until = as_sys_seconds(mute_until); + base.exp_mode = static_cast(exp_mode); + base.exp_timer = std::chrono::seconds{exp_timer}; + + // The same branch that decided which identity the row joined to decides which kind it is: + // exactly one of them is set, which is what the table's CHECK constraint enforces. + if (gid) + out.emplace_back(Group{std::move(base)}); + else if (url && room) + out.emplace_back(Community{std::move(base)}); + else { + bool me = client.is_me(*sid); + DM dm{std::move(base)}; + dm.request = !me && !approved; + dm.awaiting_approval = !me && !approved_me; + dm.note_to_self = me; + dm.blocked = blocked != 0; + dm.name = name.value_or(""); + dm.nickname = nickname.value_or(""); + out.emplace_back(std::move(dm)); + } + } + + // After the loop, not inside it: the point of batching is that the statement above has finished + // and every previewed message is known, so this is one query rather than one per row. + load_preview_attachments(c, out, preview_msgs); + return out; +} + +// Our own session ID, or empty when no account exists yet. Reads are expected to work before +// onboarding under defer_account -- "no account" and "no conversations" are the same answer -- so +// nothing on a read path may reach for the identity unconditionally. +std::span Client::_self_or_none() { + if (!core.globals.have_account()) + return {}; + return core.globals.session_id(); +} + +std::vector Client::_conversations() { + auto c = core.database().conn(); + return query_conversations( + *this, + c, + // Hidden (negative priority) conversations are not part of the list at all; pinned ones + // lead it, and equal priorities form a block that sorts among itself by recency. + "{} WHERE c.priority >= 0 AND NOT {} ORDER BY c.priority DESC, c.last_activity DESC, c.id"_format( + CONVO_COLUMNS, IS_REQUEST), + _self_or_none()); +} + +std::vector Client::_message_requests() { + auto c = core.database().conn(); + // No priority ordering: a request cannot be pinned -- pinning is a property of the config entry + // and there is nothing there to pin until it is approved -- so recency is the only order there + // is. Hidden ones are still omitted, since hiding is the one thing another device *can* say + // about a request it does not want to see. + return query_conversations( + *this, + c, + "{} WHERE c.priority >= 0 AND {} ORDER BY c.last_activity DESC, c.id"_format( + CONVO_COLUMNS, IS_REQUEST), + _self_or_none()); +} + +std::optional Client::_conversation(const ConversationId& id) { + auto c = core.database().conn(); + auto convo = find_conversation(c, id); + if (!convo) + return std::nullopt; + auto found = query_conversations(*this, c, "{} WHERE c.id = ?"_format(CONVO_COLUMNS), *convo); + if (found.empty()) + return std::nullopt; + return std::move(found.front()); +} + +AnyConversation Client::_create_conversation(const ConversationId& id) { + bool created, contacted = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + created = ensure_conversation(c, id, clock_now_ms()).created; + + // Opening a conversation with someone makes them an approved contact: choosing to write to + // them is what approving them is. It is also the only way the conversation can be synced + // at all, since everything a one-to-one conversation carries lives in that entry. + // + // Deliberately not what an incoming message from a stranger does -- that is a message + // request, and stays one until we answer it. + if (id.type() == ConversationId::Type::dm && !is_note_to_self(id)) + contacted = ensure_contact(c, identity_id(c, id), true); + tx.commit(); + } + if (created || contacted) + _sync_conversation(id); + if (created) + _emit_conversation_added(id); + return *_conversation(id); +} + +void Client::_mark_read(const ConversationId& id, std::optional up_to) { + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + int64_t target; + if (up_to) + target = epoch_ms(*up_to); + else { + // "Everything" means every message that exists now, not every message that ever will: + // parking the watermark at infinity would silently mark all future arrivals read. + auto newest = c.prepared_get>( + "SELECT max(timestamp) FROM messages WHERE conversation = ? AND {}"_format( + UNREAD), + *convo); + if (!newest) + return; + target = *newest; + } + + // Moving last_read changes what counts as unread without touching any message, so the + // triggers cannot see it; recompute here, where it is rare, rather than per list query. + changed = c.prepared_exec( + R"( + UPDATE conversations + SET last_read = ?1, + unread_count = (SELECT COUNT(*) FROM messages + WHERE conversation = ?2 AND {} AND timestamp > ?1) + WHERE id = ?2 AND last_read < ?1 + )"_format(UNREAD), + target, + *convo); + + // Reading a conversation undoes having marked it unread, which is the only thing that + // could still be holding it bold. Separate from the watermark because it survives having + // read everything, so moving the watermark cannot clear it as a side effect. + changed += c.prepared_exec( + "UPDATE conversations SET marked_unread = 0 WHERE id = ? AND marked_unread", + *convo); + tx.commit(); + } + if (changed > 0) { + _sync_convo_volatile(id); + _touch(id); + } +} + +void Client::_set_marked_unread(const ConversationId& id, bool unread) { + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + changed = c.prepared_exec( + "UPDATE conversations SET marked_unread = ?2" + " WHERE id = ?1 AND marked_unread IS NOT ?2", + *convo, + unread ? 1 : 0); + tx.commit(); + } + if (changed > 0) { + _sync_convo_volatile(id); + _touch(id); + } +} + +// The settings below all live on the conversation row and all reach the config the same way, so +// they share one shape: update where it differs, and if it did, re-derive and report. +template +void Client::_set_conversation_setting(const ConversationId& id, std::string_view column, T value) { + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + changed = c.prepared_exec( + "UPDATE conversations SET {0} = ?2 WHERE id = ?1 AND {0} IS NOT ?2"_format(column), + *convo, + value); + tx.commit(); + } + if (changed > 0) { + _sync_conversation(id); + _touch(id); + } +} + +void Client::_set_notifications(const ConversationId& id, config::notify_mode mode) { + _set_conversation_setting(id, "notifications", static_cast(mode)); +} + +void Client::_set_mute_until(const ConversationId& id, std::chrono::sys_seconds until) { + _set_conversation_setting(id, "mute_until", epoch_seconds(until)); +} + +void Client::_set_expiry( + const ConversationId& id, config::expiration_mode mode, std::chrono::seconds timer) { + // A timer with no mode never expires anything, so storing one would be a value that reads as a + // setting and behaves as nothing. + if (mode == config::expiration_mode::none) + timer = 0s; + + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + changed = c.prepared_exec( + R"( + UPDATE conversations SET exp_mode = ?2, exp_timer = ?3 + WHERE id = ?1 AND (exp_mode, exp_timer) IS NOT (?2, ?3) + )", + *convo, + static_cast(mode), + static_cast(timer.count())); + tx.commit(); + } + if (changed > 0) { + _sync_conversation(id); + _touch(id); + } +} + +void Client::_set_auto_download(const ConversationId& id, AutoDownload mode) { + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + changed = c.prepared_exec( + R"( + UPDATE conversations SET auto_download = ?2 + WHERE id = ?1 AND auto_download IS NOT ?2 + )", + *convo, + static_cast(mode)); + tx.commit(); + } + // No `_sync_conversation`: this one is ours alone. Every other setting here is derived into a + // config on the way out, and putting this in one would publish a decision about this machine's + // disk to every other device on the account. + if (changed > 0) + _touch(id); +} + +void Client::_set_nickname(const ConversationId& id, std::string_view nickname) { + bool changed = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto account = identity_id(c, id); + + // A nickname is a fact about a relationship, so it needs the row that carries one -- the + // same reasoning as blocking. Naming somebody is not approving them. + bool made = ensure_contact(c, account, false); + bool set = c.prepared_exec( + "UPDATE contacts SET nickname = ?2" + " WHERE account = ?1 AND nickname IS NOT ?2", + account, + nickname.empty() ? std::optional{} + : std::optional{nickname}) > 0; + changed = made || set; + tx.commit(); + } + if (changed) { + _sync_contact(id); + _touch(id); + } +} + +void Client::_set_priority(const ConversationId& id, int priority) { + int changed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + // Not ensure_conversation() for one that already exists: that moves last_activity forward, + // and pinning is not activity -- it would reorder the very list it is being used to order. + auto existing = find_conversation(c, id); + auto convo = existing ? *existing : ensure_conversation(c, id, clock_now_ms()).id; + + changed = c.prepared_exec( + "UPDATE conversations SET priority = ?1 WHERE id = ?2 AND priority IS NOT ?1", + priority, + convo); + tx.commit(); + } + + if (changed > 0) { + _sync_conversation(id); + _emit_lists_replaced(); + } +} + +void Client::_set_blocked(const ConversationId& id, bool blocked) { + bool changed = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + auto account = identity_id(c, id); + + // Blocking someone we have never made a contact of is the ordinary case rather than the + // exception -- a message request is exactly that -- so the row is created rather than + // required. Not approved by it: refusing someone's messages is not accepting them. + changed = ensure_contact(c, account, false); + changed |= c.prepared_exec( + "UPDATE contacts SET blocked = ?2" + " WHERE account = ?1 AND blocked IS NOT ?2", + account, + blocked ? 1 : 0) > 0; + tx.commit(); + } + + if (!changed) + return; + _sync_contact(id); + _touch(id); +} + +void Client::_clear_messages(const ConversationId& id) { + auto now = clock_now_ms(); + bool emptied = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + emptied = delete_messages_before(c, *convo, now); + tx.commit(); + } + + // Published whether or not this device had anything to delete: the instruction is about what + // every device holds, and finding none here says nothing about the rest. + _set_delete_before(id, now); + + if (emptied) { + _emit_history_replaced(id); + _touch(id); + } +} + +void Client::_delete_conversation(const ConversationId& id, bool keep_messages) { + auto now = clock_now_ms(); + bool emptied = false, hidden = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return; + + if (!keep_messages) + emptied = delete_messages_before(c, *convo, now); + + // Hidden rather than deleted, and that is the whole difference from delete_contact: what + // says the conversation exists is the config entry, which stays, so removing the row would + // only have it reinstated by the next reconciliation. A pinned conversation loses its pin + // along the way, which is why this is not conditional on the priority being 0. + hidden = c.prepared_exec( + "UPDATE conversations SET priority = -1 WHERE id = ? AND priority >= 0", + *convo) > 0; + tx.commit(); + } + + if (!keep_messages) + _set_delete_before(id, now); + _sync_conversation(id); + + if (emptied) + _emit_history_replaced(id); + if (hidden) + _emit_lists_replaced(); +} + +void Client::_delete_contact(const ConversationId& id) { + bool removed = false; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto account = c.prepared_maybe_get( + "SELECT id FROM accounts WHERE session_id = ?", id.session_id()); + if (!account) + return; + + // The nickname, both approvals and the block are columns of the row being deleted, so + // there is nothing to reset first: they exist only for as long as the relationship does. + c.prepared_exec("DELETE FROM contacts WHERE account = ?", *account); + + // Messages, their raw content and their attachments follow the conversation by cascade; + // the files those attachments name belong to the user and are not unlinked. The account + // row stays -- see _reconcile_contacts, which deletes the same way for the same reasons. + removed = c.prepared_exec("DELETE FROM conversations WHERE dm = ?", *account) > 0; + tx.commit(); + } + + // No delete-before instruction here, and none is owed: the entry going from the config is + // itself the instruction, and a device merging that removes the conversation and its history. + _sync_contact(id); + + if (removed) { + _emit_conversation_removed(id); + _emit_lists_replaced(); + } +} + +bool Client::_delete_message(int64_t message_id, Deletion how_far) { + std::optional convo; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo_row = c.prepared_maybe_get( + "SELECT conversation FROM messages WHERE id = ?", message_id); + if (!convo_row) + return false; + + // Not `= ?` : a message deleted here and then deleted everywhere has to move, while one + // already at `everywhere` must not be walked back to `here` by a later local delete. The + // reach of a deletion only ever grows. + c.prepared_exec( + R"( + UPDATE messages + SET body = '', deleted = max(coalesce(deleted, 0), ?2) + WHERE id = ?1 + )", + message_id, + static_cast(how_far)); + + // The decrypted Content, which is where the body actually survives: leaving it would make + // this a deletion only of the copy that happens to be easy to read. + c.prepared_exec("DELETE FROM message_raw_content WHERE message = ?", message_id); + + // The rows describing the attachments, not the files they name. Those belong to the user + // in both directions -- a file they chose to send, or a place they asked a download to be + // put -- and nothing here has ever unlinked one. + c.prepared_exec("DELETE FROM message_attachments WHERE message = ?", message_id); + + // There is nothing left to read, so it stops being unread. Recomputed rather than + // decremented because whether it counted in the first place is the policy in UNREAD, and + // guessing that here is how the cached count drifts from what the query would say. + c.prepared_exec( + R"( + UPDATE conversations SET unread_count = ( + SELECT COUNT(*) FROM messages + WHERE messages.conversation = conversations.id AND {} + AND messages.timestamp > conversations.last_read) + WHERE id = ? + )"_format(UNREAD), + *convo_row); + + convo = conversation_id_at(c, *convo_row); + tx.commit(); + } + + if (convo) { + // Updated rather than removed: the row is still there, and a client that draws a gap where + // the message was needs to be told what it now says rather than that it went. + _emit_message(false, *convo, message_id); + // The list shows the newest message's body, which may be the one just emptied. + _touch(*convo); + } + return true; +} + +void Client::_on_unsend_request( + std::span sender, const SessionProtos::UnsendRequest& req) { + if (!req.has_author() || !req.has_msgtimestamp()) + return; + + b33 author; + { + auto raw = oxenc::from_hex(req.author()); + if (raw.size() != author.size()) + return; + std::memcpy(author.data(), raw.data(), author.size()); + } + + // Honoured from the message's own author, or from ourselves -- which is how this arrives when + // another of our devices deletes something we sent. Anyone else asking us to destroy a message + // is asking for something that is not theirs to ask. + if (!std::ranges::equal(sender, author) && !is_me(sender)) { + log::warning(cat, "Ignoring unsend request from someone who did not write the message"); + return; + } + + auto ts = epoch_ms(from_epoch_ms(static_cast(req.msgtimestamp()))); + + auto c = core.database().conn(); + std::vector found; + if (req.has_msgid()) { + // Both halves: this identifies one message even among several sent in the same millisecond, + // which is the case the id exists for. + for (auto id : c.prepared_results( + R"( + SELECT m.id FROM messages m JOIN accounts a ON a.id = m.sender + WHERE m.timestamp = ? AND a.session_id = ? AND m.msgid = ? + )", + ts, + author, + req.msgid())) + found.push_back(id); + } else { + for (auto id : c.prepared_results( + R"( + SELECT m.id FROM messages m JOIN accounts a ON a.id = m.sender + WHERE m.timestamp = ? AND a.session_id = ? + )", + ts, + author)) + found.push_back(id); + } + + if (found.empty()) + return; + + // A request that names more than one message is not a weaker match but an ambiguous one, and + // the messages a sender most often unsends -- several lines pasted and thought better of -- are + // exactly the ones that share a millisecond. Deleting the wrong one destroys something nobody + // asked to lose, so leave it rather than guess. + if (found.size() > 1) { + log::warning( + cat, + "Ignoring unsend request matching {} messages: cannot tell which was meant", + found.size()); + return; + } + + // Marked as deleted everywhere rather than only here: this is the author saying it is gone, not + // us tidying our own copy. + _delete_message(found.front(), Deletion::everywhere); + + // And our own swarm's copy of it, so it cannot be delivered to us again and so our other + // devices stop seeing it too. Ours to delete: it was stored in our swarm for us. + if (auto hash = c.prepared_maybe_get>( + "SELECT swarm_hash FROM messages WHERE id = ?", found.front()); + hash && *hash && core.network()) + core.delete_from_swarm({**hash}, [](bool ok) { + if (!ok) + log::warning(cat, "Could not delete our swarm copy of an unsent message"); + }); +} + +bool Client::_delete_message_everywhere(int64_t message_id) { + std::optional swarm_hash; + std::optional recipient; + std::optional msgid; + sys_ms timestamp{}; + { + auto c = core.database().conn(); + auto row = c.prepared_maybe_get< + int, + std::optional, + int64_t, + std::optional, + sqlite::blob_guts>( + R"( + SELECT m.outgoing, m.swarm_hash, m.timestamp, m.msgid, a.session_id + FROM messages m + JOIN conversations c ON c.id = m.conversation + JOIN accounts a ON a.id = c.dm + WHERE m.id = ? + )", + message_id); + if (!row) + return false; + + auto [outgoing, hash, ts, mid, peer] = *row; + // The other end honours an unsend only from the author or from one of their own devices, so + // for a message we did not write there is nothing to ask for. + if (!outgoing) + return false; + + swarm_hash = std::move(hash); + timestamp = from_epoch_ms(ts); + msgid = mid; + recipient = peer; + } + + // The local half first, and on its own terms: it is the part that is certain, and the caller is + // told about this rather than about what the network does with the rest. + if (!_delete_message(message_id, Deletion::everywhere)) + return false; + + // Our own swarm's copy -- what our other devices read. Not the recipient's: that one is in + // their swarm and only they can remove it, which is what the request below asks for. + if (swarm_hash && core.network()) + core.delete_from_swarm({*swarm_hash}, [](bool ok) { + if (!ok) + log::warning(cat, "Could not delete our own swarm copy of an unsent message"); + }); + + // Best effort, and deliberately not waited on: there is no acknowledgement, and a recipient may + // be offline, or running something that ignores this entirely. + if (recipient && core.network() && !is_me(*recipient)) { + auto now = clock_now_ms(); + SessionProtos::Content content; + content.set_sigtimestamp(static_cast(epoch_ms(now))); + auto* req = content.mutable_unsendrequest(); + req->set_msgtimestamp(static_cast(epoch_ms(timestamp))); + req->set_author(oxenc::to_hex(core.globals.session_id())); + // Both halves of the identity where we have them: the timestamp alone cannot separate two + // messages sent in the same millisecond, and those are exactly what someone deletes. + if (msgid) + req->set_msgid(*msgid); + + // Registered rather than fired blind: Core reports on every send, and a status for an id + // nobody claims would sit in _early_status for the life of the process. + _quiet_sends.insert(core.send_dm(*recipient, content, now)); + } + + return true; +} + +// The two purges below are the only thing here that removes a message row outright rather than +// emptying it, so both are written to be incapable of touching anything a deletion did not leave: +// the predicate is part of the statement, not a check made before it. +void Client::_auto_download(const ConversationId& convo_id, int64_t message_id) { + // Nowhere to put it: fetching early exists to have the file to hand later, and without a cache + // the download would be decrypted, held, and dropped. + if (_cache_dir.empty()) + return; + + auto convo = _conversation(convo_id); + if (!convo) + return; + + // Unset means nobody has been asked yet, which is not consent. Nothing is fetched until a + // client has put the question to somebody. + auto mode = convo->auto_download(); + if (!mode || *mode == AutoDownload::none) + return; + + auto msg = _message(message_id); + if (!msg || msg->attachments.empty()) + return; + + // A gallery is the display this arrived ready for, so it is decided now rather than left to + // whoever opens the conversation: the setting may have changed by then, and the answer is meant + // to describe how the message arrived. + if (msg->gallery_viewable) + _set_gallery(message_id, true); + + auto max_size = core.globals.get_integer(AUTO_DL_MAX_KEY); + + for (const auto& a : msg->attachments) { + if (*mode == AutoDownload::image_attachments && + !(a.content_type && a.content_type->starts_with("image/"))) + continue; + + // The sender's claim, and all we have before fetching anything. A sender who under-reports + // is not caught here -- what catches that is the transfer being held to the size it + // declared -- but this is what stops us starting a download nobody wanted. + if (max_size && (!a.size || *a.size > *max_size)) + continue; + + // No completion handler: this wants the cache filled and nothing else, and whoever + // eventually displays the file joins the transfer rather than starting another. + // + // Progress is broadcast rather than handed to a caller, because there is no caller: a + // display that opens midway learns from this that something is already happening. + _attachment_data( + message_id, + a.index, + [this, convo_id](const AttachmentProgress& p) { + if (const auto& h = _cbs->attachment_progress) + h(convo_id, p); + }, + nullptr); + } +} + +bool Client::_set_gallery(int64_t message_id, bool gallery) { + // Asked of the message rather than of the row: `gallery_viewable` is about the attachments, and + // `_message` is what assembles those. It also applies the same drop of a stale decision that + // every other read does, so this cannot be the one path that disagrees. + auto msg = _message(message_id); + if (!msg) + return false; + + // Turning it off is always allowed -- refusing that would leave a message stuck showing a way + // it is no longer willing to be shown. Turning it on is not, so that what is stored can never + // contradict the rule. + if (gallery && !msg->gallery_viewable) + return false; + + std::optional convo; + { + auto c = core.database().conn(); + if (c.prepared_exec( + "UPDATE messages SET gallery = ?2 WHERE id = ?1 AND gallery IS NOT ?2", + message_id, + gallery ? 1 : 0) == 0) + return true; // already what was asked for + convo = msg->conversation; + } + + if (convo) + _emit_message(false, *convo, message_id); + return true; +} + +bool Client::_purge_deleted_message(int64_t message_id) { + std::optional convo; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo_row = c.prepared_maybe_get( + "SELECT conversation FROM messages WHERE id = ? AND deleted IS NOT NULL", + message_id); + if (!convo_row) + return false; + + c.prepared_exec("DELETE FROM messages WHERE id = ? AND deleted IS NOT NULL", message_id); + convo = conversation_id_at(c, *convo_row); + tx.commit(); + } + + if (convo) { + _emit_history_replaced(*convo); + _touch(*convo); + } + return true; +} + +size_t Client::_purge_deleted(const ConversationId& id) { + int removed = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = find_conversation(c, id); + if (!convo) + return 0; + + removed = c.prepared_exec( + "DELETE FROM messages WHERE conversation = ? AND deleted IS NOT NULL", *convo); + tx.commit(); + } + + // `count` follows by trigger; `unread_count` cannot have changed, since a deleted message had + // already stopped counting as unread when it was deleted. + if (removed > 0) { + _emit_history_replaced(id); + _touch(id); + } + return static_cast(removed); +} + +void Client::_set_delete_before(const ConversationId& id, sys_ms before) { + // The configs carry seconds. Truncating rather than rounding is what keeps the instruction + // from reaching a message sent just after it: this device has already deleted to the + // millisecond, so the others delete the same set less any straggler within the same second. + auto secs = std::chrono::floor(before); + + // Never backwards, in either config: an instruction to destroy history is not something a + // later but smaller value is entitled to take back. + if (is_note_to_self(id)) { + auto& profile = core.configs.user_profile(); + if (profile.get_nts_delete_before() < secs) + profile.set_nts_delete_before(secs); + return; + } + + auto& contacts = core.configs.contacts(); + auto entry = contacts.get(oxenc::to_hex(id.session_id())); + if (!entry || entry->delete_before >= secs) + return; + entry->delete_before = secs; + contacts.set(*entry); +} + +// Both lists, always, and deliberately not one or the other: what moves a conversation between them +// is approval, what removes it from either is hiding or deletion, and a caller that had to work out +// which of those it just did would eventually get it wrong. A replacement is idempotent, so the +// cost of sending one nobody needed is a query. +void Client::_emit_lists_replaced() { + auto convos = _conversations(); + auto requests = _message_requests(); + _emit([convos = std::move(convos), + requests = std::move(requests)](const callbacks& cbs) mutable { + if (cbs.conversation_list_replaced) + cbs.conversation_list_replaced(std::move(convos)); + if (cbs.request_list_replaced) + cbs.request_list_replaced(std::move(requests)); + }); +} + +// -- Config reconciliation ---------------------------------------------------------------------- + +void Client::_on_configs_changed(std::span changed) { + bool conversations_moved = false; + for (auto ns : changed) { + switch (ns) { + case config::Namespace::UserProfile: + _reconcile_user_profile(); + conversations_moved = true; + break; + case config::Namespace::Contacts: + _reconcile_contacts(); + conversations_moved = true; + break; + default: break; // The rest land with the configs that model them. + } + } + + // Last, and also whenever anything above may have created a conversation: read state is about + // conversations rather than a statement that they exist, so an entry for one we do not have is + // skipped -- and the config that would have created it may only just have been applied. + if (conversations_moved || + std::ranges::find(changed, config::Namespace::ConvoInfoVolatile) != changed.end()) + _reconcile_convo_volatile(); +} + +void Client::_reconcile_all() { + // Outward before inward, and the order is load-bearing: reconciling inward can delete a contact + // the config does not mention, and a row whose dump was never written looks exactly like one + // deleted elsewhere. Publishing what we hold first tells those two apart. + _sync_all_contacts(); + _sync_all_convo_volatile(); + + // Each config joins this as it gains a reconciler; the sweep is what makes adding one apply to + // state that arrived before it existed, rather than only to the next change after it. + _reconcile_user_profile(); + _reconcile_contacts(); + + // After the two above, which are what bring conversations into being. + _reconcile_convo_volatile(); +} + +bool Client::_update_profile( + sqlite::Connection& c, + int64_t account, + const std::optional& name, + const std::optional& pic_url, + const std::optional>& pic_key, + int64_t updated, + ProfileSource source) { + + auto was = c.prepared_maybe_get( + "SELECT profile_pic_url FROM accounts WHERE id = ?", account); + + // `IS NOT` rather than `!=`: these columns are NULL until a profile is known, and `NULL != 'x'` + // is NULL, so `!=` would never fire for the first name or picture we learn. + bool changed = source == ProfileSource::config ? c.prepared_exec( + R"( +UPDATE accounts SET name = ?2, profile_pic_url = ?3, profile_pic_key = ?4, profile_updated = ?5 +WHERE id = ?1 + AND (name, profile_pic_url, profile_pic_key, profile_updated) IS NOT (?2, ?3, ?4, ?5) +)", + account, + name, + pic_url, + pic_key, + updated) > 0 + : c.prepared_exec( + R"( +UPDATE accounts + SET name = coalesce(?2, name), + profile_pic_url = coalesce(?3, profile_pic_url), + profile_pic_key = coalesce(?4, profile_pic_key), + profile_updated = ?5 + WHERE id = ?1 + AND (name, profile_pic_url, profile_pic_key, profile_updated) + IS NOT (coalesce(?2, name), + coalesce(?3, profile_pic_url), + coalesce(?4, profile_pic_key), + ?5) +)", + account, + name, + pic_url, + pic_key, + updated) > 0; + + // Read back rather than compared against the argument: under `message` a null url leaves the + // old one in place, so what the account now points at is not what was passed in. + if (changed) { + auto now = c.prepared_maybe_get( + "SELECT profile_pic_url FROM accounts WHERE id = ?", account); + if (now != was) { + if (was) + _drop_unused_picture(c, *was); + // `call_soon`, not `call`: this runs inside the reconcile's transaction, and `call` + // runs inline when it is already on the loop, so the fetch would take a *different* + // pooled connection and read the url this update is in the middle of replacing -- + // fetching the picture we are on our way to discarding. `call_soon` always queues, so + // it runs after the commit and sees what was actually written. + // `call_soon`, not `call`: this runs inside the reconcile's transaction, and `call` + // runs inline when it is already on the loop, which would put a network fetch inside + // the write transaction and read the row through a second pooled connection that + // cannot see it yet. + if (now) + _prefetch_picture(c, account, *now); + } + } + + return changed; +} + +void Client::_prefetch_picture(sqlite::Connection& c, int64_t account, const std::string& url) { + // Nowhere to keep it: fetching now would only mean fetching again when it is asked for. + if (_cache_dir.empty()) + return; + + try { + // Read here, on the connection that is *in* the transaction, and handed to the fetch as + // values. The fetch itself runs later and cannot read any of this back: the write it is + // reacting to belongs to a transaction that commits somewhere above us, and a second pooled + // connection sees the row either as it was or not at all. + // + // `blob_guts` rather than a loose byte string: a display picture key is 32 bytes under + // both schemes, so anything else is not a key we could decrypt with, and rejecting it here + // is the same answer the download would give later. + auto row = c.prepared_maybe_get, sqlite::blob_guts>( + "SELECT session_id, profile_pic_key FROM accounts " + "WHERE id = ? AND profile_pic_key IS NOT NULL", + account); + if (!row) + return; + + auto& [sid, key] = *row; + + loop.call_soon([this, + id = ConversationId::dm(sid), + url, + key = std::vector{key.begin(), key.end()}]() mutable { + _fetch_picture(id, std::move(url), std::move(key)); + }); + } catch (const std::exception& e) { + // Best-effort by nature: nothing asked for this, so nothing is owed an error, and this is + // inside somebody's transaction -- letting it out would undo the profile update that + // prompted it. The picture is fetched anyway the first time something asks for it. + log::warning(cat, "Could not start a profile picture fetch: {}", e.what()); + } +} + +void Client::_fetch_picture(const ConversationId& id, std::string url, std::vector key) { + try { + std::function)> progress; + if (_cbs->display_picture_progress) + progress = _dispatch_progress( + [this, id](int64_t done, int64_t total, std::optional r) { + _cbs->display_picture_progress(id, done, total, r); + }); + + // No handler for the bytes: this is not fetching *for* anyone, it is putting the file where + // whoever asks next will find it. Nobody is waiting on an answer to be told it is not + // coming, and a picture that will not come down now is tried again the moment something + // asks for it. + _fetch_cached( + {url, + std::move(key), + {}, + std::nullopt, + DownloadKind::display_pic, + cache::PROFILE_DIR}, + std::move(progress), + nullptr, + nullptr, + _store_picture(url)); + } catch (const std::exception& e) { + log::warning(cat, "Could not fetch a profile picture: {}", e.what()); + } +} + +void Client::_drop_unused_picture(sqlite::Connection& c, std::string_view url) { + if (url.empty() || _cache_dir.empty()) + return; + + // Cheap, and the cost of being wrong is a contact's picture disappearing for no reason they or + // we could explain. + if (c.prepared_get( + "SELECT EXISTS(SELECT 1 FROM accounts WHERE profile_pic_url = ?)", url)) + return; + + std::error_code ec; + std::filesystem::remove(cache::path_for(_cache_dir, cache::PROFILE_DIR, url), ec); +} + +void Client::_reconcile_contacts() { + auto& contacts = core.configs.contacts(); + auto me = core.globals.session_id(); + + std::vector touched; + std::vector added; + std::vector removed; + std::vector cleared; + bool order_changed = false, requests_changed = false; + + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + // Ordered rather than a vector because the deletion pass below looks every stored contact + // up in it: scanning instead would make a routine merge quadratic in the size of the + // contact list, which is exactly the size that is allowed to be large. + std::set in_config; + + for (const auto& entry : contacts) { + // Our own entry has no business being here -- our profile is UserProfile's, and we are + // not our own contact -- but another client may have written one, so it is skipped + // rather than trusted. + auto raw = oxenc::from_hex(entry.session_id); + if (raw.size() != 33) + continue; + b33 sid; + std::memcpy(sid.data(), raw.data(), 33); + if (is_me(sid)) + continue; + + in_config.insert(sid); + + auto id = ConversationId::dm(sid); + auto account = identity_id(c, id); + + std::optional pic_url; + std::optional> pic_key; + if (!entry.profile_picture.empty()) { + pic_url = entry.profile_picture.url; + pic_key = std::span{ + entry.profile_picture.key.data(), entry.profile_picture.key.size()}; + } + + // Only when the config's stamp is at least as new as ours: a name observed from a + // message that arrived late must not displace a newer one. + bool renamed = false; + if (epoch_seconds(entry.profile_updated) >= + c.prepared_get( + "SELECT profile_updated FROM accounts WHERE id = ?", account)) + renamed = _update_profile( + c, + account, + entry.name.empty() ? std::optional{} + : std::optional{entry.name}, + pic_url, + pic_key, + epoch_seconds(entry.profile_updated), + ProfileSource::config); + + // Pro flags are the profile's claim about itself, so they follow the profile. + c.prepared_exec( + "UPDATE accounts SET pro_flags = ?2 WHERE id = ?1 AND pro_flags IS NOT ?2", + account, + static_cast(entry.profile_flags)); + + // Read before the upsert rather than derived from it: what moves a conversation between + // the two lists is this one column, and the upsert reports only that *something* in the + // row changed. + bool was_request = !c.prepared_get( + "SELECT coalesce((SELECT approved FROM contacts WHERE account = ?), 0)", + account); + + // Existence here *is* being a contact, so this is an upsert rather than an update: the + // row appearing is the fact being recorded. + // + // Approval only ever goes up, in either direction, because neither has a reverse: what + // sets it is a message having been sent, and no later config can make that not have + // happened. Copying the value verbatim would let an un-approval reach us -- other + // clients clear both flags on their way to deleting a contact, and a device that merged + // the clearing but not the deletion would otherwise file the conversation back under + // message requests. + bool new_contact = + c.prepared_exec( + R"( +INSERT INTO contacts (account, nickname, approved, approved_me, blocked) VALUES (?1, ?2, ?3, ?4, ?5) +ON CONFLICT (account) DO UPDATE + SET nickname = ?2, approved = max(approved, ?3), approved_me = max(approved_me, ?4), blocked = ?5 +WHERE (nickname, approved, approved_me, blocked) + IS NOT (?2, max(approved, ?3), max(approved_me, ?4), ?5) +)", + account, + entry.nickname.empty() ? std::optional{} + : std::optional{entry.nickname}, + entry.approved ? 1 : 0, + entry.approved_me ? 1 : 0, + entry.blocked ? 1 : 0) > 0; + + // The config saying a contact exists is what brings the conversation into being -- a + // conversation is not defined by having messages in it, or emptying one would lose it. + auto convo = find_conversation(c, id); + bool created = false; + if (!convo) { + auto row = ensure_conversation( + c, id, entry.created > 0 ? from_epoch_s(entry.created) : clock_now_ms()); + convo = row.id; + created = row.created; + } + + auto settings_changed = + c.prepared_exec( + R"( +UPDATE conversations SET priority = ?2, notifications = ?3, mute_until = ?4, exp_mode = ?5, + exp_timer = ?6, created = min(created, ?7) +WHERE id = ?1 + AND (priority, notifications, mute_until, exp_mode, exp_timer, created) + IS NOT (?2, ?3, ?4, ?5, ?6, min(created, ?7)) +)", + *convo, + entry.priority, + static_cast(entry.notifications), + entry.mute_until, + static_cast(entry.exp_mode), + static_cast(entry.exp_timer.count()), + entry.created > 0 ? entry.created : epoch_seconds(clock_now_ms())) > 0; + + // Retroactive by definition: what a delete-before instruction is about is the history + // that was there when someone chose to destroy it, so it is applied to what we hold and + // not only to what arrives afterwards. Idempotent, so re-applying it on every merge + // costs a lookup and deletes nothing the second time. + bool history_changed = false; + if (entry.delete_before > std::chrono::sys_seconds{}) + history_changed = delete_messages_before(c, *convo, sys_ms{entry.delete_before}); + if (entry.delete_attach_before > std::chrono::sys_seconds{}) + history_changed |= + delete_attachments_before(c, *convo, sys_ms{entry.delete_attach_before}); + if (history_changed) + cleared.push_back(id); + + if (created) + added.push_back(id); + else if (renamed || new_contact || settings_changed || history_changed) + touched.push_back(id); + if (created || settings_changed) + order_changed = true; + if (was_request && entry.approved) + requests_changed = true; + } + + // A contact we hold that the merged config does not mention was removed on another device, + // and removing a contact takes the conversation and its history with it -- deliberately: + // someone who deletes a conversation means it deleted, not hidden on one device. Merely + // *hiding* is a negative priority and arrives as an ordinary settings change above, so an + // absent entry can only mean the stronger thing. + // + // The account row stays: we may have seen them in a group or community, and their profile + // is needed to render that. What is deleted is the relationship, not the person. + // + // Safe because the outward sweep runs first (see _reconcile_all): a contact of ours the + // config has never heard of gets published rather than reaching here. + std::vector doomed; + for (auto sid : c.prepared_results>( + "SELECT a.session_id FROM contacts ct JOIN accounts a ON a.id = ct.account")) + if (!in_config.count(sid)) + doomed.push_back(ConversationId::dm(sid)); + + for (const auto& id : doomed) { + auto account = c.prepared_get( + "SELECT id FROM accounts WHERE session_id = ?", id.session_id()); + c.prepared_exec("DELETE FROM contacts WHERE account = ?", account); + + // Messages, their raw content and their attachments go with the conversation, by + // cascade. The files those attachments name are the user's -- theirs to attach, or + // theirs to have saved -- so nothing is unlinked. + // + // That stops being the whole story once attachments are cached: a cached file *is* + // ours, and cascade will take the row naming it without any of our code running, + // leaving the file on disk with nothing left pointing at it. Whatever adds that cache + // has to collect the paths here, before this delete, rather than trusting the cascade. + if (c.prepared_exec("DELETE FROM conversations WHERE dm = ?", account) > 0) + removed.push_back(id); + } + + tx.commit(); + } + + for (const auto& id : added) + _emit_conversation_added(id); + for (const auto& id : cleared) + _emit_history_replaced(id); + for (const auto& id : touched) + _touch(id); + for (const auto& id : removed) + _emit_conversation_removed(id); + if (order_changed || requests_changed || !removed.empty()) + _emit_lists_replaced(); +} + +void Client::_sync_all_contacts() { + std::vector ids; + { + auto c = core.database().conn(); + for (auto sid : c.prepared_results>( + "SELECT a.session_id FROM contacts ct JOIN accounts a ON a.id = ct.account")) + ids.push_back(ConversationId::dm(sid)); + } + // Collected before syncing rather than while iterating: _sync_contact takes its own connection. + for (const auto& id : ids) + _sync_contact(id); +} + +void Client::_reconcile_convo_volatile() { + auto& volatiles = core.configs.convo_info_volatile(); + + std::vector touched; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + for (auto it = volatiles.begin_1to1(); it != volatiles.end(); ++it) { + const auto& entry = *it; + + auto raw = oxenc::from_hex(entry.session_id); + if (raw.size() != 33) + continue; + b33 sid; + std::memcpy(sid.data(), raw.data(), sid.size()); + auto id = ConversationId::dm(sid); + + // Read state about a conversation we do not have says nothing worth acting on, and + // acting on it would be wrong: an entry outlives the conversation it describes -- this + // config is pruned by age rather than by anything noticing a deletion -- so creating + // one here would resurrect what another device deleted. Whatever creates the + // conversation is reconciled first, and this runs again after it. + auto convo = find_conversation(c, id); + if (!convo) + continue; + + // Session Pro lives here rather than with the rest of what we know about an account, + // and it is about the account rather than the conversation, so it lands on `accounts`. + // The two halves are meaningless apart: an expiry with no tag cannot be checked. + std::optional> tag; + std::optional expiry; + if (entry.pro_revocation_tag && entry.pro_expiry_at > std::chrono::sys_seconds{}) { + tag = std::span{*entry.pro_revocation_tag}; + expiry = epoch_seconds(entry.pro_expiry_at); + } + bool pro_changed = c.prepared_exec( + R"( +UPDATE accounts SET pro_revocation_tag = ?2, pro_expiry = ?3 +WHERE id = ?1 AND (pro_revocation_tag, pro_expiry) IS NOT (?2, ?3) +)", + identity_id(c, id), + tag, + expiry) > 0; + + // Forwards only. The config does not enforce this -- it lets a value be written + // backwards on purpose, so that a client can reset one -- and a conflict between two + // devices at the same seqno resolves by a tie-break that knows nothing about which + // value is newer. Left alone, that would make messages someone has read unread again. + bool read_changed = c.prepared_exec( + R"( +UPDATE conversations +SET last_read = ?2, + unread_count = (SELECT COUNT(*) FROM messages + WHERE conversation = ?1 AND {} AND timestamp > ?2) +WHERE id = ?1 AND last_read < ?2 +)"_format(UNREAD), + *convo, + entry.last_read) > 0; + + // The flag is an ordinary setting, though: someone marking a conversation unread on + // another device is telling us to, and there is no ordering to preserve. + bool unread_changed = c.prepared_exec( + "UPDATE conversations SET marked_unread = ?2" + " WHERE id = ?1 AND marked_unread IS NOT ?2", + *convo, + entry.unread ? 1 : 0) > 0; + + if (pro_changed || read_changed || unread_changed) + touched.push_back(id); + } + + tx.commit(); + } + + // No deletion pass, and there must not be one: entries here are pruned by age -- thirty days + // unread, forty-five on push -- so an absent one means only that nothing has been read in it + // lately. Treating that as a removal the way Contacts does would destroy conversations for + // having been quiet. + + for (const auto& id : touched) + _touch(id); +} + +void Client::_sync_all_convo_volatile() { + std::vector ids; + { + auto c = core.database().conn(); + for (auto sid : c.prepared_results>( + "SELECT a.session_id FROM conversations c JOIN accounts a ON a.id = c.dm")) + ids.push_back(ConversationId::dm(sid)); + } + for (const auto& id : ids) + _sync_convo_volatile(id); +} + +void Client::_sync_convo_volatile(const ConversationId& id) { + if (id.type() != ConversationId::Type::dm) + return; // Groups and communities need UserGroups first. + + auto c = core.database().conn(); + auto row = c.prepared_maybe_get( + R"( + SELECT last_read, marked_unread FROM conversations + WHERE dm = (SELECT id FROM accounts WHERE session_id = ?) + )", + id.session_id()); + if (!row) + return; + auto [last_read, marked_unread] = *row; + + auto& volatiles = core.configs.convo_info_volatile(); + + // Built on the entry that is there, which is what keeps the Pro fields alive: they are set from + // a proof we verified rather than from any row here, so there is nothing to re-derive them + // from. + auto entry = volatiles.get_or_construct_1to1(oxenc::to_hex(id.session_id())); + + // Forwards only here too, and for the same reason as the merge: our value can be the stale one, + // and publishing it would tell every other device to unread what it has read. + entry.last_read = std::max(entry.last_read, last_read); + entry.unread = marked_unread != 0; + volatiles.set(entry); +} + +void Client::_sync_conversation(const ConversationId& id) { + if (id.type() != ConversationId::Type::dm) + return; // Groups and communities land with UserGroups. + + if (!is_note_to_self(id)) + return _sync_contact(id); + + auto c = core.database().conn(); + auto priority = c.prepared_maybe_get( + R"( + SELECT priority FROM conversations + WHERE dm = (SELECT id FROM accounts WHERE session_id = ?) + )", + id.session_id()); + + // No row means there is no note-to-self conversation, which is what a negative priority says + // in a config that has no entry to be absent -- see _reveal_note_to_self. + core.configs.user_profile().set_nts_priority(priority.value_or(-1)); +} + +void Client::_sync_contact(const ConversationId& id) { + if (id.type() != ConversationId::Type::dm || is_me(id.session_id())) + return; + + auto& contacts = core.configs.contacts(); + auto hex = oxenc::to_hex(id.session_id()); + + auto c = core.database().conn(); + auto row = c.prepared_maybe_get< + std::optional, // name + std::optional, // profile_pic_url + // By value rather than as a `blob`, which is a view into the column: the statement is + // finished by the time this tuple is returned, so a view in it is already dangling and + // the key reads back with its first bytes overwritten. + std::optional>, // profile_pic_key + int64_t, // profile_updated + int64_t, // pro_flags + std::optional, // nickname + int, // approved + int, // approved_me + int>( // blocked + R"( + SELECT a.name, a.profile_pic_url, a.profile_pic_key, a.profile_updated, a.pro_flags, + ct.nickname, ct.approved, ct.approved_me, ct.blocked + FROM accounts a JOIN contacts ct ON ct.account = a.id + WHERE a.session_id = ? + )", + id.session_id()); + + // No contacts row means this account is not a contact, so the config should not say it is. + // Presence following the row is what makes removing a contact just "delete the row and sync", + // rather than a second place that has to remember to erase the entry too. + if (!row) { + contacts.erase(hex); + return; + } + + auto [name, + pic_url, + pic_key, + profile_updated, + pro_flags, + nickname, + approved, + approved_me, + blocked] = *row; + + auto convo = c.prepared_maybe_get( + R"( + SELECT priority, notifications, mute_until, exp_mode, exp_timer, created + FROM conversations WHERE dm = (SELECT id FROM accounts WHERE session_id = ?) + )", + id.session_id()); + + // Built on top of whatever entry is already there, which is also how the delete-before + // instruction survives this: no row holds it -- it is an instruction about history rather than + // a property of the contact -- so there is nothing here to re-derive it from, and overwriting + // the entry wholesale would quietly revoke it. + auto entry = contacts.get_or_construct(hex); + + entry.set_name(name.value_or("")); + entry.set_nickname(nickname.value_or("")); + if (pic_url && pic_key) + entry.profile_picture = {*pic_url, *pic_key}; + else + entry.profile_picture.clear(); + entry.profile_updated = as_sys_seconds(profile_updated); + entry.profile_flags = static_cast(pro_flags); + entry.approved = approved != 0; + entry.approved_me = approved_me != 0; + entry.blocked = blocked != 0; + + if (convo) { + auto [priority, notifications, mute_until, exp_mode, exp_timer, created] = *convo; + entry.priority = priority; + entry.notifications = static_cast(notifications); + entry.mute_until = mute_until; + entry.exp_mode = static_cast(exp_mode); + entry.exp_timer = std::chrono::seconds{exp_timer}; + // The earliest anyone knows about wins, so that two devices disagreeing about when a + // conversation began settle on the earlier rather than alternating. + entry.created = entry.created > 0 ? std::min(entry.created, created) : created; + } + + contacts.set(entry); +} + +void Client::_reveal_note_to_self(const ConversationId& id) { + if (id.type() != ConversationId::Type::dm || !is_me(id.session_id())) + return; + + auto& profile = core.configs.user_profile(); + if (profile.get_nts_priority() >= 0) + return; + + profile.set_nts_priority(0); + + // Then apply the config as a whole rather than just the priority. Until now there was no + // conversation for anything else it holds to attach to -- a disappearing timer set on another + // device has been waiting with nowhere to go -- and reconciling is what puts all of it in place + // at once. It cannot arrive by the usual route, since a local change is not a merge and so + // reports nothing back to us. + _reconcile_user_profile(); +} + +void Client::_reconcile_user_profile() { + auto& profile = core.configs.user_profile(); + auto me = ConversationId::dm(core.globals.session_id()); + + auto name = profile.get_name(); + auto pic = profile.get_profile_pic(); + auto priority = profile.get_nts_priority(); + auto expiry = profile.get_nts_expiry(); + + // Note-to-self carries a duration but no mode, because there is only one that means anything + // when the reader is also the writer: there is no moment at which someone else reads it. + int exp_mode = expiry && expiry->count() > 0 + ? static_cast(config::expiration_mode::after_send) + : static_cast(config::expiration_mode::none); + int64_t exp_timer = expiry ? expiry->count() : 0; + + bool profile_changed = false, convo_changed = false, order_changed = false, created = false, + history_changed = false; + + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + std::optional pic_url; + std::optional> pic_key; + if (!pic.empty()) { + pic_url = pic.url; + pic_key = std::span{pic.key}; + } + + profile_changed = _update_profile( + c, + identity_id(c, me), + name ? std::optional{*name} : std::nullopt, + pic_url, + pic_key, + epoch_seconds(profile.get_profile_updated()), + ProfileSource::config); + + // The config is what says a conversation exists -- a conversation is not defined by having + // messages in it, or emptying one would lose it, and reimporting an account would lose + // every conversation that happened to be empty. For note to self that statement is the + // priority itself: negative means there is no such conversation, so no row is made for one. + // + // Guarded by find_conversation rather than calling ensure_conversation outright, because + // that helper bumps last_activity on a row that already exists: unguarded, every config + // merge would shove note to self back to the top of the list. + auto convo = find_conversation(c, me); + if (!convo && priority >= 0) { + auto row = ensure_conversation(c, me, clock_now_ms()); + convo = row.id; + created = row.created; + } + + if (convo) { + order_changed = c.prepared_exec( + "UPDATE conversations SET priority = ?2" + " WHERE id = ?1 AND priority IS NOT ?2", + *convo, + priority) > 0; + convo_changed = c.prepared_exec( + R"( +UPDATE conversations SET exp_mode = ?2, exp_timer = ?3 +WHERE id = ?1 AND (exp_mode, exp_timer) IS NOT (?2, ?3) +)", + *convo, + exp_mode, + exp_timer) > 0; + + // Retroactive, and idempotent, for the reasons given in _reconcile_contacts. + auto delete_before = profile.get_nts_delete_before(); + auto delete_attach_before = profile.get_nts_delete_attach_before(); + if (delete_before > std::chrono::sys_seconds{}) + history_changed = delete_messages_before(c, *convo, sys_ms{delete_before}); + if (delete_attach_before > std::chrono::sys_seconds{}) + history_changed |= + delete_attachments_before(c, *convo, sys_ms{delete_attach_before}); + } + + tx.commit(); + } + + if (created) + _emit_conversation_added(me); + else if (profile_changed || convo_changed || history_changed) + _touch(me); + if (history_changed) + _emit_history_replaced(me); + if (order_changed) + _emit_lists_replaced(); +} + +// -- Messages --------------------------------------------------------------------------------- + +// Resolves what a wire reference names -- the target's author and send time, plus the sender's +// msgId for it where they set one -- to a local message id, correlated against `m`. +// +// Written once because reactions address their target identically (see Reaction.msgTimestamp in +// SessionProtos.proto, which mirrors Quote exactly); two copies of this would eventually disagree +// about the ambiguous case. Unqualified apart from `m`, following UNREAD's precedent, so it reads +// the same wherever it is substituted. +// +// `min(q.id)` does double duty. It applies the ambiguity rule: with no msgid, every message that +// sender stamped in that millisecond matches, and the lowest id is arbitrary but stable across +// reads -- which matters more than being right in a case the wire cannot disambiguate. And it +// collapses a multi-match to one row rather than multiplying the outer query. +// +// Guarded by the NULL check so a message that is not a reply -- almost all of them -- costs +// nothing. +static constexpr auto WIRE_REF_TARGET = R"( + CASE WHEN m.reply_timestamp IS NULL THEN NULL ELSE ( + SELECT min(q.id) FROM messages q + WHERE q.conversation = m.conversation AND q.sender = m.reply_author + AND q.timestamp = m.reply_timestamp + AND (m.reply_msgid IS NULL OR q.msgid = m.reply_msgid) + ) END +)"sv; + +// No join back to conversations: every row of a given query belongs to one conversation, so its +// ConversationId is resolved once by the caller rather than rebuilt per row. +// +// `ra` is the replied-to message's author, joined for its session id: the column holds an accounts +// row id, and what a caller needs is who that is. +static const std::string MESSAGE_COLUMNS = R"( + SELECT m.id, m.swarm_hash, a.session_id, m.outgoing, m.timestamp, m.body, m.send_state, + m.sync_send_state, m.deleted, m.gallery, + ra.session_id, m.reply_timestamp, {} + FROM messages m + JOIN accounts a ON a.id = m.sender + LEFT JOIN accounts ra ON ra.id = m.reply_author +)"_format(WIRE_REF_TARGET); + +// Whether a message is one we are willing to show as a gallery. Ours to tighten -- to particular +// formats, or a size ceiling -- which is why it is computed here rather than stored: narrowing it +// then applies to messages that arrived under the old rule, instead of leaving their stored answer +// behind to be honoured forever. +static bool gallery_viewable(const std::vector& attachments) { + return !attachments.empty() && std::ranges::all_of(attachments, [](const Attachment& a) { + return a.content_type && a.content_type->starts_with("image/"); + }); +} + +// Fills in the attachments of every message in `msgs`. +// +// One query for the whole page rather than one per message: a conversation view re-reads its page +// on every render, so the per-message alternative pays its cost there. The placeholder list makes +// this a distinct query string per page size, and so one prepared-statement cache entry per limit +// an application actually asks for -- few, since a page size is normally fixed. +static void load_attachments(sqlite::Connection& c, std::vector& msgs) { + if (msgs.empty()) + return; + + std::unordered_map by_id; + for (auto& m : msgs) + by_id.emplace(m.id, &m); + + auto st = c.prepared_st( + R"( + SELECT message, idx, content_type, filename, caption, flags, width, height, + size, url IS NOT NULL, saved_at + FROM message_attachments WHERE message IN ({}) ORDER BY message, idx + )"_format(sqlite::placeholders(msgs.size()))); + + int n = 1; + for (const auto& m : msgs) + st->bind(n++, m.id); + + for (auto&& [message, + idx, + ctype, + fname, + caption, + flags, + width, + height, + size, + uploaded, + saved_at] : + sqlite::IterableStatementWrapper< + int64_t, + int64_t, + std::optional, + std::optional, + std::optional, + int, + std::optional, + std::optional, + std::optional, + int, + std::optional>{std::move(st)}) { + auto found = by_id.find(message); + if (found == by_id.end()) + continue; + + found->second->attachments.push_back(Attachment{ + .index = static_cast(idx), + .content_type = std::move(ctype), + .filename = std::move(fname), + .caption = std::move(caption), + .voice_message = (flags & ATTACHMENT_FLAG_VOICE_MESSAGE) != 0, + .width = width ? std::optional{static_cast(*width)} : std::nullopt, + .height = height ? std::optional{static_cast(*height)} : std::nullopt, + .size = size, + .uploaded = uploaded != 0, + .saved_at = saved_at ? std::optional{from_epoch_ms(*saved_at)} : std::nullopt}); + } +} + +// How deep a read goes when a message turns out to be a reply. +enum class ReplyDepth { + // Load the replied-to message, so a caller can draw the reply from one read. + with_target, + // Resolve the reference but leave `Reply::message` null. This is what a *nested* message gets, + // and is the whole of the depth limit: without it, reading one message could walk a chain of + // replies of unbounded length. + reference_only, +}; + +// How many distinct reply targets a page may have before its lookup stops being worth caching a +// compiled statement for; see `load_reply_targets`. +static constexpr size_t REPLY_TARGET_CACHE_MAX = 4; + +static void load_reply_targets( + sqlite::Connection& c, const ConversationId& convo, std::vector& msgs); + +// Turns a bound statement over MESSAGE_COLUMNS into whole Messages: attachments loaded, gallery +// decided, and replied-to messages filled in unless this is already a nested read. +// +// Takes a bare statement rather than a `StatementWrapper` so that it serves both a cached statement +// and a one-off; see `load_reply_targets` for why one of its callers cannot use the cache. +static std::vector build_messages( + sqlite::Connection& c, + const ConversationId& convo, + ReplyDepth depth, + SQLite::Statement& st) { + std::vector out; + while (st.executeStep()) { + auto [id, + swarm_hash, + sender, + outgoing, + ts, + body, + send_state, + sync_send_state, + deleted, + gallery, + reply_author, + reply_ts, + reply_id] = + sqlite::get< + int64_t, + std::optional, + sqlite::blob_guts, + int, + int64_t, + std::string, + std::optional, + std::optional, + std::optional, + int, + std::optional>, + std::optional, + std::optional>(st); + out.push_back(Message{ + .id = id, + .conversation = convo, + .sender = sender, + .outgoing = outgoing != 0, + .timestamp = from_epoch_ms(ts), + .body = std::move(body), + .send_state = send_state ? std::optional{static_cast(*send_state)} + : std::nullopt, + .sync_send_state = sync_send_state + ? std::optional{static_cast(*sync_send_state)} + : std::nullopt, + .hash = std::move(swarm_hash), + .gallery = gallery != 0, + // The author and timestamp are what the wire carried, so they are known + // whether or not the reference resolved; `message_id` is the resolution. + // The message itself is filled in afterwards, in one query for the page. + .reply = reply_author && reply_ts ? std::optional{Reply{ + .author = *reply_author, + .timestamp = from_epoch_ms(*reply_ts), + .message_id = reply_id}} + : std::nullopt, + .deleted = + deleted ? std::optional{static_cast(*deleted)} : std::nullopt}); + } + + // Done here rather than by each caller so that every path that produces Messages produces whole + // ones: a Message with its attachments silently missing is worse than no accessor at all. + load_attachments(c, out); + + // Only now can the question be answered, since it is about the attachments. A stored decision + // that the current rule no longer supports is dropped rather than honoured -- and dropped in + // the database too, so that the next read does not have to reach the same conclusion again, and + // so that a client toggling it sees the same state we just reported. + for (auto& m : out) { + m.gallery_viewable = gallery_viewable(m.attachments); + if (m.gallery && !m.gallery_viewable) { + m.gallery = false; + c.prepared_exec("UPDATE messages SET gallery = 0 WHERE id = ?", m.id); + } + } + + if (depth == ReplyDepth::with_target) + load_reply_targets(c, convo, out); + + return out; +} + +template +static std::vector query_messages( + sqlite::Connection& c, + const ConversationId& convo, + ReplyDepth depth, + const std::string& query, + const Bind&... bind) { + auto st = c.prepared_st(query); + bind_oneshot(st, bind...); + return build_messages(c, convo, depth, *st); +} + +static void load_reply_targets( + sqlite::Connection& c, const ConversationId& convo, std::vector& msgs) { + // Deduplicated: a conversation where several people answer the same message should read it + // once, and then share the one copy rather than each holding its own. + std::set wanted; + for (const auto& m : msgs) + if (m.reply && m.reply->message_id) + wanted.insert(*m.reply->message_id); + if (wanted.empty()) + return; + + auto query = + "{} WHERE m.id IN ({})"_format(MESSAGE_COLUMNS, sqlite::placeholders(wanted.size())); + + // Cached only up to a few targets. A prepared statement is kept forever against its query + // text, and this text varies with how many distinct messages the page replies to -- so caching + // every shape would keep a compiled statement per count ever seen, most of them to save + // recompiling a query unlikely to recur with exactly that many placeholders. + // + // The small counts are worth it though, because they are the common case: a page of history + // usually answers a handful of distinct messages at most, so these few shapes recur constantly + // while the long ones are one-offs. + std::optional cached; + std::optional one_off; + if (wanted.size() <= REPLY_TARGET_CACHE_MAX) + cached.emplace(c.prepared_st(query)); + else + one_off.emplace(c.sql, query); + SQLite::Statement& st = cached ? **cached : *one_off; + + int n = 1; + for (auto id : wanted) + st.bind(n++, id); + + // `reference_only`, which is the depth limit: these targets keep the reference to whatever + // *they* replied to, but not the message, so one read cannot walk a chain. + std::map> loaded; + for (auto& t : build_messages(c, convo, ReplyDepth::reference_only, st)) { + auto id = t.id; + loaded.emplace(id, std::make_shared(std::move(t))); + } + + for (auto& m : msgs) + if (m.reply && m.reply->message_id) + if (auto found = loaded.find(*m.reply->message_id); found != loaded.end()) + m.reply->message = found->second; +} + +std::vector Client::_messages( + const ConversationId& id, + int limit, + std::optional before, + bool include_deleted) { + auto c = core.database().conn(); + auto convo = find_conversation(c, id); + if (!convo) + return {}; + + // In the query rather than dropped from the result, so that `limit` counts rows the caller will + // actually see: filtering afterwards would return short pages with nothing to say that another + // page is warranted. + auto visible = include_deleted ? ""sv : "AND m.deleted IS NULL"sv; + + if (!before) + return query_messages( + c, + id, + ReplyDepth::with_target, + R"( + {} WHERE m.conversation = ? {} ORDER BY m.timestamp DESC, m.id DESC LIMIT ? + )"_format(MESSAGE_COLUMNS, visible), + *convo, + limit); + + // Strictly-older-than comparison on (timestamp, id), spelled out rather than as an SQL row + // value so this does not depend on the SQLite version's row-value support. + return query_messages( + c, + id, + ReplyDepth::with_target, + R"( + {} WHERE m.conversation = ?1 {} + AND (m.timestamp < ?2 OR (m.timestamp = ?2 AND m.id < ?3)) + ORDER BY m.timestamp DESC, m.id DESC LIMIT ?4 + )"_format(MESSAGE_COLUMNS, visible), + *convo, + epoch_ms(before->timestamp), + before->id, + limit); +} + +std::optional Client::_message(int64_t id) { + auto c = core.database().conn(); + auto convo = + c.prepared_maybe_get("SELECT conversation FROM messages WHERE id = ?", id); + if (!convo) + return std::nullopt; + + auto found = query_messages( + c, + conversation_id_at(c, *convo), + ReplyDepth::with_target, + "{} WHERE m.id = ?"_format(MESSAGE_COLUMNS), + id); + if (found.empty()) + return std::nullopt; + return std::move(found.front()); +} + +std::optional Client::_message_debug(int64_t message_id) { + auto c = core.database().conn(); + auto raw = c.prepared_maybe_get( + "SELECT content FROM message_raw_content WHERE message = ?", message_id); + if (!raw) + return std::nullopt; + + SessionProtos::Content content; + if (!content.ParseFromString(*raw)) { + // Stored after it parsed, or serialized by us, so this is the bytes having rotted rather + // than a message we never understood. Worth a warning; still nothing to show. + log::warning(cat, "Stored wire content for message {} did not parse", message_id); + return std::nullopt; + } + return proto::debug_print(content); +} + +namespace { + + // The reference a quote puts on the wire, read from the message being replied to. + struct WireRef { + b33 author; + sys_ms timestamp; + std::optional msgid; + }; + + // Nullopt if the message is gone by the time the send happens: a send is not worth failing over + // a reference that can simply be omitted. + std::optional wire_ref(sqlite::Connection& c, int64_t message_id) { + auto row = c.prepared_maybe_get, int64_t, std::optional>( + R"( + SELECT a.session_id, m.timestamp, m.msgid + FROM messages m JOIN accounts a ON a.id = m.sender + WHERE m.id = ? + )", + message_id); + if (!row) + return std::nullopt; + + auto& [author, ts, msgid] = *row; + return WireRef{.author = author, .timestamp = from_epoch_ms(ts), .msgid = msgid}; + } + + // Writes a quote naming `ref` onto an outgoing DataMessage. + // + // Only the reference goes on. `text` and the quoted attachments are left unset, because + // current clients do not populate them either, and one that arrives is not to be trusted: a + // sender can put whatever words they like in someone else's mouth that way. Receivers render + // from their own copy of the message, or from nothing. + void set_quote(SessionProtos::DataMessage& data, const WireRef& ref) { + auto* q = data.mutable_quote(); + q->set_msgtimestamp(static_cast(epoch_ms(ref.timestamp))); + q->set_author(oxenc::to_hex(ref.author.begin(), ref.author.end())); + if (ref.msgid) + q->set_msgid(*ref.msgid); + } + +} // namespace + +int64_t Client::_send_message(const ConversationId& id, const OutgoingMessage& msg) { + auto body = msg.body; + if (id.type() != ConversationId::Type::dm) + throw std::invalid_argument{ + "send_message: only DM conversations are supported so far (got type {})"_format( + static_cast(id.type()))}; + + auto now = clock_now_ms(); + auto self = core.globals.session_id(); + + // Generated before the copies below, so both carry it: that is what makes it the identifier + // every party agrees on, unlike a hash of copies that differ. + auto msgid = new_msgid(); + + // Read before the content is built, because the quote goes inside it. `_require_sendable` has + // already established that this message exists and belongs to this conversation. + std::optional reply; + if (msg.reply_to) { + auto c = core.database().conn(); + reply = wire_ref(c, *msg.reply_to); + } + + SessionProtos::Content content; + auto* data = fill_outgoing_content(content, now, msgid, body); + if (reply) + set_quote(*data, *reply); + + // Two artifacts: the copy the recipient gets, and the copy we deposit in our own swarm so that + // our other devices see it. They differ only in syncTarget, which is what tells those devices + // which conversation an outgoing message belongs to -- the sender being us either way. + // + // What we store locally is the *sync* copy, so that our row is identical whether we sent the + // message or a linked device did. Anything re-sent out of the database therefore has to clear + // syncTarget again before it goes to a recipient. + SessionProtos::Content synced = content; + synced.mutable_datamessage()->set_synctarget(oxenc::to_hex(id.session_id())); + + auto serialised = synced.SerializeAsString(); + auto raw = std::span{reinterpret_cast(serialised.data()), serialised.size()}; + + // Note to self is one swarm, not two: sending both copies there would deposit the same message + // twice, and both would come back. So there is no separate sync send to have a state for. + bool to_self = is_me(id.session_id()); + + bool created, approved; + int64_t client_id; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = ensure_conversation(c, id, now); + created = convo.created; + + c.prepared_exec( + R"( + INSERT INTO messages + (conversation, msgid, sender, outgoing, timestamp, body, send_state, + sync_send_state, reply_author, reply_timestamp, reply_msgid) + VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) + )", + convo.id, + msgid, + account_id(c, self), + epoch_ms(now), + body, + static_cast(SendState::pending), + to_self ? std::optional{} + : std::optional{static_cast(SendState::pending)}, + // Stored as the wire form, exactly as an incoming reply is, so that our own message + // resolves through the same rule as everyone else's rather than a second one. + reply ? std::optional{account_id(c, reply->author)} : std::nullopt, + reply ? std::optional{epoch_ms(reply->timestamp)} : std::nullopt, + reply ? reply->msgid : std::nullopt); + client_id = c.sql.getLastInsertRowid(); + log::debug( + cat, "send_message: stored message {} ({}B content)", client_id, serialised.size()); + + c.prepared_exec( + "INSERT INTO message_raw_content (message, content) VALUES (?, ?)", client_id, raw); + approved = approve_recipient(c, id, *this); + tx.commit(); + } + + if (approved) { + _sync_contact(id); + _emit_lists_replaced(); + } + if (created) + _emit_conversation_added(id); + _reveal_note_to_self(id); + _emit_message(true, id, client_id); + _touch(id); + + log::debug(cat, "send_message: message {} to conversation {}", client_id, id.to_string()); + + _dispatch_sends(client_id, id, content, synced, now, to_self); + + return client_id; +} + +void Client::_dispatch_sends( + int64_t client_id, + const ConversationId& id, + const SessionProtos::Content& content, + const SessionProtos::Content& synced, + sys_ms now, + bool to_self) { + auto self = core.globals.session_id(); + + auto core_id = + to_self ? core.send_dm(self, synced, now) : core.send_dm(id.session_id(), content, now); + _send_ids[core_id] = OutgoingSend{client_id, to_self}; + + // send_dm() reports its first status synchronously, i.e. before we knew the id to map it to. + if (auto stashed = _early_status.extract(core_id)) { + auto& [status, hash] = stashed.mapped(); + _apply_send_status(client_id, status, false, to_self ? opt_view(hash) : std::nullopt); + } + + if (!to_self) { + auto sync_id = core.send_dm(self, synced, now); + _sync_sends[sync_id] = client_id; + if (auto stashed = _early_status.extract(sync_id)) { + auto& [status, hash] = stashed.mapped(); + _apply_send_status(client_id, status, true, opt_view(hash)); + } + } +} + +// -- Attachments ------------------------------------------------------------------------------ + +// The content type to advertise for a file whose sender did not name one. +// +// Deliberately shallow: this is a display hint a recipient uses to pick a viewer, not something +// any decision depends on, and a platform that has a real UTI database (as every GUI client does) +// should pass `OutgoingAttachment::content_type` rather than rely on this. Everything unrecognised +// is application/octet-stream, which is what Session's other clients also fall back to. +static std::string infer_content_type(const std::filesystem::path& path) { + static const std::unordered_map types{ + {"jpg", "image/jpeg"}, + {"jpeg", "image/jpeg"}, + {"png", "image/png"}, + {"gif", "image/gif"}, + {"webp", "image/webp"}, + {"heic", "image/heic"}, + {"bmp", "image/bmp"}, + {"tiff", "image/tiff"}, + {"svg", "image/svg+xml"}, + {"mp4", "video/mp4"}, + {"mov", "video/quicktime"}, + {"webm", "video/webm"}, + {"mkv", "video/x-matroska"}, + {"avi", "video/x-msvideo"}, + {"mp3", "audio/mpeg"}, + {"m4a", "audio/mp4"}, + {"aac", "audio/aac"}, + {"ogg", "audio/ogg"}, + {"opus", "audio/opus"}, + {"flac", "audio/flac"}, + {"wav", "audio/wav"}, + {"pdf", "application/pdf"}, + {"txt", "text/plain"}, + {"md", "text/markdown"}, + {"csv", "text/csv"}, + {"html", "text/html"}, + {"json", "application/json"}, + {"xml", "application/xml"}, + {"zip", "application/zip"}, + {"gz", "application/gzip"}, + {"bz2", "application/x-bzip2"}, + {"xz", "application/x-xz"}, + {"7z", "application/x-7z-compressed"}, + {"tar", "application/x-tar"}, + }; + + auto ext = path.extension().string(); + if (ext.starts_with('.')) + ext.erase(0, 1); + for (auto& ch : ext) + ch = static_cast(std::tolower(static_cast(ch))); + + if (auto found = types.find(ext); found != types.end()) + return std::string{found->second}; + return "application/octet-stream"; +} + +int64_t Client::_send_message( + const ConversationId& id, + const OutgoingMessage& msg, + std::function)> on_upload) { + const auto& attachments = msg.attachments; + if (attachments.empty()) + return _send_message(id, msg); + + auto body = msg.body; + auto now = clock_now_ms(); + auto self = core.globals.session_id(); + bool to_self = is_me(id.session_id()); + + // Stored with the body alone for now. What finally goes to the swarms also names the uploaded + // files, which nothing knows yet -- that is what the uploads are for -- so the content is + // rewritten in _finish_attachment_send once they do. The identifier is not: it belongs to the + // message rather than to any particular rendering of it, so it is generated here and reused. + auto msgid = new_msgid(); + + std::optional reply; + if (msg.reply_to) { + auto c = core.database().conn(); + reply = wire_ref(c, *msg.reply_to); + } + + SessionProtos::Content content; + fill_outgoing_content(content, now, msgid, body); + + auto serialised = content.SerializeAsString(); + auto raw = std::span{reinterpret_cast(serialised.data()), serialised.size()}; + + bool created, approved; + int64_t client_id; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + auto convo = ensure_conversation(c, id, now); + created = convo.created; + + c.prepared_exec( + R"( + INSERT INTO messages + (conversation, msgid, sender, outgoing, timestamp, body, send_state, + sync_send_state, reply_author, reply_timestamp, reply_msgid) + VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) + )", + convo.id, + msgid, + account_id(c, self), + epoch_ms(now), + body, + static_cast(SendState::uploading), + to_self ? std::optional{} + : std::optional{static_cast(SendState::uploading)}, + // The reference is stored now even though the content that goes out is rewritten + // later: `_finish_attachment_send` rebuilds the quote from these columns, so what + // is sent and what we show come from the same place. + reply ? std::optional{account_id(c, reply->author)} : std::nullopt, + reply ? std::optional{epoch_ms(reply->timestamp)} : std::nullopt, + reply ? reply->msgid : std::nullopt); + client_id = c.sql.getLastInsertRowid(); + + c.prepared_exec( + "INSERT INTO message_raw_content (message, content) VALUES (?, ?)", client_id, raw); + + for (size_t i = 0; i < attachments.size(); i++) { + const auto& a = attachments[i]; + c.prepared_exec( + R"( + INSERT INTO message_attachments + (message, idx, path, content_type, filename, caption, flags, width, height) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + )", + client_id, + static_cast(i), + a.path.string(), + a.content_type ? *a.content_type : infer_content_type(a.path), + a.filename ? a.filename : std::optional{a.path.filename().string()}, + a.caption, + a.voice_message ? ATTACHMENT_FLAG_VOICE_MESSAGE : 0, + a.width ? std::optional{*a.width} : std::nullopt, + a.height ? std::optional{*a.height} : std::nullopt); + } + + approved = approve_recipient(c, id, *this); + tx.commit(); + } + + if (approved) { + _sync_contact(id); + _emit_lists_replaced(); + } + if (created) + _emit_conversation_added(id); + _reveal_note_to_self(id); + _emit_message(true, id, client_id); + _touch(id); + + log::debug( + cat, + "send_message: message {} to conversation {} with {} attachment(s)", + client_id, + id.to_string(), + attachments.size()); + + _upload_next(client_id, std::move(on_upload)); + return client_id; +} + +void Client::_upload_next( + int64_t client_id, + std::function)> on_upload) { + std::optional> next; + { + auto c = core.database().conn(); + next = c.prepared_maybe_get( + R"( + SELECT idx, path FROM message_attachments + WHERE message = ? AND url IS NULL + ORDER BY idx LIMIT 1 + )", + client_id); + } + + // Nothing left without a url means every file is up, and the message can finally be built. + if (!next) + return _finish_attachment_send(client_id); + + auto [idx, path] = *next; + auto index = static_cast(idx); + + // Checked here rather than only at send_message, because that check is on the other side of + // however long the message sat waiting: a file present when it was attached can be gone by the + // time its turn comes, and a message resumed in a later run may have been waiting for days. + std::error_code ec; + auto plaintext_size = static_cast(std::filesystem::file_size(path, ec)); + if (ec || !std::filesystem::is_regular_file(path, ec) || ec) { + log::warning( + cat, + "Attachment {} of message {} is gone ({}); the message cannot be sent", + idx, + client_id, + path); + if (on_upload) + on_upload(index, 0, 0, ATTACHMENT_FILE_MISSING); + return _fail_attachment_send(client_id, /*permanent=*/true); + } + + auto net = core.network(); + if (!net) { + log::warning( + cat, + "Cannot upload attachment {} of message {}: no network is attached", + idx, + client_id); + if (on_upload) + on_upload(index, 0, 0, network::ERROR_NO_TRANSPORT_LAYER); + return _fail_attachment_send(client_id); + } + + network::FileUploadRequest req; + req.file = path; + req.domain = attachment::Domain::ATTACHMENT; + // The limit this guards is the one onion requests impose, and an attachment goes to the file + // server rather than through them; the server enforces its own. + req.allow_large = true; + req.request_timeout = ATTACHMENT_REQUEST_TIMEOUT; + req.overall_timeout = ATTACHMENT_OVERALL_TIMEOUT; + + if (on_upload) { + // This attachment's own throttle, so that uploads running alongside each other cannot + // squelch one another. Only progress passes through it: starting and finishing are things + // a caller must always hear, and they are reported from elsewhere. + auto throttle = std::make_shared(_high_freq_dispatch_interval); + req.on_progress = [index, on_upload, throttle](int64_t sent, int64_t total) { + if (throttle->allow()) + on_upload(index, sent, total, std::nullopt); + }; + } + + req.on_complete = [this, client_id, index, on_upload, plaintext_size]( + std::variant, int16_t> + result, + bool /*timeout*/) { + // Delivered on the network's loop; everything below touches the database, which is + // Core's loop's alone. Nobody is waiting on a callback here -- this is Client's + // own continuation -- so a failure has to be turned into the message failing, which + // is what the application is watching. + loop.call([this, client_id, index, on_upload, plaintext_size, result = std::move(result)] { + try { + if (auto* err = std::get_if(&result)) { + log::warning( + cat, + "Upload of attachment {} of message {} failed: {}", + index, + client_id, + *err); + if (on_upload) + on_upload(index, 0, 0, *err); + return _fail_attachment_send(client_id); + } + + const auto& [meta, key] = std::get<0>(result); + // upload_file always encrypts with the stream scheme, so the url has to say + // so: without the fragment a recipient reaches for the legacy scheme and + // cannot open the file at all. + auto url = network::file_server::generate_download_url( + meta.id, + core.network()->file_server_config, + /*stream_encrypted=*/true); + + { + auto c = core.database().conn(); + // The file's own size, not the encrypted one the server reports back: + // `size` on the pointer means the plaintext length everywhere else in + // Session, and for a legacy-encrypted attachment it is what a recipient + // trims the padding by, so an encrypted size there would be wrong in a + // way that breaks decryption rather than merely misreporting. + c.prepared_exec( + "UPDATE message_attachments SET url = ?, key = ?, size = ?" + " WHERE message = ? AND idx = ?", + url, + std::span{key}, + plaintext_size, + client_id, + static_cast(index)); + } + + log::debug( + cat, + "Uploaded attachment {} of message {} ({} bytes) to {}", + index, + client_id, + meta.size, + url); + + if (on_upload) + on_upload(index, meta.size, meta.size, 0); + + _upload_next(client_id, on_upload); + } catch (const std::exception& e) { + log::error( + cat, + "Recording attachment {} of message {} failed: {}", + index, + client_id, + e.what()); + _fail_attachment_send(client_id); + } + }); + }; + + log::debug(cat, "Uploading attachment {} of message {}: {}", idx, client_id, path); + // Bound to a name: the accessor's span is deliberately unavailable on a temporary. + auto seed_access = core.globals.account_seed(); + net->upload_file(std::move(req), seed_access.seed()); +} + +bool Client::_retry_send( + int64_t client_id, + std::function)> on_upload) { + std::optional convo_id; + bool has_uploads_left; + { + auto c = core.database().conn(); + + auto row = c.prepared_maybe_get>( + "SELECT conversation, outgoing, send_state FROM messages WHERE id = ?", client_id); + if (!row) { + log::warning(cat, "Cannot retry message {}: it does not exist", client_id); + return false; + } + auto [convo_row, outgoing, state] = *row; + + // Only a send of ours can be retried, and only one that is over: retrying something still + // in flight would double it up on the swarm rather than rescue it. + if (!outgoing || !state) { + log::warning(cat, "Cannot retry message {}: it is not an outgoing send", client_id); + return false; + } + auto send_state = static_cast(*state); + if (send_state != SendState::failed && send_state != SendState::interrupted) { + log::warning( + cat, + "Cannot retry message {}: it is in state {}, which is not a failure to retry", + client_id, + *state); + return false; + } + + convo_id = conversation_id_at(c, convo_row); + + has_uploads_left = c.prepared_get( + "SELECT COUNT(*) FROM message_attachments WHERE message = ? AND " + "url IS NULL", + client_id) > 0; + + // Only worth saying while there is uploading left to do; otherwise the message goes + // straight back to a send, and _finish_attachment_send sets that state itself. + if (has_uploads_left) + c.prepared_exec( + "UPDATE messages SET send_state = ?, sync_send_state =" + " CASE WHEN sync_send_state IS NULL THEN NULL ELSE ? END WHERE id = ?", + static_cast(SendState::uploading), + static_cast(SendState::uploading), + client_id); + } + + if (has_uploads_left) + _emit_message(false, *convo_id, client_id); + + log::debug( + cat, + "Retrying message {}{}", + client_id, + has_uploads_left ? " (attachments still to upload)" : ""); + + // Resumes wherever it was left: _upload_next takes the first attachment with no url, so the + // ones that already got up are not sent again, and a message needing none goes straight to + // being dispatched. + _upload_next(client_id, std::move(on_upload)); + return true; +} + +// Opens the file a download is written to until it is known to be whole, and returns the name it +// claimed. A save that dies partway leaves this rather than something that looks like the file the +// user asked for. +// +// `.sessiondl` rather than the conventional `.part` because opening it destroys whatever is already +// there: somebody's own `report.pdf.part`, from a browser or another downloader, is a plausible +// file to find next to `report.pdf`, and one named after us is not. Numbered when even that is +// taken, which is what lets two saves of the same file into one directory proceed at once. +// +// Claiming the name *is* the open, given `std::ios::noreplace` -- C++23, so not here yet, since +// this builds at C++20 and the macro is guarded on the language version rather than on the library. +// With it the open fails when the file exists and two saves cannot pick the same name however they +// are timed. Without it the check and the open are two steps, so they still can; that is a much +// narrower window than the fixed name it replaces, which collided every time, and the loser of the +// race gets a failed save rather than anyone losing a file of their own. Building at C++23 closes +// it with no other change. +static std::filesystem::path open_partial(std::ofstream& out, const std::filesystem::path& dest) { +#ifdef __cpp_lib_ios_noreplace + constexpr auto mode = std::ios::binary | std::ios::noreplace; +#else + constexpr auto mode = std::ios::binary | std::ios::trunc; +#endif + + for (int n = 0; n < 1000; n++) { + auto p = dest; + p += n == 0 ? ".sessiondl"s : ".sessiondl{}"_format(n); + +#ifndef __cpp_lib_ios_noreplace + std::error_code ec; + if (std::filesystem::exists(p, ec)) + continue; +#endif + + out.open(p, mode); + if (out) + return p; + out.clear(); + } + throw std::runtime_error{"Cannot write a scratch file beside {}"_format(dest.string())}; +} + +// `name (2).pdf`, not `name.pdf (2)`: only the first still opens on a double-click, and it is what +// browsers and both desktop file managers produce. +static std::filesystem::path numbered(const std::filesystem::path& dest, int n) { + auto out = dest; + out.replace_filename("{} ({}){}"_format(dest.stem().string(), n, dest.extension().string())); + return out; +} + +// The name to give the finished file. `dest` unless something has appeared there since the caller +// last looked, which is the whole point: whether it was free was decided when the *prompt* was +// answered, and the rename happens when the download finishes, which may be minutes later. +// +// A caller that already had its user approve a replacement passes `replace` and gets `dest` +// regardless. Renaming in that case would be worse than the bug it avoids: the approved overwrite +// would land beside the file it was meant to replace, and the answer the user gave would be +// silently discarded. +static std::filesystem::path final_path(const std::filesystem::path& dest, bool replace) { + if (replace) + return dest; + + std::error_code ec; + if (!std::filesystem::exists(dest, ec)) + return dest; + for (int n = 1; n < 1000; n++) { + auto p = numbered(dest, n); + if (!std::filesystem::exists(p, ec)) + return p; + } + throw std::runtime_error{"Cannot find a free name beside {}"_format(dest.string())}; +} + +std::function)> Client::_dispatch_progress( + std::function)> cb) { + if (!cb) + return {}; + return [this, cb = std::move(cb)](int64_t done, int64_t total, std::optional r) { + // Copied rather than moved into the hop: this fires once per chunk, and moving would leave + // the next report with nothing to reach. + _dispatch_out([cb, done, total, r] { cb(done, total, r); }); + }; +} + +void Client::_download_decrypted( + const std::string& url, + DownloadKind kind, + std::vector key, + std::vector digest, + std::optional claimed_size, + std::function)> on_plain, + std::function)> on_progress, + std::function)> on_done) { + + auto info = network::file_server::parse_download_url(url); + if (!info) + throw std::runtime_error{"{} is not a download url"_format(url)}; + + auto net = core.network(); + if (!net) + throw std::runtime_error{"Cannot download: no network is attached"}; + + // The whole of the format question, answered once. See the header for why `kind` is passed in + // rather than guessed from how long the key happens to be. + enum class Scheme { plaintext, stream, legacy_attachment, legacy_display_pic } scheme; + if (key.empty()) + scheme = Scheme::plaintext; + else if (info->wants_stream_decryption) + scheme = Scheme::stream; + else if (kind == DownloadKind::attachment) + scheme = Scheme::legacy_attachment; + else + scheme = Scheme::legacy_display_pic; + + auto need_key = [&](size_t want) { + if (key.size() != want) + throw std::runtime_error{ + "Cannot decrypt {}: this download needs a {}-byte key and we have {}"_format( + url, want, key.size())}; + }; + switch (scheme) { + case Scheme::plaintext: break; + case Scheme::stream: need_key(attachment::ENCRYPT_KEY_SIZE); break; + case Scheme::legacy_display_pic: need_key(attachment::LEGACY_DISPLAY_PIC_KEY_SIZE); break; + case Scheme::legacy_attachment: + need_key(attachment::LEGACY_KEY_SIZE); + if (digest.size() != attachment::LEGACY_DIGEST_SIZE) + throw std::runtime_error{ + "Cannot authenticate {}: a legacy attachment needs a {}-byte digest and we " + "have {}"_format(url, attachment::LEGACY_DIGEST_SIZE, digest.size())}; + break; + } + bool stream = scheme == Scheme::stream; + + // Shared with the network's thread, where every callback below runs. The stream case hands + // plaintext over as it decrypts and so never holds the whole thing; the legacy case has to + // accumulate, because its MAC and digest cover the whole ciphertext and neither can be checked + // until all of it is here. + struct DownloadState { + std::vector buffered; + std::optional decryptor; + std::optional failure; + int64_t received = 0; + int64_t delivered = 0; + }; + auto state = std::make_shared(); + + network::DownloadRequest req; + auto cancel = req.cancelled; + + if (stream) { + std::array k; + std::ranges::copy(key, k.begin()); + // The stream format pads at the front, with a marker decryption already verifies, so what + // comes out here is the file and nothing else -- its length is a fact about the data rather + // than the sender's word for it. Counting as it emerges means a file longer than its + // sender said is known the moment it passes that length, rather than at the end. + // + // Too short cannot be known here; that is what the check at completion is for. + state->decryptor.emplace( + k, [state, on_plain, claimed_size, cancel](std::span plain) { + if (state->failure) + return; + state->delivered += static_cast(plain.size()); + if (claimed_size && state->delivered > *claimed_size) { + state->failure = "attachment is longer than the {}B its sender said"_format( + *claimed_size); + cancel->store(true); + return; + } + on_plain(plain); + }); + } + + auto throttle = std::make_shared(_high_freq_dispatch_interval); + + // As uploads do: the 0/0 that says this one has started, before the size is known. + if (on_progress) + on_progress(0, 0, std::nullopt); + + req.download_url = url; + req.request_timeout = ATTACHMENT_REQUEST_TIMEOUT; + req.overall_timeout = ATTACHMENT_OVERALL_TIMEOUT; + + req.on_data = [state, stream, on_progress, throttle, cancel]( + const network::file_metadata& meta, std::span data) { + if (state->failure) + return; + // Asks for the transfer to stop, and today only asks: the download path does not consult + // the flag -- only uploads do -- so the rest of the file arrives and is dropped by the + // guard above before we report the failure. Set anyway, because it is the right request to + // make and the plumbing is the part that is missing. + // + // Worth having once it works: the stream scheme authenticates each chunk as it arrives, so + // a failure surfaces when the bad chunk does, wherever in the file that is, and everything + // after it is bandwidth spent on a file already known to be unusable. + auto give_up = [&](std::string why) { + state->failure = std::move(why); + cancel->store(true); + }; + try { + state->received += static_cast(data.size()); + + // Enforced against bytes actually arriving rather than against anything the sender or + // the server claimed, so an over-long transfer is cut off rather than accumulated. + if (state->received > static_cast(attachment::LEGACY_MAX_ENCRYPTED_SIZE)) + return give_up("download is larger than the file server's maximum"); + + if (stream) { + if (!state->decryptor->update(data)) + return give_up("decryption failed"); + } else + state->buffered.insert(state->buffered.end(), data.begin(), data.end()); + + if (on_progress && throttle->allow()) + on_progress(state->received, meta.size, std::nullopt); + } catch (const std::exception& e) { + give_up(e.what()); + } + }; + + req.on_complete = [state, + scheme, + on_plain, + on_progress, + on_done, + key = std::move(key), + digest = std::move(digest), + claimed_size]( + std::variant result, bool timeout) { + auto fail = [&](std::string why, int code) { + if (on_progress) + on_progress(0, 0, code); + on_done(std::move(why)); + }; + + if (auto* err = std::get_if(&result)) + return fail( + timeout ? "download timed out"s : "download failed with status {}"_format(*err), + *err); + + if (state->failure) + return fail(std::move(*state->failure), ATTACHMENT_UNREADABLE); + + try { + switch (scheme) { + case Scheme::stream: + if (!state->decryptor->finalize()) + throw std::runtime_error{"download ended mid-stream"}; + // The claim is exact or it is wrong: the file server is told the precise byte + // count at upload and refuses anything else, so there is no reason to accept a + // file that turns out to be a different size from the one described. Nothing + // else checks this for a stream attachment -- the format strips its own padding + // and never consults the pointer -- so without it a sender can describe one + // file and deliver another. + if (claimed_size && state->delivered != *claimed_size) + throw std::runtime_error{"attachment is {}B but its sender said {}B"_format( + state->delivered, *claimed_size)}; + break; + + case Scheme::plaintext: + // Nothing to undo: community images are stored as they are. + on_plain(state->buffered); + break; + + case Scheme::legacy_attachment: { + std::array k; + std::array d; + std::ranges::copy(key, k.begin()); + std::ranges::copy(digest, d.begin()); + on_plain(attachment::legacy_decrypt( + state->buffered, + k, + d, + claimed_size ? static_cast(*claimed_size) : 0)); + break; + } + + case Scheme::legacy_display_pic: { + std::array k; + std::ranges::copy(key, k.begin()); + on_plain(attachment::legacy_display_pic_decrypt(state->buffered, k)); + break; + } + } + } catch (const std::exception& e) { + return fail(e.what(), ATTACHMENT_UNREADABLE); + } + + if (on_progress) + on_progress(state->received, state->received, 0); + on_done(std::nullopt); + }; + + net->download(std::move(req)); +} + +Client::StoredPointer Client::_attachment_pointer(int64_t message_id, size_t index) { + StoredPointer p; + { + auto c = core.database().conn(); + // `key` and `digest` vary in length -- 32 bytes for the stream scheme, 64 for legacy -- so + // they are read as blob views from a live statement and copied out before it steps. + auto st = c.prepared_bind( + "SELECT url, key, digest, size FROM message_attachments WHERE message = ? AND idx" + " = ?", + message_id, + static_cast(index)); + if (!st->executeStep()) + throw std::runtime_error{"Message {} has no attachment {}"_format(message_id, index)}; + + auto [u, k, d, sz] = sqlite::get< + std::optional, + std::optional, + std::optional, + std::optional>(*st); + if (!u) + throw std::runtime_error{ + "Attachment {} of message {} cannot be fetched: its sender gave no url"_format( + index, message_id)}; + p.url = std::move(*u); + if (k) + p.key.assign(k->begin(), k->end()); + if (d) + p.digest.assign(d->begin(), d->end()); + p.size = sz; + } + return p; +} + +void Client::_cache_attachment( + const std::string& url, + std::span key, + std::span data) { + auto file = cache::path_for(_cache_dir, cache::ATTACHMENT_DIR, url); + try { + cache::write(file, key, data); + } catch (const std::exception& e) { + // A cache that cannot be written is a cache that misses next time, which is not worth + // failing the caller's fetch over. + log::warning(cat, "Could not cache an attachment: {}", e.what()); + return; + } + + // Recorded after the file exists, so a row never describes something that is not there. The + // size is what the file actually takes, read back rather than computed: the cache limit is a + // limit on disk, and padding and framing are part of what it costs. + std::error_code ec; + auto on_disk = std::filesystem::file_size(file, ec); + if (ec) + return; + + auto name = file.filename().string(); + core.database().conn().prepared_exec( + R"( + INSERT INTO attachment_cache (name, size, last_used) VALUES (?1, ?2, ?3) + ON CONFLICT (name) DO UPDATE SET size = ?2, last_used = ?3 + )", + name, + static_cast(on_disk), + epoch_ms(clock_now_ms())); + + // Checked when something is added, which is the only moment the total can grow. + _evict_cache(name); +} + +void Client::_evict_cache(const std::string& keep) { + auto c = core.database().conn(); + + auto limit = core.globals.get_integer(CACHE_LIMIT_KEY); + if (!limit) + return; + + auto total = c.prepared_get>("SELECT sum(size) FROM attachment_cache") + .value_or(0); + if (total <= *limit) + return; + + // Oldest use first. `keep` is excluded rather than relying on it sorting last: it does, having + // just been used, but a cache limit smaller than a single file would otherwise delete the + // download that provoked the eviction, which reads as the fetch having silently failed. Better + // to sit over the limit by one file than to throw away what was just asked for. + for (auto [name, size] : c.prepared_results( + R"( + SELECT name, size FROM attachment_cache WHERE name IS NOT ?1 ORDER BY last_used + )", + keep)) { + if (total <= *limit) + break; + + // The row is an index, not the truth: a file that is already gone still costs a row, and + // removing it is as much progress as unlinking one. + std::error_code ec; + std::filesystem::remove(_cache_dir / cache::ATTACHMENT_DIR / name, ec); + c.prepared_exec("DELETE FROM attachment_cache WHERE name = ?", name); + total -= size; + } +} + +void Client::_touch_cached(const std::string& name) { + core.database().conn().prepared_exec( + "UPDATE attachment_cache SET last_used = ?2 WHERE name = ?1", + name, + epoch_ms(clock_now_ms())); +} + +void Client::_save_attachment( + int64_t message_id, + size_t index, + std::filesystem::path dest, + std::function on_progress, + failable_function cb, + bool notify_sender, + bool replace) { + + auto [url, key, digest, claimed_size] = _attachment_pointer(message_id, index); + + // Written to a temporary name beside the destination and renamed only once it is whole, so an + // interrupted save leaves nothing that looks finished. + struct SaveState { + std::filesystem::path dest, partial, saved_to; + std::ofstream out; + }; + auto state = std::make_shared(); + state->dest = std::move(dest); + state->partial = open_partial(state->out, state->dest); + + // The download reports only numbers; which attachment they are about is this layer's to say, so + // the identity is filled in before the shared hop. + std::function)> identified; + if (on_progress) + identified = [on_progress = std::move(on_progress), message_id, index]( + int64_t done, int64_t total, std::optional r) { + on_progress(AttachmentProgress{message_id, index, done, total, r}); + }; + auto report = _dispatch_progress(std::move(identified)); + + auto finish = [this, state, message_id, index, notify_sender, replace, cb]( + std::optional error) { + auto fail = [&](std::string why) { + state->out.close(); + std::error_code ec; + std::filesystem::remove(state->partial, ec); + _report(cb, std::optional{std::move(why)}, std::filesystem::path{}); + }; + + if (error) + return fail(std::move(*error)); + + try { + state->out.close(); + if (!state->out) + throw std::runtime_error{"writing {} failed"_format(state->partial.string())}; + + // Only now does it get a name a user would recognise -- and only now can it be + // known whether the one they asked for is still free, which is why the choice + // is here rather than where the caller made it. + state->saved_to = final_path(state->dest, replace); + std::filesystem::rename(state->partial, state->saved_to); + } catch (const std::exception& e) { + return fail(e.what()); + } + + _report(cb, std::optional{}, state->saved_to); + + // Both of these say the same thing -- that the recipient now has the file -- one to + // ourselves and one to the sender, and neither is true until it is on disk under + // its final name, which is why they are here rather than anywhere earlier. + // Recording it does not depend on telling them: a caller who saved privately still + // gets to see that they did. + // + // Both are also only true when the recipient is *us*. `saved_at` says the + // recipient saved it, which is what lets a sender read it as "the file reached a + // person rather than a file server", so stamping it for our own save of something + // we sent would claim they have a file they may never have opened. A note to self + // is exempt: there the recipient is us, so saving it really is the recipient + // saving it. + loop.call([this, message_id, index, notify_sender] { + if (_saved_by_recipient(message_id)) + _record_saved(message_id, index, clock_now_ms()); + + // The account's own answer overrides the caller's, and only downwards. + // Somebody who has said not to report their saves has said it for every client + // on the account, and a client that forgot to ask -- or never grew the setting + // -- would otherwise report them anyway. A caller passing false is still + // respected: this can refuse a notification, never require one. + if (notify_sender && core.configs.user_profile().get_notify_media_saved()) + _notify_media_saved(message_id, index); + }); + }; + + // Writes what was fetched to the destination and finishes as a completed download would, for + // the two paths that hand over a whole buffer rather than streaming into it. + auto write_and_finish = + [state, finish](std::optional error, const std::vector& data) { + if (error) + return finish(std::move(error)); + state->out.write( + reinterpret_cast(data.data()), + static_cast(data.size())); + finish(std::nullopt); + }; + + // Served from the cache when it is there. Indistinguishable to everyone else: the file lands + // where it was asked to, and the sender is still told we saved it, because being able to skip + // the download is our business and says nothing about whether the recipient has the file. + // + // No progress is reported for it -- there is no transfer to watch, and a bar that appears and + // completes in the same frame is noise. A save does not *fill* the cache, only read it: it has + // a destination of its own, and writing a second encrypted copy would double what it costs. + auto name = cache::path_for(_cache_dir, cache::ATTACHMENT_DIR, url).filename().string(); + if (!_cache_dir.empty()) { + auto file = cache::path_for(_cache_dir, cache::ATTACHMENT_DIR, url); + if (auto cached = cache::read(file, _cache_encryption_key())) { + _touch_cached(name); + write_and_finish(std::nullopt, *cached); + return; + } + } + + // Already being accumulated for somebody else -- a gallery, or the auto-downloader. Waiting on + // that costs nothing: the buffer is committed either way, and asking for the same bytes again + // would mean two transfers of one file. + // + // The reverse does not hold, which is why nothing is registered below: this streams to the + // destination as bytes arrive and keeps none of them, so there would be nothing to give a + // joiner that turned up midway. + if (auto found = _in_flight.find(name); found != _in_flight.end()) { + if (report) { + report(found->second.done, found->second.total, std::nullopt); + found->second.progress.push_back(report); + } + found->second.waiting.push_back( + [write_and_finish](std::optional error, std::vector data) { + write_and_finish(std::move(error), data); + }); + return; + } + + _download_decrypted( + url, + DownloadKind::attachment, + std::move(key), + std::move(digest), + claimed_size, + [state](std::span plain) { + state->out.write(reinterpret_cast(plain.data()), plain.size()); + }, + report, + finish); +} + +void Client::_on_media_saved( + std::span sender, + const SessionProtos::DataExtractionNotification& note, + sys_ms when) { + + // Both halves of the reference are required. The timestamp alone cannot identify a message -- + // several sent in the same millisecond share one, which is what msgId exists to fix -- and + // msgId alone is far too small to be an identifier. A notification carrying only the + // deprecated `timestamp` field says nothing usable: the three other clients disagree about + // what they put in it, so it is not read at all. + if (!note.has_msgtimestamp() || !note.has_msgid()) { + log::debug(cat, "Ignoring a media-saved notification that names no message"); + return; + } + + // Ours to have been saved: they can only have saved something we sent them, so the message + // must be one of ours, in the conversation with them. Matching on sender as well as + // conversation stops a peer claiming anything about a message they were not sent. + auto convo_id = ConversationId::dm(sender); + std::optional message_id; + { + auto c = core.database().conn(); + auto convo = find_conversation(c, convo_id); + if (!convo) + return; + + message_id = c.prepared_maybe_get( + R"( + SELECT id FROM messages + WHERE conversation = ? AND timestamp = ? AND msgid = ? AND outgoing = 1 + )", + *convo, + static_cast(note.msgtimestamp()), + static_cast(note.msgid())); + } + + if (!message_id) { + log::debug(cat, "A media-saved notification named a message we do not have"); + return; + } + + // -1 means they saved all of the message's attachments together, which is what saving from a + // gallery view does; anything else names one by position. An absent index is neither, and is + // not read as "the first one". + std::optional index; + if (note.has_attindex()) { + if (note.attindex() >= 0) + index = static_cast(note.attindex()); + else if (note.attindex() != -1) + return; + } else + return; + + // When they sent the notification, which is when they saved it -- *not* msgTimestamp, which + // identifies the message being talked about and may be days older. Their clock rather than + // ours, and unauthenticated, but the alternative is our receive time, which is wrong by however + // long the notification sat in the swarm. + _record_saved(*message_id, index, when); +} + +bool Client::_saved_by_recipient(int64_t message_id) { + auto c = core.database().conn(); + auto row = c.prepared_maybe_get>>( + R"( + SELECT m.outgoing, a.session_id + FROM messages m + JOIN conversations c ON c.id = m.conversation + LEFT JOIN accounts a ON a.id = c.dm + WHERE m.id = ? + )", + message_id); + if (!row) + return false; + auto [outgoing, with] = *row; + if (!outgoing) + return true; + return with && is_me(*with); +} + +void Client::_record_saved(int64_t message_id, std::optional index, sys_ms when) { + std::optional convo_id; + { + auto c = core.database().conn(); + + // An unset index means every attachment of the message, which is what a peer saving from a + // gallery view reports. Later saves overwrite earlier ones: what an application asks of + // this is "is there any point offering save again", not a history. + auto changed = index ? c.prepared_exec( + "UPDATE message_attachments SET saved_at = ?" + " WHERE message = ? AND idx = ?", + epoch_ms(when), + message_id, + static_cast(*index)) + : c.prepared_exec( + "UPDATE message_attachments SET saved_at = ?" + " WHERE message = ?", + epoch_ms(when), + message_id); + if (changed == 0) + return; + + auto convo = c.prepared_maybe_get( + "SELECT conversation FROM messages WHERE id = ?", message_id); + if (!convo) + return; + convo_id = conversation_id_at(c, *convo); + } + + _emit_message(false, *convo_id, message_id); +} + +void Client::_notify_media_saved(int64_t message_id, size_t index) { + std::optional convo_id; + int64_t timestamp = 0; + std::optional msgid; + { + auto c = core.database().conn(); + auto row = c.prepared_maybe_get>( + "SELECT conversation, outgoing, timestamp, msgid FROM messages WHERE id = ?", + message_id); + if (!row) + return; + auto [convo_row, outgoing, ts, id] = *row; + + // Saving from a message we sent notifies nobody: the person who would be told is us. That + // covers a note to self as well, which is outgoing. + if (outgoing) + return; + + convo_id = conversation_id_at(c, convo_row); + timestamp = ts; + msgid = id; + } + + if (convo_id->type() != ConversationId::Type::dm) + return; + + SessionProtos::Content content; + auto now = clock_now_ms(); + content.set_sigtimestamp(static_cast(epoch_ms(now))); + content.set_msgid(new_msgid()); + + auto* note = content.mutable_dataextractionnotification(); + note->set_type(SessionProtos::DataExtractionNotification::MEDIA_SAVED); + note->set_msgtimestamp(static_cast(timestamp)); + // Deliberately not set when the message we saved from carries none: the pair is what identifies + // a message, and half of it is not a weaker match but an ambiguous one. + if (msgid) + note->set_msgid(*msgid); + note->set_attindex(static_cast(index)); + + log::debug(cat, "Telling the sender we saved attachment {} of message {}", index, message_id); + + // Registered rather than fired blind: Core reports on every send, and a status for an id nobody + // claims would sit in _early_status for the life of the process. + _quiet_sends.insert(core.send_dm(convo_id->session_id(), content, now)); +} + +void Client::_finish_attachment_send(int64_t client_id) { + auto self = core.globals.session_id(); + + // Rebuilt from the database rather than from what send_message was handed, so that a message + // whose uploads finished can be completed by anything that finds it -- a retry, or a later run. + std::optional convo_id; + std::string body; + int64_t timestamp = 0; + SessionProtos::Content content; + + { + auto c = core.database().conn(); + + auto row = c.prepared_maybe_get< + int64_t, + std::string, + int64_t, + std::optional, + std::optional>, + std::optional, + std::optional>( + R"( + SELECT m.conversation, m.body, m.timestamp, m.msgid, + ra.session_id, m.reply_timestamp, m.reply_msgid + FROM messages m LEFT JOIN accounts ra ON ra.id = m.reply_author + WHERE m.id = ? + )", + client_id); + if (!row) { + log::warning(cat, "Cannot finish message {}: it is gone", client_id); + return; + } + auto [convo_row, msg_body, msg_ts, msg_id, reply_author, reply_ts, reply_msgid] = *row; + convo_id = conversation_id_at(c, convo_row); + body = std::move(msg_body); + timestamp = msg_ts; + + // The identifier the row was stored with, not a fresh one: rebuilding the content does not + // make this a different message, and a new identifier here would leave the copy we already + // showed the user and the copy we send disagreeing about which message they are. + auto* data = fill_outgoing_content(content, from_epoch_ms(timestamp), msg_id, body); + + // Rebuilt from the row for the same reason as everything else here: what goes out has to be + // reconstructible by whatever finds the message, not only by the call that started it. + if (reply_author && reply_ts) + set_quote( + *data, + WireRef{.author = *reply_author, + .timestamp = from_epoch_ms(*reply_ts), + .msgid = reply_msgid}); + + for (auto&& [url, key, size, ctype, fname, caption, flags, width, height] : + c.prepared_results< + std::string, + sqlite::blobn<32>, + int64_t, + std::optional, + std::optional, + std::optional, + int, + std::optional, + std::optional>( + R"( + SELECT url, key, size, content_type, filename, caption, flags, width, height + FROM message_attachments WHERE message = ? ORDER BY idx + )", + client_id)) { + auto* attach = data->add_attachments(); + attach->set_url(url); + + // Deprecated in favour of `url`, which is what current clients read, but still required + // by the protobuf and still read by old ones. It is the url's last segment, so it is + // taken back out of the url rather than tracked separately -- and it is only a number + // for as long as the file server hands out numbers, which it is expected to stop doing. + uint64_t legacy_id = 0; + if (auto parsed = network::file_server::parse_download_url(url)) { + const auto& fid = parsed->file_id; + const auto* end = fid.data() + fid.size(); + if (auto [stop, ec] = std::from_chars(fid.data(), end, legacy_id); + ec != std::errc{} || stop != end) + legacy_id = 0; + } + attach->set_id(legacy_id); + + // `key` views the statement's current row, so it has to be copied out before the loop + // steps to the next one. + attach->set_key(reinterpret_cast(key.data()), key.size()); + attach->set_size(static_cast(size)); + + if (ctype) + attach->set_contenttype(*ctype); + if (fname) + attach->set_filename(*fname); + if (caption) + attach->set_caption(*caption); + if (flags != 0) + attach->set_flags(static_cast(flags)); + if (width) + attach->set_width(static_cast(*width)); + if (height) + attach->set_height(static_cast(*height)); + } + } + + bool to_self = is_me(convo_id->session_id()); + + SessionProtos::Content synced = content; + synced.mutable_datamessage()->set_synctarget(oxenc::to_hex(convo_id->session_id())); + + // What we store is the sync copy, as the plain send does: it is what the copy coming back off + // our own swarm is recognised as, and the provisional content written before the uploads + // describes something nobody will ever send. + auto serialised = synced.SerializeAsString(); + auto raw = std::span{reinterpret_cast(serialised.data()), serialised.size()}; + + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + c.prepared_exec( + "UPDATE messages SET send_state = ?, sync_send_state = ? WHERE id = ?", + static_cast(SendState::pending), + to_self ? std::optional{} + : std::optional{static_cast(SendState::pending)}, + client_id); + c.prepared_exec( + "UPDATE message_raw_content SET content = ? WHERE message = ?", raw, client_id); + tx.commit(); + } + + _emit_message(false, *convo_id, client_id); + + _dispatch_sends( + client_id, + *convo_id, + content, + synced, + sys_ms{std::chrono::milliseconds{timestamp}}, + to_self); +} + +void Client::_fail_attachment_send(int64_t client_id, bool permanent) { + auto state = static_cast(permanent ? SendState::unsendable : SendState::failed); + std::optional convo_id; + { + auto c = core.database().conn(); + auto convo = c.prepared_maybe_get( + "SELECT conversation FROM messages WHERE id = ?", client_id); + if (!convo) + return; + convo_id = conversation_id_at(c, *convo); + + // The rows are left alone: what has already been uploaded stays recorded, which is what + // makes retrying send only what did not get through. + c.prepared_exec( + "UPDATE messages SET send_state = ?, sync_send_state =" + " CASE WHEN sync_send_state IS NULL THEN NULL ELSE ? END WHERE id = ?", + state, + state, + client_id); + } + + _emit_message(false, *convo_id, client_id); +} + +// -- Core event handling ---------------------------------------------------------------------- + +// Records what an arriving message says about the files it carries. Nothing is fetched here: an +// AttachmentPointer is a url and a key, and whether to spend bandwidth on it is the application's +// decision, made later through save_attachment. +// +// Everything the sender listed gets a row, including a pointer too malformed to ever fetch. The +// alternative -- dropping the unusable ones -- would make a message of three files look like a +// message of two, which is a worse lie than a row that cannot be saved: the count and the ordering +// are what a reader is being shown, and they should match what was actually sent. +static void store_incoming_attachments( + sqlite::Connection& c, int64_t message_id, const SessionProtos::DataMessage& data) { + for (int i = 0; i < data.attachments_size(); i++) { + const auto& ptr = data.attachments(i); + + // The key is 32 bytes for the stream scheme and 64 for the legacy one; anything else cannot + // decrypt, but is still stored so the attachment is at least visible. + std::optional> key; + if (ptr.has_key()) + key = to_span(ptr.key()); + + std::optional> digest; + if (ptr.has_digest()) + digest = to_span(ptr.digest()); + + c.prepared_exec( + R"( + INSERT INTO message_attachments + (message, idx, url, key, digest, size, content_type, filename, caption, flags, + width, height) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + )", + message_id, + static_cast(i), + ptr.has_url() ? std::optional{ptr.url()} : std::nullopt, + key, + digest, + ptr.has_size() ? std::optional{ptr.size()} : std::nullopt, + ptr.has_contenttype() ? std::optional{ptr.contenttype()} : std::nullopt, + ptr.has_filename() ? std::optional{ptr.filename()} : std::nullopt, + ptr.has_caption() ? std::optional{ptr.caption()} : std::nullopt, + static_cast(ptr.flags()), + ptr.has_width() ? std::optional{ptr.width()} : std::nullopt, + ptr.has_height() ? std::optional{ptr.height()} : std::nullopt); + } +} + +void Client::_on_message_received(core::ReceivedMessage&& msg) { + SessionProtos::Content content; + if (!content.ParseFromArray(msg.content.data(), static_cast(msg.content.size()))) { + log::warning(cat, "Dropping message {}: Content protobuf did not parse", msg.hash); + return; + } + + // Someone telling us they saved a file we sent them. Not history -- it changes an attachment + // we already have rather than adding anything -- so it is handled here and goes no further. + if (content.has_dataextractionnotification()) { + const auto& note = content.dataextractionnotification(); + if (note.type() == SessionProtos::DataExtractionNotification::MEDIA_SAVED) + _on_media_saved( + msg.sender_session_id, + note, + content.has_sigtimestamp() + ? from_epoch_ms(static_cast(content.sigtimestamp())) + : msg.timestamp); + return; + } + + // Someone asking us to remove a message they sent. Not history either -- it takes something + // away rather than adding anything -- so it is handled here and goes no further. + if (content.has_unsendrequest()) { + _on_unsend_request(msg.sender_session_id, content.unsendrequest()); + return; + } + + // Receipts, typing indicators and call signalling are not conversation history; ignore them + // rather than materialising an empty conversation for a stranger who is merely typing. + if (!content.has_datamessage()) + return; + const auto& data = content.datamessage(); + + // A message of nothing but files is an ordinary message here: what makes something not history + // is having no content of either kind, which is what the callbacks above are. + if (data.body().empty() && data.attachments_size() == 0) + return; + + // A one-to-one message is stored on both participants' swarms, so our own sent messages come + // back to us from our own swarm. For those the sender is us, which says nothing about which + // conversation they belong to; the recipient is carried in syncTarget instead. + // + // syncTarget is only honoured on a self-send. Session's other clients honour it on any + // incoming message, which lets a peer file their message into a conversation they are not part + // of; there is no case where a message from someone else needs it, so this does not. + bool outgoing = is_me(msg.sender_session_id); + + b33 convo_with = msg.sender_session_id; + if (outgoing && data.has_synctarget()) { + const auto& target = data.synctarget(); + auto parsed = parse_session_id(target); + if (!parsed) { + log::warning( + cat, + "Dropping message {}: syncTarget is not a session ID: {}", + msg.hash, + target); + return; + } + convo_with = *parsed; + } + + if (convo_with[0] != std::byte{0x05}) { + log::warning( + cat, "Dropping message {}: conversation target is not a 0x05 session ID", msg.hash); + return; + } + auto convo_id = ConversationId::dm(convo_with); + + // The signed timestamp is the authenticated one; the swarm's upload time is not. + auto ts = content.has_sigtimestamp() + ? from_epoch_ms(static_cast(content.sigtimestamp())) + : msg.timestamp; + + // What the sender says about themselves, and when they last changed it. All three move + // together: `profile_updated` is what decides between this and the Contacts config, so applying + // a name or a picture without the stamp that justifies it would leave the next comparison + // arguing from the wrong date. + std::optional name; + if (data.has_profile() && data.profile().has_displayname() && + !data.profile().displayname().empty()) + name = data.profile().displayname(); + + // A picture arrives split across two fields: the url inside their profile, and the key that + // decrypts it beside it rather than in it. Neither alone is worth storing -- a url we cannot + // decrypt is a download that can only fail. + std::optional pic_url; + std::optional> pic_key; + if (data.has_profile() && data.profile().has_profilepicture() && + !data.profile().profilepicture().empty() && data.has_profilekey() && + data.profilekey().size() == 32) { + pic_url = data.profile().profilepicture(); + pic_key = std::as_bytes(std::span{data.profilekey()}); + } + + // Zero when they did not say, which only a client old enough to predate the field does. That + // then loses to anything we have already been told, which is the right way round: a client that + // cannot say when its profile changed has no claim to overwrite one we know the date of. + int64_t profile_stamp = 0; + if (data.has_profile() && data.profile().has_lastupdateseconds()) + profile_stamp = static_cast(data.profile().lastupdateseconds()); + + auto msgid = msgid_of(content); + + bool created = false, inserted = false, renamed = false, contact_changed = false, + approved_them = false; + int64_t client_id = 0; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + + // A blocked account is refused here rather than filtered when drawing, so that nothing it + // sends becomes history, an unread count or a notification. Only what *they* sent: our own + // copy of a conversation we later blocked is still ours. + if (!outgoing && c.prepared_get( + R"( + SELECT count(*) FROM contacts ct JOIN accounts a ON a.id = ct.account + WHERE a.session_id = ? AND ct.blocked + )", + msg.sender_session_id) > 0) + return; + + auto convo = ensure_conversation(c, convo_id, ts); + created = convo.created; + auto sender = account_id(c, msg.sender_session_id); + + // Approval is recorded by messages flowing rather than by anyone setting it: sending to + // someone approves them, and receiving from someone means they approved us -- you cannot + // write to an account you have not accepted. So the two flags are a side effect here, and + // an unapproved contact with a message in it *is* a message request. + // + // `convo_with` and not the sender: a copy of our own outgoing message, arriving from our + // own swarm, is us approving whoever it was addressed to. + // + // Note to self is left alone -- we are not our own contact, and our own entry has no + // business in the Contacts config. + if (!is_me(convo_with)) { + auto with = account_id(c, convo_with); + auto made = ensure_contact(c, with, outgoing); + auto flagged = c.prepared_exec( + outgoing ? "UPDATE contacts SET approved = 1" + " WHERE account = ? AND NOT approved" + : "UPDATE contacts SET approved_me = 1" + " WHERE account = ? AND NOT approved_me", + with) > 0; + contact_changed = made || flagged; + + // Only our own approval moves a conversation between the lists. Learning that *they* + // approved *us* changes what we know about them and nothing about where they belong, + // and a request appearing for the first time is reported as the conversation it is. + approved_them = outgoing && contact_changed; + } + + // Delivery is at-least-once -- the swarm cursor advances per batch, so a crash mid-batch + // re-delivers it -- and either unique index is enough to recognise the redelivery. + // + // The msgid index is what also catches our own message coming back off our own swarm, which + // the swarm hash cannot: that copy is stored before it has one. A sender that set no msgid + // leaves NULL there, and SQLite treats NULLs as distinct, so those fall back to the swarm + // hash alone -- enough for a redelivery, not enough for a sender who stored twice. + // What this replies to, if anything. Stored as the sender addressed it -- their author and + // timestamp -- and resolved to a local message only on read. + // + // A quote whose author is not a session id we can parse is dropped rather than failing the + // message: an unusable reference is worth less than the message carrying it. The author is + // interned like any other, since in a group it may be someone we have no other row for. + std::optional reply_author; + std::optional reply_ts; + std::optional reply_msgid; + if (data.has_quote()) { + const auto& q = data.quote(); + auto author = parse_session_id(q.author()); + if (author && (*author)[0] == std::byte{0x05}) { + reply_author = account_id(c, *author); + reply_ts = static_cast(q.msgtimestamp()); + if (q.has_msgid()) + reply_msgid = q.msgid(); + } else + log::warning(cat, "Ignoring a quote with an unparseable author"); + } + + inserted = c.prepared_exec( + R"( + INSERT OR IGNORE INTO messages + (conversation, msgid, swarm_hash, sender, outgoing, timestamp, body, + send_state, reply_author, reply_timestamp, reply_msgid) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + )", + convo.id, + msgid, + msg.hash, + sender, + outgoing ? 1 : 0, + epoch_ms(ts), + data.body(), + // Retrieving it from a swarm is proof it got there, whichever device + // put it there. + outgoing ? std::optional{static_cast(SendState::sent)} + : std::optional{}, + reply_author, + reply_ts, + reply_msgid) > 0; + if (inserted) { + client_id = c.sql.getLastInsertRowid(); + c.prepared_exec( + "INSERT INTO message_raw_content (message, content) VALUES (?, ?)", + client_id, + std::span{msg.content}); + + store_incoming_attachments(c, client_id, data); + + // Whether an arrival is unread is this layer's decision, not the trigger's. Today + // that is just "newer than the watermark"; mutes and message requests will land here. + if (!outgoing) + c.prepared_exec( + R"( + UPDATE conversations SET unread_count = unread_count + 1 + WHERE id = ?1 AND ?2 > last_read + )", + convo.id, + epoch_ms(ts)); + } + + // Skipped for a self-send: the LokiProfile on one of those is our own, which belongs to the + // UserProfile config rather than to anything observed on the wire, and `name` here is used + // to name the conversation partner. + // + // Only when what they sent is at least as new as what we hold, which is the same test the + // Contacts config reconcile makes and against the same column: a message that took a week + // to arrive must not undo a profile change made since. + if (!outgoing && data.has_profile() && + profile_stamp >= c.prepared_get( + "SELECT profile_updated FROM accounts WHERE id = ?", sender)) + renamed = _update_profile( + c, sender, name, pic_url, pic_key, profile_stamp, ProfileSource::message); + + tx.commit(); + } + + if (contact_changed || renamed) + _sync_contact(convo_id); + if (created) + _emit_conversation_added(convo_id); + if (inserted) { + _reveal_note_to_self(convo_id); + // Before the message is announced, so that a display reacting to it already sees whether + // this is a gallery rather than being told once and corrected a moment later. + _auto_download(convo_id, client_id); + _emit_message(true, convo_id, client_id); + } + if (inserted || renamed) + _touch(convo_id); + + // Approval moves a conversation between the two lists, so both changed and neither changed in a + // way that naming one row would describe. + if (approved_them) + _emit_lists_replaced(); +} + +void Client::_on_send_status( + int64_t core_id, + core::MessageSendStatus status, + std::optional swarm_hash) { + if (auto quiet = _quiet_sends.find(core_id); quiet != _quiet_sends.end()) { + if (is_terminal(status)) { + _quiet_sends.erase(quiet); + if (status != core::MessageSendStatus::success) + log::debug( + cat, + "A send nobody is waiting on failed with status {}", + static_cast(status)); + } + return; + } + + if (auto sync = _sync_sends.find(core_id); sync != _sync_sends.end()) { + log::debug( + cat, "sync copy of send {} reached status {}", core_id, static_cast(status)); + auto client_id = sync->second; + if (is_terminal(status)) + _sync_sends.erase(sync); + _apply_send_status(client_id, status, true, swarm_hash); + return; + } + + auto it = _send_ids.find(core_id); + if (it == _send_ids.end()) { + // Fired from inside send_dm(), before it returned us the id to map. send_message() drains + // this as soon as it has the mapping. + _early_status.insert_or_assign( + core_id, + EarlyStatus{ + status, + swarm_hash ? std::optional{std::string{*swarm_hash}} : std::nullopt}); + return; + } + + auto [client_id, own_swarm] = it->second; + if (is_terminal(status)) + _send_ids.erase(it); + _apply_send_status(client_id, status, false, own_swarm ? swarm_hash : std::nullopt); +} + +void Client::_apply_send_status( + int64_t client_id, + core::MessageSendStatus status, + bool sync, + std::optional swarm_hash) { + auto state = state_for(status); + std::optional convo; + { + auto c = core.database().conn(); + SQLite::Transaction tx{c.sql}; + // The two sends own one column each, so a status for one never disturbs the other. + auto column = sync ? "sync_send_state"sv : "send_state"sv; + bool changed = + c.prepared_exec( + "UPDATE messages SET {0} = ?1 WHERE id = ?2 AND {0} IS NOT ?1"_format( + column), + static_cast(state), + client_id) > 0; + + // Set once and never revised: the hash is what a redelivery of this same message off our + // swarm dedupes against, so pointing it at a later store would strand the row it came from. + if (swarm_hash) + changed |= c.prepared_exec( + "UPDATE messages SET swarm_hash = ? WHERE id = ? AND swarm_hash IS " + "NULL", + *swarm_hash, + client_id) > 0; + + if (changed) { + auto convo_row = c.prepared_maybe_get( + "SELECT conversation FROM messages WHERE id = ?", client_id); + if (convo_row) + convo = conversation_id_at(c, *convo_row); + } + tx.commit(); + } + + if (convo) + _emit_message(false, *convo, client_id); +} + +} // namespace session::client diff --git a/src/client/conversation.cpp b/src/client/conversation.cpp new file mode 100644 index 000000000..01713242e --- /dev/null +++ b/src/client/conversation.cpp @@ -0,0 +1,235 @@ +#include + +// Every operation here is one line of dispatch onto a `_`-form on Client: the work itself lives +// there, next to the rest of what touches the database, and this file is only about which thread it +// happens on and who is told. The pairing is mechanical -- a handler form hands the work to +// `_async`, a waiting form hands it to the loop and lets it throw -- which is why they are together +// rather than split by direction. +// +// **A handler form must not capture `this`.** It returns before the work runs, and the caller is +// under no obligation to keep this object alive until then: the natural way to write any of these +// is on a temporary (`client.conversation(id, …)` hands one to a handler, and +// `*client.conversation(id, await)` is a temporary outright), and a list element dies whenever the +// list is replaced. So each captures the two things the work needs -- the Client, which outlives +// everything here, and the id, which is a value -- and nothing else. Capturing `this` reads the id +// out of freed memory, which surfaces as a `ConversationId::Type` outside the enum and an +// "unhandled conversation kind" from a long way away. +// +// The waiting forms are exempt: `call_get` runs the work before returning, inline when it is +// already the loop's thread, so there is no window in which the object could have gone. That +// asymmetry is also why tests written entirely in the waiting form prove nothing about the handler +// form. + +namespace session::client { + +// -- Reading ------------------------------------------------------------------------------------ + +void Conversation::messages(failable_function)> cb) const { + messages(50, std::nullopt, std::move(cb)); +} +void Conversation::messages(int limit, failable_function)> cb) const { + messages(limit, std::nullopt, std::move(cb)); +} +void Conversation::messages( + int limit, + std::optional before, + failable_function)> cb) const { + messages(limit, before, false, std::move(cb)); +} +void Conversation::messages( + int limit, + std::optional before, + bool include_deleted, + failable_function)> cb) const { + _client->_require_page("messages", limit); + _client->_async( + [c = _client, id = id, limit, before, include_deleted] { + return c->_messages(id, limit, before, include_deleted); + }, + std::move(cb)); +} + +std::vector Conversation::messages(await_t) const { + return messages(50, std::nullopt, await); +} +std::vector Conversation::messages(int limit, await_t) const { + return messages(limit, std::nullopt, await); +} +std::vector Conversation::messages( + int limit, std::optional before, await_t) const { + return messages(limit, before, false, await); +} +std::vector Conversation::messages( + int limit, std::optional before, bool include_deleted, await_t) const { + _client->_require_page("messages", limit); + return _client->loop.call_get([this, limit, before, include_deleted] { + return _client->_messages(id, limit, before, include_deleted); + }); +} + +void Conversation::purge_deleted(failable_function cb) { + _client->_require_dm("purge_deleted", id); + _client->_async([c = _client, id = id] { return c->_purge_deleted(id); }, std::move(cb)); +} +size_t Conversation::purge_deleted(await_t) { + _client->_require_dm("purge_deleted", id); + return _client->loop.call_get([this] { return _client->_purge_deleted(id); }); +} + +// -- Read state --------------------------------------------------------------------------------- + +void Conversation::mark_read(failable_function cb) { + mark_read(std::nullopt, std::move(cb)); +} +void Conversation::mark_read(std::optional up_to, failable_function cb) { + _client->_async([c = _client, id = id, up_to] { c->_mark_read(id, up_to); }, std::move(cb)); +} +void Conversation::mark_read(await_t) { + mark_read(std::nullopt, await); +} +void Conversation::mark_read(std::optional up_to, await_t) { + _client->loop.call_get([this, up_to] { _client->_mark_read(id, up_to); }); +} + +void Conversation::set_marked_unread(bool unread, failable_function cb) { + _client->_async( + [c = _client, id = id, unread] { c->_set_marked_unread(id, unread); }, std::move(cb)); +} +void Conversation::set_marked_unread(bool unread, await_t) { + _client->loop.call_get([this, unread] { _client->_set_marked_unread(id, unread); }); +} + +// -- Settings ----------------------------------------------------------------------------------- + +void Conversation::set_priority(int priority, failable_function cb) { + _client->_async( + [c = _client, id = id, priority] { c->_set_priority(id, priority); }, std::move(cb)); +} +void Conversation::set_priority(int priority, await_t) { + _client->loop.call_get([this, priority] { _client->_set_priority(id, priority); }); +} + +void Conversation::set_notifications(config::notify_mode mode, failable_function cb) { + _client->_async( + [c = _client, id = id, mode] { c->_set_notifications(id, mode); }, std::move(cb)); +} +void Conversation::set_notifications(config::notify_mode mode, await_t) { + _client->loop.call_get([this, mode] { _client->_set_notifications(id, mode); }); +} + +void Conversation::set_mute_until(std::chrono::sys_seconds until, failable_function cb) { + _client->_async( + [c = _client, id = id, until] { c->_set_mute_until(id, until); }, std::move(cb)); +} +void Conversation::set_mute_until(std::chrono::sys_seconds until, await_t) { + _client->loop.call_get([this, until] { _client->_set_mute_until(id, until); }); +} + +void Conversation::set_expiry( + config::expiration_mode mode, std::chrono::seconds timer, failable_function cb) { + _client->_async( + [c = _client, id = id, mode, timer] { c->_set_expiry(id, mode, timer); }, + std::move(cb)); +} +void Conversation::set_expiry(config::expiration_mode mode, std::chrono::seconds timer, await_t) { + _client->loop.call_get([this, mode, timer] { _client->_set_expiry(id, mode, timer); }); +} + +void Conversation::set_auto_download(AutoDownload mode, failable_function cb) { + _client->_async( + [c = _client, id = id, mode] { c->_set_auto_download(id, mode); }, std::move(cb)); +} +void Conversation::set_auto_download(AutoDownload mode, await_t) { + _client->loop.call_get([this, mode] { _client->_set_auto_download(id, mode); }); +} + +// -- Sending ------------------------------------------------------------------------------------ + +void Conversation::send_message( + OutgoingMessage msg, + upload_progress on_upload, + failable_function cb) { + _client->_require_sendable("send_message", id, msg); + _client->_async( + [c = _client, id = id, msg = std::move(msg), on_upload = std::move(on_upload)] { + return c->_send_message(id, msg, on_upload); + }, + std::move(cb)); +} +void Conversation::send_message( + OutgoingMessage msg, failable_function cb) { + send_message(std::move(msg), nullptr, std::move(cb)); +} +int64_t Conversation::send_message(OutgoingMessage msg, await_t) { + return send_message(std::move(msg), nullptr, await); +} +int64_t Conversation::send_message(OutgoingMessage msg, upload_progress on_upload, await_t) { + _client->_require_sendable("send_message", id, msg); + return _client->loop.call_get( + [&] { return _client->_send_message(id, msg, std::move(on_upload)); }); +} + +// -- Destroying --------------------------------------------------------------------------------- + +void Conversation::clear_messages(failable_function cb) { + _client->_require_dm("clear_messages", id); + _client->_async([c = _client, id = id] { c->_clear_messages(id); }, std::move(cb)); +} +void Conversation::clear_messages(await_t) { + _client->_require_dm("clear_messages", id); + _client->loop.call_get([this] { _client->_clear_messages(id); }); +} + +void Conversation::delete_conversation(failable_function cb) { + delete_conversation(false, std::move(cb)); +} +void Conversation::delete_conversation(bool keep_messages, failable_function cb) { + _client->_require_dm("delete_conversation", id); + _client->_async( + [c = _client, id = id, keep_messages] { c->_delete_conversation(id, keep_messages); }, + std::move(cb)); +} +void Conversation::delete_conversation(await_t) { + delete_conversation(false, await); +} +void Conversation::delete_conversation(bool keep_messages, await_t) { + _client->_require_dm("delete_conversation", id); + _client->loop.call_get( + [this, keep_messages] { _client->_delete_conversation(id, keep_messages); }); +} + +// -- One-to-one only ---------------------------------------------------------------------------- + +void DM::set_blocked(bool blocked, failable_function cb) { + _client->_require_contact("set_blocked", id); + _client->_async( + [c = _client, id = id, blocked] { c->_set_blocked(id, blocked); }, std::move(cb)); +} +void DM::set_blocked(bool blocked, await_t) { + _client->_require_contact("set_blocked", id); + _client->loop.call_get([this, blocked] { _client->_set_blocked(id, blocked); }); +} + +void DM::set_nickname(std::string_view nickname, failable_function cb) { + _client->_require_contact("set_nickname", id); + _client->_async( + [c = _client, id = id, nickname = std::string{nickname}] { + c->_set_nickname(id, nickname); + }, + std::move(cb)); +} +void DM::set_nickname(std::string_view nickname, await_t) { + _client->_require_contact("set_nickname", id); + _client->loop.call_get([this, nickname] { _client->_set_nickname(id, nickname); }); +} + +void DM::delete_contact(failable_function cb) { + _client->_require_contact("delete_contact", id); + _client->_async([c = _client, id = id] { c->_delete_contact(id); }, std::move(cb)); +} +void DM::delete_contact(await_t) { + _client->_require_contact("delete_contact", id); + _client->loop.call_get([this] { _client->_delete_contact(id); }); +} + +} // namespace session::client diff --git a/src/client/conversation_id.cpp b/src/client/conversation_id.cpp new file mode 100644 index 000000000..98ba35f92 --- /dev/null +++ b/src/client/conversation_id.cpp @@ -0,0 +1,108 @@ +#include + +#include +#include +#include +#include +#include + +namespace session::client { + +using namespace std::literals; + +static constexpr std::string_view COMMUNITY_PREFIX = "community:"sv; + +ConversationId ConversationId::_from_prefixed( + std::span id, std::byte want, Type type) { + if (id[0] != want) + throw std::invalid_argument{ + "Invalid conversation id: expected a 0x{:02x} prefix, got 0x{:02x}"_format( + std::to_integer(want), std::to_integer(id[0]))}; + return ConversationId{type, std::string{reinterpret_cast(id.data()), id.size()}}; +} + +ConversationId ConversationId::dm(std::span session_id) { + return _from_prefixed(session_id, std::byte{0x05}, Type::dm); +} + +ConversationId ConversationId::group(std::span group_id) { + return _from_prefixed(group_id, std::byte{0x03}, Type::group); +} + +static std::string lowercase(std::string_view s) { + std::string out; + out.reserve(s.size()); + std::ranges::transform(s, std::back_inserter(out), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return out; +} + +ConversationId ConversationId::community(std::string_view base_url, std::string_view room) { + // Normalise so that the same community written two ways is one conversation, not two: the + // scheme and host are case-insensitive, room tokens are defined lowercase, and a trailing + // slash on the server URL is meaningless. + while (base_url.ends_with('/')) + base_url.remove_suffix(1); + + if (base_url.empty()) + throw std::invalid_argument{"Invalid community conversation id: empty server URL"}; + if (room.empty()) + throw std::invalid_argument{"Invalid community conversation id: empty room token"}; + if (room.find('/') != std::string_view::npos) + throw std::invalid_argument{"Invalid community conversation id: room contains a '/'"}; + + return ConversationId{Type::community, "{}/{}"_format(lowercase(base_url), lowercase(room))}; +} + +ConversationId ConversationId::parse(std::string_view s) { + if (s.starts_with(COMMUNITY_PREFIX)) { + auto rest = s.substr(COMMUNITY_PREFIX.size()); + auto slash = rest.rfind('/'); + if (slash == std::string_view::npos) + throw std::invalid_argument{"Invalid community conversation id: no room token"}; + return community(rest.substr(0, slash), rest.substr(slash + 1)); + } + + if (s.size() != 66 || !oxenc::is_hex(s)) + throw std::invalid_argument{"Invalid conversation id: not a hex session or group ID"}; + + auto raw = oxenc::from_hex(s); + std::span id{reinterpret_cast(raw.data()), 33}; + if (raw[0] == '\x05') + return dm(id); + if (raw[0] == '\x03') + return group(id); + throw std::invalid_argument{ + "Invalid conversation id: unrecognised prefix {:.2s}"_format(std::string_view{s})}; +} + +std::span ConversationId::session_id() const { + if (_type != Type::dm) + throw std::logic_error{"session_id() called on a non-DM conversation id"}; + return std::span{ + reinterpret_cast(_key.data()), _key.size()}; +} + +std::span ConversationId::group_id() const { + if (_type != Type::group) + throw std::logic_error{"group_id() called on a non-group conversation id"}; + return std::span{ + reinterpret_cast(_key.data()), _key.size()}; +} + +std::pair ConversationId::community() const { + if (_type != Type::community) + throw std::logic_error{"community() called on a non-community conversation id"}; + std::string_view key{_key}; + auto slash = key.rfind('/'); + return {key.substr(0, slash), key.substr(slash + 1)}; +} + +std::string ConversationId::to_string() const { + if (_type == Type::community) + return "{}{}"_format(COMMUNITY_PREFIX, _key); + return oxenc::to_hex(_key); +} + +} // namespace session::client diff --git a/src/client/download_cache.cpp b/src/client/download_cache.cpp new file mode 100644 index 000000000..f0997f165 --- /dev/null +++ b/src/client/download_cache.cpp @@ -0,0 +1,116 @@ +#include "download_cache.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace session::client::cache { + +namespace log = oxen::log; +static auto cat = log::Cat("client"); + +namespace { + + std::string_view base_url(std::string_view url) { + if (auto q = url.find_first_of("?#"); q != std::string_view::npos) + url = url.substr(0, q); + return url; + } + +} // namespace + +std::filesystem::path path_for( + const std::filesystem::path& dir, std::string_view kind, std::string_view url) { + auto h = hash::blake2b<32>(base_url(url)); + return dir / kind / oxenc::to_hex(h.begin(), h.end()); +} + +std::optional> read( + const std::filesystem::path& file, std::span key) { + std::error_code ec; + if (!std::filesystem::exists(file, ec)) + return std::nullopt; + + try { + std::ifstream in{file, std::ios::binary | std::ios::ate}; + in.exceptions(std::ios::failbit | std::ios::badbit); + + std::vector encrypted(static_cast(in.tellg())); + in.seekg(0); + in.read(reinterpret_cast(encrypted.data()), + static_cast(encrypted.size())); + + return attachment::decrypt(encrypted, key); + } catch (const std::exception& e) { + // A cache that cannot answer is a cache miss; the caller fetches instead. Removed because + // nothing else ever would: it is not referenced by anything that could notice it is bad. + log::warning(cat, "Discarding unreadable cache entry {}: {}", file.string(), e.what()); + std::filesystem::remove(file, ec); + return std::nullopt; + } +} + +void write( + const std::filesystem::path& file, + std::span key, + std::span data) { + std::filesystem::create_directories(file.parent_path()); + + // Unique, so two writes of the same url cannot land on one temporary and interleave. + auto tmp = file; + tmp += "{}{}"_format(random::unique_id("-", 8), PARTIAL_SUFFIX); + + { + std::ofstream out{tmp, std::ios::binary | std::ios::trunc}; + out.exceptions(std::ios::failbit | std::ios::badbit); + + attachment::Encryptor enc{key}; + size_t pos = 0; + enc.start_encryption( + [&](std::span buf) -> size_t { + auto n = std::min(buf.size(), data.size() - pos); + std::memcpy(buf.data(), data.data() + pos, n); + pos += n; + return n; + }, + true, + data.size()); + + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) + out.write( + reinterpret_cast(chunk.data()), + static_cast(chunk.size())); + } + + // Atomic: the finished name never exists holding a partial file, so a reader either misses or + // gets the whole thing. + std::filesystem::rename(tmp, file); +} + +std::vector list(const std::filesystem::path& dir, std::string_view kind) { + std::vector names; + + std::error_code ec; + for (const auto& entry : std::filesystem::directory_iterator{dir / kind, ec}) { + auto name = entry.path().filename().string(); + // A download still running is not garbage, it is unfinished, and unlinking it mid-write + // would make the fetch fail for a reason nothing could explain. + if (name.ends_with(PARTIAL_SUFFIX)) + continue; + names.push_back(std::move(name)); + } + return names; +} + +bool remove(const std::filesystem::path& dir, std::string_view kind, std::string_view name) { + std::error_code ec; + return std::filesystem::remove(dir / kind / name, ec); +} + +} // namespace session::client::cache diff --git a/src/client/download_cache.hpp b/src/client/download_cache.hpp new file mode 100644 index 000000000..59a9954fa --- /dev/null +++ b/src/client/download_cache.hpp @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +/// A cache of files we downloaded, on disk, encrypted under one key of our own. +/// +/// Re-encrypted rather than kept as the file server gave them to us, because the key that opened a +/// download belongs to something that goes away: an attachment's key lives in the message, a +/// profile picture's in the Contacts config, and a deleted message or a changed picture would leave +/// us holding a file we can no longer read. A key we own outlives both. +/// +/// Encrypted rather than kept as plaintext because the disk is not a trusted place: it ends up in +/// backups, in disk images, and in whoever's hands the machine does. That is also why the padding +/// is kept -- see `attachment::Encryptor`'s key-taking constructor. +/// +/// The two halves are kept differently, because only one of them expires. A cached picture is just +/// a file: what still references it is already recorded in `accounts.profile_pic_url`, and a second +/// record of it would be a second thing to keep in step. A cached attachment additionally has a +/// row in `attachment_cache`, because eviction has to answer "what is the total, and what was used +/// longest ago" without stat-ing the whole directory to find out. +/// +/// That row is an index over the file rather than a second copy of the truth, and either can be +/// missing the other after a crash: a sweep reconciles both directions. +namespace session::client::cache { + +/// The subdirectory a kind of download lives in. Separate so that a sweep of one cannot consider +/// the other's files unreferenced, which it would, since neither knows the other's urls. +inline constexpr std::string_view PROFILE_DIR = "profile"; +inline constexpr std::string_view ATTACHMENT_DIR = "attachments"; + +/// Where `url`'s cached file lives beneath `dir`. +/// +/// Named for a hash of the url with any query string and fragment removed. Those are not part of +/// which file this is: the fragment carries what is needed to *reach* and unpack the bytes -- the +/// server's key, connection details, decompression hints -- while the bytes at the base url are the +/// bytes. Two references differing only there name the same file, and hashing the whole thing +/// would cache it twice. +/// +/// Hashed rather than escaped because a url is not a filename: it is long, it contains separators, +/// and its length is unbounded. A fixed-width hex name is none of those things. +std::filesystem::path path_for( + const std::filesystem::path& dir, std::string_view kind, std::string_view url); + +/// Reads and decrypts a cached file, or nullopt if it is not there. +/// +/// Nullopt for an unreadable or corrupted file too, rather than throwing: a cache that cannot +/// answer is a cache miss, and the caller's next move -- fetch it again -- is the same either way. +/// The unreadable file is removed, since nothing else would ever remove it. +std::optional> read( + const std::filesystem::path& file, std::span key); + +/// Encrypts `data` and writes it to `file`, creating the directory if needed. +/// +/// Written to a temporary name in the same directory and renamed into place, so a crash or a +/// concurrent read never sees a half-written file: the rename is atomic and the name only ever +/// exists complete. The temporary carries a suffix a sweep knows to leave alone. +void write( + const std::filesystem::path& file, + std::span key, + std::span data); + +/// The suffix an in-progress write carries. A sweep must skip these: it decides what to delete by +/// what is *not* referenced, and a download that has not finished is not referenced yet. +inline constexpr std::string_view PARTIAL_SUFFIX = ".part"; + +/// The names of the finished files in `dir/kind`, in no particular order. +/// +/// Filesystem only, reading nothing else, so this half of a sweep can run off the event loop -- +/// which is the point of it being its own function. Walking a directory of many thousands of files +/// is the slow part; deciding what to do about them is a handful of queries. +/// +/// In-progress writes are left out, since a download that has not finished is not yet referenced by +/// anything and a sweep would take it for garbage. +std::vector list(const std::filesystem::path& dir, std::string_view kind); + +/// Unlinks a file `list` named. True if it went, false if it was already gone -- which is not an +/// error: between listing a name and acting on it, eviction may have removed it anyway. +bool remove(const std::filesystem::path& dir, std::string_view kind, std::string_view name); + +} // namespace session::client::cache diff --git a/src/client/schema/000_message_deletion.sql b/src/client/schema/000_message_deletion.sql new file mode 100644 index 000000000..3bb0a2928 --- /dev/null +++ b/src/client/schema/000_message_deletion.sql @@ -0,0 +1 @@ +ALTER TABLE messages ADD COLUMN deleted INTEGER; diff --git a/src/client/schema/001_auto_download.sql b/src/client/schema/001_auto_download.sql new file mode 100644 index 000000000..caa09f690 --- /dev/null +++ b/src/client/schema/001_auto_download.sql @@ -0,0 +1 @@ +ALTER TABLE conversations ADD COLUMN auto_download INTEGER; diff --git a/src/client/schema/002_attachment_cache.sql b/src/client/schema/002_attachment_cache.sql new file mode 100644 index 000000000..4d240467c --- /dev/null +++ b/src/client/schema/002_attachment_cache.sql @@ -0,0 +1,7 @@ +CREATE TABLE attachment_cache ( + name TEXT PRIMARY KEY NOT NULL, + size INTEGER NOT NULL, + last_used INTEGER NOT NULL +) STRICT; + +CREATE INDEX attachment_cache_lru ON attachment_cache(last_used); diff --git a/src/client/schema/003_gallery.sql b/src/client/schema/003_gallery.sql new file mode 100644 index 000000000..3d8c0ff3c --- /dev/null +++ b/src/client/schema/003_gallery.sql @@ -0,0 +1 @@ +ALTER TABLE messages ADD COLUMN gallery INTEGER NOT NULL DEFAULT 0; diff --git a/src/client/schema/004_replies.sql b/src/client/schema/004_replies.sql new file mode 100644 index 000000000..00e072f97 --- /dev/null +++ b/src/client/schema/004_replies.sql @@ -0,0 +1,6 @@ +ALTER TABLE messages ADD COLUMN reply_author INTEGER REFERENCES accounts(id); +ALTER TABLE messages ADD COLUMN reply_timestamp INTEGER; +ALTER TABLE messages ADD COLUMN reply_msgid INTEGER; + +CREATE INDEX messages_reply_target ON messages(conversation, reply_author, reply_timestamp) + WHERE reply_timestamp IS NOT NULL; diff --git a/src/client/schema/CMakeLists.txt b/src/client/schema/CMakeLists.txt new file mode 100644 index 000000000..f3a516309 --- /dev/null +++ b/src/client/schema/CMakeLists.txt @@ -0,0 +1,5 @@ +session_schema_dir( + TARGET client + NAMESPACE session::client::schema + DECLARE_HEADER session/client/schema/schema_registry.hpp +) diff --git a/src/client/schema/full_schema.sql b/src/client/schema/full_schema.sql new file mode 100644 index 000000000..fafdc1442 --- /dev/null +++ b/src/client/schema/full_schema.sql @@ -0,0 +1,411 @@ +-- The Client schema with every migration in this directory applied. A database with none of +-- them applied is built from this in one step; the migrations are then recorded without being +-- run. Keep this in step with the migrations: test_client.cpp compares the two. + +-- Every account we have seen, whether or not it is a contact: a message sender needs a row, so +-- this is "accounts we know of". Being a contact is a row in `contacts`, not a property here. +-- +-- The profile fields are what an account says about itself, learned either from the LokiProfile on +-- an incoming message or from the Contacts config. `profile_updated` decides between them: it is +-- the account's *own* stamp of when it last changed its profile, not ours of when we saw it, so a +-- message arriving out of order cannot replace a newer profile with an older one. Out of order is +-- the normal case rather than the exception, since a profile is observed from group messages as +-- readily as from a DM. A message carrying no stamp counts as 0, so it applies only when we have +-- never had one. +-- +-- The same stamp is what keeps this from churning: every message from someone repeats the value +-- they last set, so the common case compares equal, changes nothing, and dirties no config. +CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + -- 33 bytes. The prefix says which kind of identity it is: 0x05 for an account, 0x15 or 0x25 + -- for a community-blinded one, which carries a profile of its own and so gets its own row. + session_id BLOB NOT NULL UNIQUE, + name TEXT, -- NULL when no name is known; never an empty string + -- Set together, or both NULL: a URL without its key cannot be decrypted, so either one missing + -- is no picture at all. + profile_pic_url TEXT, + profile_pic_key BLOB, -- 32 bytes + profile_updated INTEGER NOT NULL DEFAULT 0, -- unix seconds + -- Session Pro. `pro_flags` is what the profile claims -- a badge, an animated avatar -- and + -- the proof is what lets the claim be checked. A NULL tag means we have never seen a proof, + -- which is not the same as holding one that has expired. + pro_flags INTEGER NOT NULL DEFAULT 0, + -- Set together or both NULL, like the picture above: an expiry with no tag beside it cannot be + -- checked against the revocation list, so neither half says anything on its own. + pro_revocation_tag BLOB, -- 32 bytes + pro_expiry INTEGER -- unix seconds +) STRICT; + +-- One row per entry in the Contacts config. Existence here *is* being a contact, so there is no +-- flag that could disagree with itself, and a contact removed on another device becomes a row that +-- goes away -- while the `accounts` row stays, because we have still seen that person. +-- +-- Only what needs a relationship to mean anything lives here. A name and a picture belong to the +-- account, since we learn those for people who are not contacts at all. +CREATE TABLE contacts ( + account INTEGER PRIMARY KEY REFERENCES accounts(id) ON DELETE CASCADE, + nickname TEXT, -- our own name for them, synced to our other devices + -- Approval in each direction: whether we have accepted messages from this account, and whether + -- it has accepted ours. This is what separates a conversation from a message request. + approved INTEGER NOT NULL DEFAULT 0, + approved_me INTEGER NOT NULL DEFAULT 0, + blocked INTEGER NOT NULL DEFAULT 0 +) STRICT; + +CREATE TABLE groups ( + id INTEGER PRIMARY KEY, + group_id BLOB NOT NULL UNIQUE -- 33 bytes, 0x03-prefixed +) STRICT; + +CREATE TABLE communities ( + id INTEGER PRIMARY KEY, + base_url TEXT NOT NULL, -- lowercased, no trailing slash + room TEXT NOT NULL, -- lowercased + UNIQUE (base_url, room) +) STRICT; + +CREATE TABLE conversations ( + id INTEGER PRIMARY KEY, + -- What the conversation is with. Exactly one of these is set, and which one it is *is* the + -- conversation's kind -- there is no separate type column that could disagree with it. Each + -- is UNIQUE so a given peer, group or room has at most one conversation; SQLite treats NULLs + -- as distinct, so the unused columns do not collide across rows. + -- ("closed_group" rather than "group" because the latter is a SQL keyword.) + dm INTEGER UNIQUE REFERENCES accounts(id), + closed_group INTEGER UNIQUE REFERENCES groups(id), + community INTEGER UNIQUE REFERENCES communities(id), + -- Time units differ by column here, deliberately and in step with what the configs carry: a + -- value that gets compared against a message timestamp is in milliseconds, and everything else + -- is unix seconds. Nobody needs the sub-second moment a conversation was created. + created INTEGER NOT NULL, -- unix seconds + last_activity INTEGER NOT NULL, -- ms since epoch; conversation list ordering + -- Read watermark: incoming messages with a strictly greater timestamp are unread. A timestamp + -- rather than a per-message flag because that is what ConvoInfoVolatile syncs. + -- + -- Milliseconds, for the reason above: the other side of the comparison is `messages.timestamp`. + -- At second resolution, reading a message stamped mid-second would force a choice between + -- leaving that message unread and marking everything else in the same second read. + last_read INTEGER NOT NULL DEFAULT 0, + -- Cached rather than counted per query: a conversation list is redrawn far more often than its + -- messages change. `count` is maintained by the triggers below; `unread_count` is maintained + -- by the application, for the reason given there. + count INTEGER NOT NULL DEFAULT 0, + unread_count INTEGER NOT NULL DEFAULT 0, + -- Pinning, mirroring the value the Contacts and UserGroups configs sync: 0 is unpinned, a + -- positive value is pinned with higher values sorting first, and a negative value is hidden. + -- Kept numerically identical to the config so that reconciling one into the other is a copy + -- rather than a translation. + priority INTEGER NOT NULL DEFAULT 0, + -- The settings below are carried per-kind by the configs -- `contact_info` for a DM, + -- `base_group_info` for a group or community -- but they mean the same thing in each, so they + -- are one set of columns here rather than three. + -- + -- Set by the user to make a conversation unread again after reading it. Independent of + -- unread_count, which counts messages: this survives having read all of them. + marked_unread INTEGER NOT NULL DEFAULT 0, + -- 0 default, 1 all, 2 disabled, 3 mentions only. Mentions-only is a group notion; a DM + -- carrying it reads as `all`. + notifications INTEGER NOT NULL DEFAULT 0, + mute_until INTEGER NOT NULL DEFAULT 0, -- unix seconds; 0 is not muted + -- Disappearing messages: 0 none, 1 after send, 2 after read. The timer is meaningless when the + -- mode is none, and is a duration rather than a moment, so seconds either way. + exp_mode INTEGER NOT NULL DEFAULT 0, + exp_timer INTEGER NOT NULL DEFAULT 0, -- seconds + -- What to fetch without being asked, for this conversation: 0 nothing, 1 images, 2 everything. + -- + -- NULL is the fourth state and the reason this is nullable: nobody has been asked yet. Session + -- asks within a conversation the first time and remembers the answer, which a client can only + -- do if "never asked" is distinguishable from "asked, said no". + -- + -- Device-local, deliberately, and so absent from every config: whether to spend this machine's + -- bandwidth and disk is a property of the machine. Auto-downloading everything on a desktop + -- and nothing on a phone is the case this exists for, and syncing it would make that + -- impossible. Nothing in the reconcile reads or writes it. + auto_download INTEGER, + CHECK ((dm IS NOT NULL) + (closed_group IS NOT NULL) + (community IS NOT NULL) = 1) +) STRICT; + +CREATE INDEX conversations_order ON conversations(priority DESC, last_activity DESC); + +CREATE TABLE messages ( + id INTEGER PRIMARY KEY, + conversation INTEGER NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + -- The sender's Content.msgId: 8 random bytes separating this message from another they sent in + -- the same millisecond. Every copy of a message carries the same value -- it is set before the + -- copy for the recipient and the copy for our own swarm diverge -- which is what makes it the + -- one identifier both parties agree on. + -- + -- Only ever meaningful together with `timestamp`; it is far too small to identify a message on + -- its own. See the field's comment in SessionProtos.proto. + -- + -- NULL for a message whose sender predates the field, which cannot then be told apart from + -- another they sent in the same millisecond. + -- + -- An opaque 64-bit pattern rather than a number: SQLite integers are signed, so a value above + -- INT64_MAX is stored negative. Compared for equality and nothing else. + msgid INTEGER, + -- Swarm-assigned hash; NULL for an outgoing message not yet stored on the swarm. SQLite + -- treats NULLs as distinct in a unique index, so those never collide with each other. + swarm_hash TEXT UNIQUE, + sender INTEGER NOT NULL REFERENCES accounts(id), + outgoing INTEGER NOT NULL, + timestamp INTEGER NOT NULL, -- ms since epoch; the Content sigTimestamp + body TEXT NOT NULL, + -- Delivery state of the copy sent to the recipient's swarm, for outgoing messages only: + -- 0 = queued locally, not yet dispatched + -- 1 = handed to the swarm, awaiting confirmation + -- 2 = accepted by a swarm node + -- 3 = terminal failure + -- 4 = in flight when the process exited, so the outcome is unknown + -- + -- ...and the same for the copy deposited in our own swarm, which is how our other devices see + -- an outgoing message. The two are independent sends, retried separately, so a message can + -- have reached its recipient while still owing our other devices a copy. + -- + -- NULL in either means there is no such send: both are NULL on an incoming message, and + -- sync_send_state is NULL for a note to self, where the recipient's swarm is our own and a + -- second store would be the same store twice. + send_state INTEGER, + sync_send_state INTEGER, + -- Set once the message's content has been removed but the row has to stay: + -- 1 = deleted here only + -- 2 = deleted everywhere, either because we asked for that or because the sender did + -- + -- A row rather than a DELETE because `swarm_hash` is what a redelivery dedupes against, and a + -- message deleted only here is still on the swarm to be delivered again. Removing the row + -- would let it come back looking new -- which is not hypothetical: a storage server stops + -- honouring a `last_hash` once it has expired, and the poll that follows returns the whole + -- retention window. + -- + -- Which of the two it is has to survive, not just that it happened: deleting for everyone is + -- still offerable on a message already deleted here, and the two are different things to draw. + -- The direction that completes the picture is `outgoing`, so there is no column for it. + deleted INTEGER, + + -- Whether this message is *shown* as a gallery rather than as a list of attachments. + -- + -- A decision, which is why it is stored: it is made when the message is processed -- on if the + -- conversation was auto-downloading then -- and the conversation's setting may have changed + -- since, so it cannot be recomputed later and mean the same thing. A client may set or clear + -- it afterwards. + -- + -- Whether a message *can* be shown that way is a separate question, derived and not stored: + -- today it is "has attachments, and every one of them is an image", but that is our rule to + -- change -- to particular formats, or a size ceiling -- and a stored answer would be a stale + -- one after any such change. So a message whose stored decision no longer agrees with the + -- current rule simply has the decision dropped, rather than being honoured or migrated. + gallery INTEGER NOT NULL DEFAULT 0, + + -- What this message is a reply to, as the sender addressed it: never a local message id. + -- + -- Whether we have the replied-to message changes in *both* directions after this row is + -- written -- a reply can arrive before the message it answers, and the target can later be + -- deleted or pruned -- so a resolved id would be a second record needing a fix-up pass to stay + -- true. Resolution is done on read instead, through `messages_wire_key`, which exists for + -- exactly this. + -- + -- `reply_author` is an accounts FK rather than a session id so that resolution is a join. + -- + -- `reply_msgid` is the sender's `Content.msgId` for the target, and NULL from senders predating + -- that field. Match on timestamp *and* msgid where both are present; with it absent, timestamp + -- and author alone are ambiguous exactly when one sender stamped several messages in the same + -- millisecond, which is the case msgid exists to fix. Nothing can be done about that here, so + -- the read side takes the lowest matching id: arbitrary, but stable across reads. + -- + -- What the wire also carries and we deliberately do not store: the sender's own snippet of the + -- replied-to text and its attachments. Current clients do not populate them, and a sender can + -- forge them -- rendering one would put chosen words on screen attributed to someone else. We + -- render from our own copy of the target or from nothing. message_raw_content keeps the whole + -- protobuf, so this is recoverable if that is ever revisited. + reply_author INTEGER REFERENCES accounts(id), + reply_timestamp INTEGER, + reply_msgid INTEGER +) STRICT; + +-- Delivery is at-least-once, so a redelivered message must not duplicate. This also catches what +-- the swarm hash cannot: our own message arriving back from our swarm, which carries the same msgid +-- as the copy we sent. Both columns are needed -- see the protobuf comment for why msgid is not an +-- identifier on its own. +CREATE UNIQUE INDEX messages_msgid ON messages(conversation, timestamp, msgid); + +CREATE INDEX messages_history ON messages(conversation, timestamp DESC, id DESC); +CREATE INDEX messages_unread ON messages(conversation, timestamp) WHERE outgoing = 0; + +-- Quotes and reactions from clients that set no msgId address their target by author and timestamp +-- alone. Needed for as long as such clients exist. +CREATE INDEX messages_wire_key ON messages(conversation, sender, timestamp); + +-- The other direction of the same question: "which messages reply to this one". Asked whenever a +-- message changes, because a reply carries a copy of what it answers and so goes stale with it. +-- +-- Partial because most messages are not replies, which keeps it a fraction of the size of the row +-- count -- and because a partial index is only consulted for queries whose WHERE matches it, which +-- is precisely the lookup this serves. +CREATE INDEX messages_reply_target ON messages(conversation, reply_author, reply_timestamp) + WHERE reply_timestamp IS NOT NULL; + +-- `count` is a structural fact about the messages table, so triggers can own it outright: there is +-- no judgement involved in how many rows a conversation has. +-- +-- `unread_count` deliberately is *not* maintained here. What counts as unread is policy, not +-- structure -- mutes, message requests, tombstones, a message arriving with a timestamp older than +-- the read watermark -- and that policy will grow. Encoding it in triggers would scatter it +-- across SQL that is awkward to test and invisible from the code that decides it, so the +-- application maintains unread_count wherever it changes what has been read. +CREATE TRIGGER messages_insert AFTER INSERT ON messages +BEGIN + UPDATE conversations SET count = count + 1 WHERE id = NEW.conversation; +END; + +CREATE TRIGGER messages_delete AFTER DELETE ON messages +BEGIN + UPDATE conversations SET count = count - 1 WHERE id = OLD.conversation; +END; + +CREATE TRIGGER messages_move AFTER UPDATE OF conversation ON messages +WHEN OLD.conversation != NEW.conversation +BEGIN + UPDATE conversations SET count = count - 1 WHERE id = OLD.conversation; + UPDATE conversations SET count = count + 1 WHERE id = NEW.conversation; +END; + +-- The full decrypted Content protobuf, kept out of the messages table so that the history scan -- +-- the hot query -- does not drag it through overflow pages. Retained so fields this schema does +-- not yet model (attachments, quotes, reactions) can be recovered without re-fetching the swarm. +-- from 002_attachment_cache.sql +-- +-- An index over the files in the attachment cache directory, so that "how much disk is this using" +-- and "what has gone longest without being wanted" are queries rather than a directory walk on +-- every download. +-- +-- Deliberately only an index: the disk is what is actually true. A crash between writing a file +-- and recording it, or between unlinking one and forgetting it, leaves this describing a directory +-- that no longer matches -- so eviction checks what it is about to remove rather than trusting a +-- row, and the sweep reconciles in both directions: rows without files are dropped, files without +-- rows are adopted at their size on disk. +-- +-- `name` is the file's name, which is the hashed base url -- the same value `cache::path_for` +-- produces -- so a row can be matched to a file, and to a `message_attachments.url`, without +-- storing either the path or the url. +-- +-- `size` is bytes on disk, encrypted and padded, because that is what the cache limit is a limit +-- on. `last_used` is touched on a cache hit as well as on write, which is what makes eviction +-- least-recently-*used* rather than oldest-first: something opened weekly should not lose to +-- something downloaded once and never looked at again. +-- +-- Display pictures are not in here at all. They are never evicted -- a contact you have not spoken +-- to in years should not lose the last picture you had of them -- and are freed only when superseded, +-- which is a question about what still references them rather than about size or age. +CREATE TABLE attachment_cache ( + name TEXT PRIMARY KEY NOT NULL, + size INTEGER NOT NULL, + last_used INTEGER NOT NULL -- ms since epoch +) STRICT; + +CREATE INDEX attachment_cache_lru ON attachment_cache(last_used); + + +CREATE TABLE message_raw_content ( + message INTEGER PRIMARY KEY REFERENCES messages(id) ON DELETE CASCADE, + content BLOB NOT NULL +) STRICT; + +-- Files attached to a message, in either direction. +-- +-- The two directions are the same set of columns and differ only in which end is known first, so +-- they share the table; `messages.outgoing` says which reading applies. `path` is the local copy +-- and `url` the remote one, and exactly one of them exists from the start: +-- +-- outgoing -- `path` is the file we uploaded, and is required; `url IS NULL` means not uploaded +-- incoming -- `url` is where the file is, and is required; `path IS NULL` means not saved +-- +-- Sending with attachments is two stages: each file is encrypted and uploaded, and only then can +-- the message be built, since it has to name where those files ended up. A failure between the two +-- leaves a message that cannot be finished from the message alone -- what it needs is the local +-- files and whatever their uploads already achieved, neither of which the protobuf carries. +-- Retrying without that would mean re-uploading files that already arrived. +-- +-- There is no state column: for an outgoing attachment `url IS NULL` *is* "not uploaded yet". +-- Retrying is then "upload every row for this message that has no url, then build the message and +-- send it", which cannot disagree with itself the way a separate state could, and is safe to run +-- twice. +-- +-- An outgoing row's `path` outlives the upload deliberately. It is what makes an attachment we +-- sent displayable without fetching our own upload back, and it is the only record of it: what +-- persists of a sent attachment otherwise is the pointer inside message_raw_content. +CREATE TABLE message_attachments ( + message INTEGER NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + + -- Position within the message's attachment list, which is the order they appear in the + -- protobuf and how a progress report names one of them. + idx INTEGER NOT NULL, + + -- Where the file is, or came from: for an outgoing attachment the one we uploaded, for an + -- incoming one where the user asked us to save the download (NULL until they do). + -- + -- Informational rather than owning -- closer to a symlink than a handle. The file at the far + -- end belongs to the user in both directions: they chose it to attach, or they chose where to + -- put it. So it may be moved or deleted behind our back (only fatal while this row still needs + -- uploading), and nothing here ever unlinks it. + -- + -- A cache of our own -- attachments kept encrypted so they outlive the file server -- would be a + -- third thing, at a path we picked, and wants its own column: that is what a + -- delete-attachments-before instruction is entitled to remove, and sharing this column would + -- leave it unable to tell our copy from the user's. + -- + -- Such a cache cannot be freed by deleting files alongside the rows that name them, because the + -- rows go by cascade when a conversation is deleted and no code of ours runs. It wants the same + -- treatment as everything else here: compare and converge. List the cached paths still + -- referenced, list the cache directory, unlink the difference. That survives a cascade, and + -- also collects what a crash mid-download left behind -- which collecting paths before a delete + -- never would. It needs a directory we own outright, since "anything not in the list" is only + -- safe there, and it must not eat a download in flight: either the row exists before the + -- transfer starts, or the sweep skips the in-progress suffix. + path TEXT, + + -- Descriptive fields, carried straight through the protobuf pointer in both directions. + -- content_type is what a recipient displays by; the rest are optional and simply omitted when + -- null. On an incoming attachment these are the sender's claims and nothing more. + content_type TEXT, + filename TEXT, + caption TEXT, + flags INTEGER NOT NULL DEFAULT 0, + width INTEGER, + height INTEGER, + + -- Where the file is on the file server, the key it is encrypted with, and how long the file + -- itself is. + -- + -- The last of those is the smallest of three different sizes and none of the other two: what + -- the server stores is bigger (IV, MAC or stream framing), and what decryption yields is also + -- bigger, since Session pads to hide the true length on top of whatever the cipher pads. This + -- is what is left after both, and `legacy_decrypt` resizes down to it -- which is why it cannot + -- be either of the others and still do that job. + -- + -- A claim by the sender, believed by nothing: it is checked against what actually arrives, and + -- decryption throws if it exceeds that. + -- + -- Set together when an upload succeeds, or read together out of an arriving pointer. + url TEXT, + key BLOB, + size INTEGER, + + -- Legacy attachments authenticate with a separate SHA-256 digest over the ciphertext, and + -- carry a 64-byte `key` (AES key then HMAC key) rather than the 32-byte stream key. Null for + -- anything encrypted with the stream scheme, which authenticates each chunk as it goes. + digest BLOB, + + -- ms since epoch: when the *recipient* of this message last saved this attachment. On an + -- incoming attachment that is us, writing it to disk; on an outgoing one it is the other party, + -- telling us they saved it. One meaning in both directions -- "the recipient saved this, then". + -- + -- Not a path: where a file went is the application's business, and one recorded here would be + -- wrong as soon as it moved the file. This answers the question an application actually has, + -- which is whether offering "save" again is pointless. + -- + -- NULL means "not known to have been saved", never "not saved": the outgoing case depends on + -- the other end volunteering a DataExtractionNotification, which many clients do not. + saved_at INTEGER, + + PRIMARY KEY (message, idx) +) STRICT; diff --git a/src/config.cpp b/src/config.cpp index a8e2cf1fc..fd056172f 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -16,6 +15,7 @@ #include "config/internal.hpp" #include "session/bt_merge.hpp" +#include "session/hash.hpp" #include "session/util.hpp" using namespace std::literals; @@ -345,9 +345,8 @@ namespace { return std::string_view{reinterpret_cast(hash.data()), hash.size()}; } - hash_t& hash_msg(hash_t& into, std::span serialized) { - crypto_generichash_blake2b( - into.data(), into.size(), serialized.data(), serialized.size(), nullptr, 0); + hash_t& hash_msg(hash_t& into, std::span serialized) { + hash::blake2b(into, serialized); return into; } @@ -429,11 +428,11 @@ namespace { void verify_config_sig( oxenc::bt_dict_consumer dict, const ConfigMessage::verify_callable& verifier, - std::optional>* verified_signature, + std::optional* verified_signature, bool trust_signature) { if (dict.skip_until("~")) { dict.consume_signature( - [&](std::span to_verify, std::span sig) { + [&](std::span to_verify, std::span sig) { if (sig.size() != 64) throw signature_error{"Config signature is invalid (not 64B)"}; if (verifier && !verifier(to_verify, sig)) @@ -482,7 +481,7 @@ void MutableConfigMessage::increment_impl() { // Append the source config's diff to the new object lagged_diffs_.emplace_hint(lagged_diffs_.end(), seqno_hash_, std::move(diff_)); seqno_hash_.first++; - seqno_hash_.second.fill(0); // Not strictly necessary, but makes it obvious if used + seqno_hash_.second.fill(std::byte{0}); // Not strictly necessary, but makes it obvious if used diff_.clear(); } @@ -523,7 +522,7 @@ ConfigMessage::ConfigMessage() { } ConfigMessage::ConfigMessage( - std::span serialized, + std::span serialized, verify_callable verifier_, sign_callable signer_, int lag, @@ -561,7 +560,7 @@ ConfigMessage::ConfigMessage( } ConfigMessage::ConfigMessage( - const std::vector>& serialized_confs, + const std::vector>& serialized_confs, verify_callable verifier_, sign_callable signer_, int lag, @@ -691,7 +690,7 @@ ConfigMessage::ConfigMessage( } MutableConfigMessage::MutableConfigMessage( - const std::vector>& serialized_confs, + const std::vector>& serialized_confs, verify_callable verifier, sign_callable signer, int lag, @@ -707,7 +706,7 @@ MutableConfigMessage::MutableConfigMessage( } MutableConfigMessage::MutableConfigMessage( - std::span config, + std::span config, verify_callable verifier, sign_callable signer, int lag) : @@ -729,13 +728,13 @@ const oxenc::bt_dict& MutableConfigMessage::diff() { return diff_; } -std::vector ConfigMessage::serialize(bool enable_signing) { +std::vector ConfigMessage::serialize(bool enable_signing) { return serialize_impl( diff(), // implicitly prunes (if actually a mutable instance) enable_signing); } -std::vector ConfigMessage::serialize_impl( +std::vector ConfigMessage::serialize_impl( const oxenc::bt_dict& curr_diff, bool enable_signing) { oxenc::bt_dict_producer outer{}; @@ -776,7 +775,7 @@ std::vector ConfigMessage::serialize_impl( reinterpret_cast(verified_signature_->data()), verified_signature_->size()}); } else if (signer && enable_signing) { - outer.append_signature("~", [this](std::span to_sign) { + outer.append_signature("~", [this](std::span to_sign) { auto sig = signer(to_sign); if (sig.size() != 64) throw std::logic_error{ @@ -784,13 +783,13 @@ std::vector ConfigMessage::serialize_impl( return sig; }); } - return to_vector(outer.view()); + return to_vector(outer.view()); } const hash_t& MutableConfigMessage::hash() { return hash(serialize()); } -const hash_t& MutableConfigMessage::hash(std::span serialized) { +const hash_t& MutableConfigMessage::hash(std::span serialized) { return hash_msg(seqno_hash_.second, serialized); } diff --git a/src/config/base.cpp b/src/config/base.cpp index 8cc503d7d..5fdece3d8 100644 --- a/src/config/base.cpp +++ b/src/config/base.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -20,10 +19,12 @@ #include "internal.hpp" #include "oxenc/bt_serialize.h" +#include "session/clock.hpp" #include "session/config/base.h" #include "session/config/encrypt.hpp" #include "session/config/protos.hpp" #include "session/export.h" +#include "session/hash.hpp" #include "session/util.hpp" using namespace std::literals; @@ -70,8 +71,8 @@ std::unique_ptr make_config_message(bool from_dirty, Args&&... ar } std::unordered_set ConfigBase::merge( - const std::vector>>& configs) { - std::vector>> config_views; + const std::vector>>& configs) { + std::vector>> config_views; config_views.reserve(configs.size()); for (auto& [hash, data] : configs) config_views.emplace_back(hash, data); @@ -79,28 +80,22 @@ std::unordered_set ConfigBase::merge( } std::unordered_set ConfigBase::merge( - const std::vector>>& configs) { + const std::vector>>& configs) { if (accepts_protobuf() && !_keys.empty()) { - std::list> keep_alive; - std::vector>> parsed; + std::list> keep_alive; + std::vector>> parsed; parsed.reserve(configs.size()); for (auto& [h, c] : configs) { try { - auto unwrapped = protos::unwrap_config( - std::span{_keys.front().data(), _keys.front().size()}, - c, - storage_namespace()); + auto unwrapped = protos::unwrap_config(_keys.front(), c, storage_namespace()); // There was a release of one of the clients which resulted in double-wrapped // config messages so we now need to try to double-unwrap in order to better // support multi-device for users running those old versions try { - auto unwrapped2 = protos::unwrap_config( - std::span{ - _keys.front().data(), _keys.front().size()}, - unwrapped, - storage_namespace()); + auto unwrapped2 = + protos::unwrap_config(_keys.front(), unwrapped, storage_namespace()); log::warning( cat, "Found double wraped message in namespace {}", @@ -120,9 +115,9 @@ std::unordered_set ConfigBase::merge( return _merge(configs); } -std::pair, std::vector>>> -ConfigBase::_handle_multipart(std::string_view msg_id, std::span message) { - assert(!message.empty() && message[0] == 'm'); +std::pair, std::vector>>> +ConfigBase::_handle_multipart(std::string_view msg_id, std::span message) { + assert(!message.empty() && message[0] == std::byte{'m'}); // Handle multipart messages. Each part of a multipart message starts with `m` and then is // immediately followed by a bt_list where: @@ -139,7 +134,7 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span()}; - auto h = c.consume>(); + auto h = c.consume>(); hash_t final_hash; if (h.size() != final_hash.size()) throw std::runtime_error{"Invalid multi-part final message hash"}; @@ -155,7 +150,7 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span(); + auto data = c.consume_span(); if (data.empty()) throw std::runtime_error{"Invalid multi-part message with empty data"}; @@ -200,7 +195,7 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span, std::vector> result{}; + std::pair, std::vector> result{}; auto& [msgids, recombined] = result; size_t final_size = 0; @@ -213,13 +208,11 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span(recombined); if (actual_hash != final_hash) throw std::runtime_error{ "recombined message hash ({}) does not match part hash ({})"_format( - oxenc::to_hex(actual_hash.begin(), actual_hash.end()), - oxenc::to_hex(final_hash.begin(), final_hash.end()))}; + actual_hash, final_hash)}; } log::debug( @@ -228,7 +221,7 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span{ + if (recombined[0] == std::byte{'z'}) { + if (auto decompressed = zstd_decompress(std::span{ recombined.data() + 1, recombined.size() - 1}); decompressed && !decompressed->empty()) { log::debug( cat, "multipart message {} inflated to {}B plaintext from {}B compressed", - oxenc::to_hex(final_hash.begin(), final_hash.end()), + final_hash, decompressed->size(), recombined.size()); recombined = std::move(*decompressed); } else throw std::runtime_error{ "Invalid recombined data (hash {}): decompression failed"_format( - oxenc::to_hex(final_hash.begin(), final_hash.end()), msg_id)}; + final_hash, msg_id)}; } if (recombined.empty()) throw std::runtime_error{"recombined data is empty"}; - if (recombined[0] != 'd') + if (recombined[0] != std::byte{'d'}) throw std::runtime_error{"Recombined data has invalid/unsupported type {:?}"_format( static_cast(recombined[0]))}; return {true, std::move(result)}; } else { - parts.expiry = std::chrono::system_clock::now() + MULTIPART_MAX_WAIT; + parts.expiry = clock_now() + MULTIPART_MAX_WAIT; log::debug( cat, "message {} (part {} of {}) stored without completing a multipart set for {}", msg_id, index, parts.size, - oxenc::to_hex(final_hash.begin(), final_hash.end())); + final_hash); return {true, std::nullopt}; } @@ -288,7 +281,7 @@ ConfigBase::_handle_multipart(std::string_view msg_id, std::span(); auto msgid = pd.consume_string_view(); - auto chunk = pd.consume_span(); + auto chunk = pd.consume_span(); pm.parts.emplace_back(index, msgid, chunk); } } @@ -364,14 +357,14 @@ void ConfigBase::_load_multiparts(oxenc::bt_dict_consumer&& multi) { } std::unordered_set ConfigBase::_merge( - std::span>> configs) { + std::span>> configs) { if (_keys.empty()) throw std::logic_error{"Cannot merge configs without any decryption keys"}; const auto old_seqno = _config->seqno(); std::vector> all_hashes; // >1 hashes for multipart configs - std::vector> all_confs; + std::vector> all_confs; all_hashes.reserve(configs.size() + 1); all_confs.reserve(configs.size() + 1); @@ -396,7 +389,7 @@ std::unordered_set ConfigBase::_merge( // at the end (rather than the beginning) so that it is identical to one of the incoming // messages, *that* one becomes the config superset rather than our current, hash-unknown value. - std::vector mine; + std::vector mine; bool mine_last = false; if (old_seqno != 0 || is_dirty()) { mine = _config->serialize(); @@ -409,7 +402,7 @@ std::unordered_set ConfigBase::_merge( } } - std::vector>> plaintexts; + std::vector>> plaintexts; std::unordered_set good_hashes; @@ -440,7 +433,7 @@ std::unordered_set ConfigBase::_merge( for (auto& [hash, plain] : plaintexts) { // Remove prefix padding: if (auto it = std::find_if( - plain.begin(), plain.end(), [](unsigned char c) { return c != 0; }); + plain.begin(), plain.end(), [](std::byte c) { return c != std::byte{0}; }); it != plain.begin() && it != plain.end()) { auto p = std::distance(plain.begin(), it); std::memmove(plain.data(), plain.data() + p, plain.size() - p); @@ -451,7 +444,7 @@ std::unordered_set ConfigBase::_merge( continue; } - bool was_multipart = plain[0] == 'm'; + bool was_multipart = plain[0] == std::byte{'m'}; if (was_multipart) { // Multipart message @@ -471,11 +464,11 @@ std::unordered_set ConfigBase::_merge( } // Single-part message - bool was_compressed = plain[0] == 'z'; + bool was_compressed = plain[0] == std::byte{'z'}; if (was_compressed) { // zstd-compressed data if (auto decompressed = zstd_decompress( - std::span{plain.data() + 1, plain.size() - 1}); + std::span{plain.data() + 1, plain.size() - 1}); decompressed && !decompressed->empty()) plain = std::move(*decompressed); else { @@ -484,7 +477,7 @@ std::unordered_set ConfigBase::_merge( } } - if (plain[0] != 'd') { + if (plain[0] != std::byte{'d'}) { log::error( cat, "invalid/unsupported config message with type {:?}", @@ -658,6 +651,10 @@ std::unordered_set ConfigBase::_merge( return good_hashes; } +seqno_t ConfigBase::seqno() const { + return _config->seqno(); +} + const std::unordered_set& ConfigBase::curr_hashes() const { return _curr_hashes; } @@ -666,7 +663,7 @@ std::unordered_set ConfigBase::active_hashes() const { // First copy any hashes that make up the currently active config: std::unordered_set hashes{_curr_hashes}; - auto now = std::chrono::system_clock::now(); + auto now = clock_now(); // Add include any pending partial configs that *might* be newer: for (const auto& [_, part] : _multiparts) if (!part.done && part.expiry > now) @@ -691,27 +688,23 @@ bool ConfigBase::needs_push() const { return !is_clean(); } -// Tries to compresses the message; if the compressed version (including the 'z' prefix tag) is -// smaller than the source message then we modify `msg` to contain the 'z'-prefixed compressed -// message, otherwise we leave it as-is. Returns true if compression was beneficial and `msg` has -// been compressed; false if compression did not reduce the size and msg was left as-is. -void compress_message(std::vector& msg, int level) { +void compress_message(std::vector& msg, int level) { if (!level) return; // "z" is our zstd compression marker prefix byte - std::vector compressed = zstd_compress(msg, level, to_span("z")); + std::vector compressed = zstd_compress(msg, level, to_span("z")); if (compressed.size() < msg.size()) msg = std::move(compressed); } -std::tuple>, std::vector> +std::tuple>, std::vector> ConfigBase::push() { if (_keys.empty()) throw std::logic_error{"Cannot push data without an encryption key!"}; auto s = _config->seqno(); - std::tuple>, std::vector> ret{ + std::tuple>, std::vector> ret{ s, {}, {}}; auto& [seqno, msgs, obs] = ret; @@ -738,8 +731,7 @@ ConfigBase::push() { // - element 3 is the chunk of data (and so, when ordered by sequence number, each data // chunk // concatenated together gives us the `msg` value we have right now in this function). - hash_t final_hash; - hash::hash(final_hash, msg); + auto final_hash = hash::blake2b<32>(msg); constexpr size_t ENCODE_OVERHEAD = 1 // The `m` prefix indicating a multipart message part @@ -764,17 +756,17 @@ ConfigBase::push() { cat, "splitting large config message ({}B, hash {}) into {} parts", msg.size(), - oxenc::to_hex(final_hash.begin(), final_hash.end()), + final_hash, num_parts); - std::span remaining{msg}; + std::span remaining{msg}; for (uint8_t index = 0; !remaining.empty(); ++index) { auto& out = msgs.emplace_back(); auto chunk = remaining.subspan(0, std::min(MAX_CHUNK_SIZE, remaining.size())); remaining = remaining.subspan(chunk.size()); out.reserve(chunk.size() + ENCODE_OVERHEAD + ENCRYPT_DATA_OVERHEAD); out.resize(chunk.size() + ENCODE_OVERHEAD); - out[0] = 'm'; + out[0] = std::byte{'m'}; { oxenc::bt_list_producer lp{reinterpret_cast(out.data() + 1), out.size() - 1}; lp.append(std::span{final_hash}); @@ -800,8 +792,7 @@ ConfigBase::push() { encrypt_inplace(msg, key(), encryption_domain()); if (accepts_protobuf() && !_keys.empty()) { - auto pbwrapped = protos::wrap_config( - {_keys.front().data(), _keys.front().size()}, msg, s, storage_namespace()); + auto pbwrapped = protos::wrap_config(_keys.front(), msg, s, storage_namespace()); // If protobuf wrapping would push us *over* the max message size then we just skip the // protobuf wrapping because older clients (that need protobuf) also don't support // multipart anyway, so we can't produce a message they will accept no matter what. @@ -835,7 +826,7 @@ void ConfigBase::confirm_pushed(seqno_t seqno, std::unordered_set m } } -std::vector ConfigBase::dump() { +std::vector ConfigBase::dump() { if (is_readonly()) _old_hashes.clear(); @@ -846,7 +837,7 @@ std::vector ConfigBase::dump() { return d; } -std::vector ConfigBase::make_dump() const { +std::vector ConfigBase::make_dump() const { auto data = _config->serialize(false /* disable signing for local storage */); auto data_sv = to_string_view(data); oxenc::bt_list old_hashes; @@ -866,9 +857,9 @@ std::vector ConfigBase::make_dump() const { } ConfigBase::ConfigBase( - std::optional> dump, - std::optional> ed25519_pubkey, - std::optional> ed25519_secretkey) { + std::optional> dump, + std::optional> ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey) { if (sodium_init() == -1) throw std::runtime_error{"libsodium initialization failed!"}; @@ -877,11 +868,10 @@ ConfigBase::ConfigBase( } void ConfigSig::init_sig_keys( - std::optional> ed25519_pubkey, - std::optional> ed25519_secretkey) { + std::optional> ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey) { if (ed25519_secretkey) { - if (ed25519_pubkey && - to_string_view(*ed25519_pubkey) != to_string_view(ed25519_secretkey->subspan(32))) + if (ed25519_pubkey && !std::ranges::equal(*ed25519_pubkey, ed25519_secretkey->pubkey())) throw std::invalid_argument{"Invalid signing keys: secret key and pubkey do not match"}; set_sig_keys(*ed25519_secretkey); } else if (ed25519_pubkey) { @@ -892,9 +882,9 @@ void ConfigSig::init_sig_keys( } void ConfigBase::init( - std::optional> dump, - std::optional> ed25519_pubkey, - std::optional> ed25519_secretkey) { + std::optional> dump, + std::optional> ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey) { if (!dump) { _state = ConfigState::Clean; _config = std::make_unique(); @@ -962,10 +952,7 @@ int ConfigBase::key_count() const { return _keys.size(); } -bool ConfigBase::has_key(std::span key) const { - if (key.size() != 32) - throw std::invalid_argument{"invalid key given to has_key(): not 32-bytes"}; - +bool ConfigBase::has_key(std::span key) const { auto* keyptr = key.data(); for (const auto& key : _keys) if (sodium_memcmp(keyptr, key.data(), KEY_SIZE) == 0) @@ -973,22 +960,19 @@ bool ConfigBase::has_key(std::span key) const { return false; } -std::vector> ConfigBase::get_keys() const { - std::vector> ret; +std::vector> ConfigBase::get_keys() const { + std::vector> ret; ret.reserve(_keys.size()); for (const auto& key : _keys) - ret.emplace_back(key.data(), key.size()); + ret.emplace_back(key); return ret; } void ConfigBase::add_key( - std::span key, bool high_priority, bool dirty_config) { + std::span key, bool high_priority, bool dirty_config) { static_assert( sizeof(Key) == KEY_SIZE, "std::array appears to have some overhead which seems bad"); - if (key.size() != KEY_SIZE) - throw std::invalid_argument{"add_key failed: key size must be 32 bytes"}; - if (!_keys.empty() && sodium_memcmp(_keys.front().data(), key.data(), KEY_SIZE) == 0) return; else if (!high_priority && has_key(key)) @@ -1021,7 +1005,7 @@ int ConfigBase::clear_keys(bool dirty_config) { } void ConfigBase::replace_keys( - const std::vector>& new_keys, bool dirty_config) { + const std::vector>& new_keys, bool dirty_config) { if (new_keys.empty()) { if (_keys.empty()) return; @@ -1029,10 +1013,6 @@ void ConfigBase::replace_keys( return; } - for (auto& k : new_keys) - if (k.size() != KEY_SIZE) - throw std::invalid_argument{"replace_keys failed: keys must be 32 bytes"}; - dirty_config = dirty_config && !is_readonly() && (_keys.empty() || sodium_memcmp(_keys.front().data(), new_keys.front().data(), KEY_SIZE) != 0); @@ -1046,7 +1026,7 @@ void ConfigBase::replace_keys( dirty(); } -bool ConfigBase::remove_key(std::span key, size_t from, bool dirty_config) { +bool ConfigBase::remove_key(std::span key, size_t from, bool dirty_config) { auto starting_size = _keys.size(); if (from >= starting_size) return false; @@ -1069,52 +1049,36 @@ bool ConfigBase::remove_key(std::span key, size_t from, boo return _keys.size() < starting_size; } -void ConfigBase::load_key(std::span ed25519_secretkey) { - if (!(ed25519_secretkey.size() == 64 || ed25519_secretkey.size() == 32)) - throw std::invalid_argument{ - encryption_domain() + " requires an Ed25519 64-byte secret key or 32-byte seed"s}; - - add_key(ed25519_secretkey.subspan(0, 32)); +void ConfigBase::load_key(const ed25519::PrivKeySpan& ed25519_secretkey) { + add_key(ed25519_secretkey.seed()); } -void ConfigSig::set_sig_keys(std::span secret) { - if (secret.size() != 64) - throw std::invalid_argument{"Invalid sodium secret: expected 64 bytes"}; +void ConfigSig::set_sig_keys(const ed25519::PrivKeySpan& secret) { clear_sig_keys(); - _sign_sk.reset(64); - std::memcpy(_sign_sk.data(), secret.data(), secret.size()); - _sign_pk.emplace(); - crypto_sign_ed25519_sk_to_pk(_sign_pk->data(), _sign_sk.data()); - - set_verifier([this](std::span data, std::span sig) { - return 0 == crypto_sign_ed25519_verify_detached( - sig.data(), data.data(), data.size(), _sign_pk->data()); + _sign_sk.assign(secret.begin(), secret.end()); + ed25519::sk_to_pk(_sign_pk.emplace(), secret); + + set_verifier([this](std::span data, std::span sig) { + return sig.size() == 64 && ed25519::verify(sig.first<64>(), *_sign_pk, data); }); - set_signer([this](std::span data) { - std::vector sig; - sig.resize(64); - if (0 != crypto_sign_ed25519_detached( - sig.data(), nullptr, data.data(), data.size(), _sign_sk.data())) - throw std::runtime_error{"Internal error: config signing failed!"}; - return sig; + set_signer([this](std::span data) { + ed25519::PrivKeySpan sk{std::span{_sign_sk.data(), 64}}; + auto sig = ed25519::sign(sk, data); + return std::vector{sig.begin(), sig.end()}; }); } -void ConfigSig::set_sig_pubkey(std::span pubkey) { - if (pubkey.size() != 32) - throw std::invalid_argument{"Invalid pubkey: expected 32 bytes"}; - _sign_pk.emplace(); - std::memcpy(_sign_pk->data(), pubkey.data(), 32); +void ConfigSig::set_sig_pubkey(std::span pubkey) { + std::ranges::copy(pubkey, _sign_pk.emplace().begin()); - set_verifier([this](std::span data, std::span sig) { - return 0 == crypto_sign_ed25519_verify_detached( - sig.data(), data.data(), data.size(), _sign_pk->data()); + set_verifier([this](std::span data, std::span sig) { + return sig.size() == 64 && ed25519::verify(sig.first<64>(), *_sign_pk, data); }); } void ConfigSig::clear_sig_keys() { _sign_pk.reset(); - _sign_sk.reset(); + _sign_sk.clear(); set_signer(nullptr); set_verifier(nullptr); } @@ -1127,25 +1091,23 @@ void ConfigBase::set_signer(ConfigMessage::sign_callable s) { _config->signer = std::move(s); } -std::array ConfigSig::seed_hash(std::string_view key) const { - if (!_sign_sk) +cleared_b32 ConfigSig::seed_hash(std::string_view key) const { + if (_sign_sk.empty()) throw std::runtime_error{"Cannot make a seed hash without a signing secret key"}; - std::array out; - crypto_generichash_blake2b( - out.data(), - out.size(), - _sign_sk.data(), - 32, // Just the seed part of the value, not the last half (which is just the pubkey) - reinterpret_cast(key.data()), - std::min(key.size(), 64)); - return out; + cleared_b32 result; + hash::blake2b_key(result, key, std::span{_sign_sk.data(), 32}); + return result; } -void set_error(config_object* conf, std::string e) { - auto& error = unbox(conf).error; - error = std::move(e); - conf->last_error = error.c_str(); -} +namespace { + + void set_error(config_object* conf, std::string e) { + auto& error = unbox(conf).error; + error = std::move(e); + conf->last_error = error.c_str(); + } + +} // namespace } // namespace session::config @@ -1171,11 +1133,10 @@ LIBSESSION_EXPORT config_string_list* config_merge( size_t count) { return wrap_exceptions(conf, [&] { auto& config = *unbox(conf); - std::vector>> confs; + std::vector>> confs; confs.reserve(count); for (size_t i = 0; i < count; i++) - confs.emplace_back( - msg_hashes[i], std::span{configs[i], lengths[i]}); + confs.emplace_back(msg_hashes[i], to_byte_span(configs[i], lengths[i])); return make_string_list(config.merge(confs)); }); @@ -1323,7 +1284,7 @@ LIBSESSION_EXPORT bool config_add_key(config_object* conf, const unsigned char* return wrap_exceptions( conf, [&] { - unbox(conf)->add_key({key, 32}); + unbox(conf)->add_key(to_byte_span<32>(key)); return true; }, false); @@ -1333,7 +1294,7 @@ LIBSESSION_EXPORT bool config_add_key_low_prio(config_object* conf, const unsign return wrap_exceptions( conf, [&] { - unbox(conf)->add_key({key, 32}, /*high_priority=*/false); + unbox(conf)->add_key(to_byte_span<32>(key), /*high_priority=*/false); return true; }, false); @@ -1342,20 +1303,20 @@ LIBSESSION_EXPORT int config_clear_keys(config_object* conf) { return unbox(conf)->clear_keys(); } LIBSESSION_EXPORT bool config_remove_key(config_object* conf, const unsigned char* key) { - return unbox(conf)->remove_key({key, 32}); + return unbox(conf)->remove_key(to_byte_span<32>(key)); } LIBSESSION_EXPORT int config_key_count(const config_object* conf) { return unbox(conf)->key_count(); } LIBSESSION_EXPORT bool config_has_key(const config_object* conf, const unsigned char* key) { try { - return unbox(conf)->has_key({key, 32}); + return unbox(conf)->has_key(to_byte_span<32>(key)); } catch (...) { return false; } } LIBSESSION_EXPORT const unsigned char* config_key(const config_object* conf, size_t i) { - return unbox(conf)->key(i).data(); + return to_unsigned(unbox(conf)->key(i).data()); } LIBSESSION_EXPORT const char* config_encryption_domain(const config_object* conf) { @@ -1366,7 +1327,7 @@ LIBSESSION_EXPORT bool config_set_sig_keys(config_object* conf, const unsigned c return wrap_exceptions( conf, [&] { - unbox(conf)->set_sig_keys({secret, 64}); + unbox(conf)->set_sig_keys(ed25519::PrivKeySpan{secret, 64}); return true; }, false); @@ -1376,7 +1337,7 @@ LIBSESSION_EXPORT bool config_set_sig_pubkey(config_object* conf, const unsigned return wrap_exceptions( conf, [&] { - unbox(conf)->set_sig_pubkey({pubkey, 32}); + unbox(conf)->set_sig_pubkey(to_byte_span<32>(pubkey)); return true; }, false); @@ -1385,7 +1346,7 @@ LIBSESSION_EXPORT bool config_set_sig_pubkey(config_object* conf, const unsigned LIBSESSION_EXPORT const unsigned char* config_get_sig_pubkey(const config_object* conf) { const auto& pk = unbox(conf)->get_sig_pubkey(); if (pk) - return pk->data(); + return to_unsigned(pk->data()); return nullptr; } diff --git a/src/config/community.cpp b/src/config/community.cpp index 1832f6f6a..fb4703a5a 100644 --- a/src/config/community.cpp +++ b/src/config/community.cpp @@ -4,14 +4,13 @@ #include #include +#include #include #include #include #include #include "internal.hpp" -#include "oxenc/base32z.h" -#include "oxenc/base64.h" #include "session/config/community.h" #include "session/export.h" #include "session/util.hpp" @@ -24,7 +23,7 @@ community::community(std::string_view base_url_, std::string_view room_) { } community::community( - std::string_view base_url, std::string_view room, std::span pubkey_) : + std::string_view base_url, std::string_view room, std::span pubkey_) : community{base_url, room} { set_pubkey(pubkey_); } @@ -46,9 +45,7 @@ void community::set_base_url(std::string_view new_url) { base_url_ = canonical_url(new_url); } -void community::set_pubkey(std::span pubkey) { - if (pubkey.size() != 32) - throw std::invalid_argument{"Invalid pubkey: expected a 32-byte pubkey"}; +void community::set_pubkey(std::span pubkey) { pubkey_.assign(pubkey.begin(), pubkey.end()); } void community::set_pubkey(std::string_view pubkey) { @@ -56,18 +53,15 @@ void community::set_pubkey(std::string_view pubkey) { } std::string community::pubkey_hex() const { - const auto& pk = pubkey(); - return oxenc::to_hex(pk.begin(), pk.end()); + return "{:x}"_format(pubkey()); } std::string community::pubkey_b32z() const { - const auto& pk = pubkey(); - return oxenc::to_base32z(pk.begin(), pk.end()); + return "{:a}"_format(pubkey()); } std::string community::pubkey_b64() const { - const auto& pk = pubkey(); - return oxenc::to_base64(pk.begin(), pk.end()); + return "{:b}"_format(pubkey()); } void community::set_room(std::string_view room) { @@ -80,13 +74,8 @@ std::string community::full_url() const { } std::string community::full_url( - std::string_view base_url, std::string_view room, std::span pubkey) { - std::string url{base_url}; - url += '/'; - url += room; - url += qs_pubkey; - url += oxenc::to_hex(pubkey); - return url; + std::string_view base_url, std::string_view room, std::span pubkey) { + return "{}/{}?public_key={:x}"_format(base_url, room, pubkey); } void community::canonicalize_url(std::string& url) { @@ -112,10 +101,8 @@ std::string community::canonical_url(std::string_view url) { std::string result; result += proto; result += host; - if (port) { - result += ':'; - result += std::to_string(*port); - } + if (port) + fmt::format_to(std::back_inserter(result), ":{}", *port); // We don't (currently) allow a /path in a community URL if (path) throw std::invalid_argument{"Invalid community URL: found unexpected trailing value"}; @@ -130,9 +117,9 @@ std::string community::canonical_room(std::string_view room) { return r; } -std::tuple>> +std::tuple>> community::parse_partial_url(std::string_view url) { - std::tuple>> result; + std::tuple>> result; auto& [base_url, room_token, maybe_pubkey] = result; // Consume the URL from back to front; first the public key: @@ -156,7 +143,7 @@ community::parse_partial_url(std::string_view url) { return result; } -std::tuple> community::parse_full_url( +std::tuple> community::parse_full_url( std::string_view full_url) { auto [base, rm, maybe_pk] = parse_partial_url(full_url); if (!maybe_pk) @@ -216,7 +203,9 @@ LIBSESSION_C_API bool community_parse_partial_url( LIBSESSION_C_API void community_make_full_url( const char* base_url, const char* room, const unsigned char* pubkey, char* full_url) { auto full = session::config::community::full_url( - base_url, room, std::span{pubkey, 32}); + base_url, + room, + std::span{reinterpret_cast(pubkey), 32}); assert(full.size() <= COMMUNITY_FULL_URL_MAX_LENGTH); std::memcpy(full_url, full.data(), full.size() + 1); } diff --git a/src/config/contacts.cpp b/src/config/contacts.cpp index 6fee7c956..be8b53309 100644 --- a/src/config/contacts.cpp +++ b/src/config/contacts.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -59,8 +58,8 @@ void contact_info::set_nickname_truncated(std::string n) { } Contacts::Contacts( - std::span ed25519_secretkey, - std::optional> dumped) { + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, std::nullopt, std::nullopt); load_key(ed25519_secretkey); } @@ -79,6 +78,8 @@ void contact_info::load(const dict& info_dict) { } profile_updated = ts_or_epoch(info_dict, "t"); + delete_before = ts_or_epoch(info_dict, "d"); + delete_attach_before = ts_or_epoch(info_dict, "D"); approved = int_or_0(info_dict, "a"); approved_me = int_or_0(info_dict, "A"); blocked = int_or_0(info_dict, "b"); @@ -116,9 +117,9 @@ void contact_info::load(const dict& info_dict) { created = to_epoch_seconds(int_or_0(info_dict, "j")); - const session::config::set* profile_bitset_set = maybe_set(info_dict, "f"); - if (profile_bitset_set) - profile_bitset.data = bitset_from_set_of_int64_or_0(*profile_bitset_set); + const session::config::set* profile_flags_set = maybe_set(info_dict, "f"); + if (profile_flags_set) + profile_flags = to_flags(*profile_flags_set); } void contact_info::into(contacts_contact& c) const { @@ -143,7 +144,9 @@ void contact_info::into(contacts_contact& c) const { if (c.exp_seconds <= 0 && c.exp_mode != CONVO_EXPIRATION_NONE) c.exp_mode = CONVO_EXPIRATION_NONE; c.created = to_epoch_seconds(created); - c.profile_bitset.data = profile_bitset.data; + c.delete_before = epoch_seconds(delete_before); + c.delete_attach_before = epoch_seconds(delete_attach_before); + c.profile_bitset = static_cast(profile_flags); } contact_info::contact_info(const contacts_contact& c) : session_id{c.session_id, 66} { @@ -154,9 +157,13 @@ contact_info::contact_info(const contacts_contact& c) : session_id{c.session_id, assert(std::strlen(c.profile_pic.url) <= profile_pic::MAX_URL_LENGTH); if (std::strlen(c.profile_pic.url)) { profile_picture.url = c.profile_pic.url; - profile_picture.key.assign(c.profile_pic.key, c.profile_pic.key + 32); + profile_picture.key.assign( + reinterpret_cast(c.profile_pic.key), + reinterpret_cast(c.profile_pic.key) + 32); } profile_updated = to_sys_seconds(c.profile_updated); + delete_before = to_sys_seconds(c.delete_before); + delete_attach_before = to_sys_seconds(c.delete_attach_before); approved = c.approved; approved_me = c.approved_me; blocked = c.blocked; @@ -168,7 +175,7 @@ contact_info::contact_info(const contacts_contact& c) : session_id{c.session_id, if (exp_timer <= 0s && exp_mode != expiration_mode::none) exp_mode = expiration_mode::none; created = to_epoch_seconds(c.created); - profile_bitset.data = c.profile_bitset.data; + profile_flags = static_cast(c.profile_bitset); } std::optional Contacts::get(std::string_view pubkey_hex) const { @@ -207,6 +214,12 @@ void Contacts::set(const contact_info& contact) { contact.profile_picture.key); set_ts(info["t"], contact.profile_updated); + set_ts(info["d"], contact.delete_before); + // Deleting the messages takes their attachments with them, so an attachment instruction at or + // before that point says nothing further and is dropped rather than stored. + set_ts(info["D"], + contact.delete_attach_before <= contact.delete_before ? std::chrono::sys_seconds{} + : contact.delete_attach_before); set_flag(info["a"], contact.approved); set_flag(info["A"], contact.approved_me); @@ -228,7 +241,7 @@ void Contacts::set(const contact_info& contact) { contact.exp_timer.count()); set_positive_int(info["j"], to_epoch_seconds(contact.created)); - set_int64_set_from_bitset(info["f"], contact.profile_bitset.data); + set_flags(info["f"], contact.profile_flags); } void Contacts::set_name(std::string_view session_id, std::string name) { @@ -299,9 +312,9 @@ void Contacts::set_created(std::string_view session_id, int64_t timestamp) { set(c); } -void Contacts::set_pro_features(std::string_view session_id, ProProfileBitset features) { +void Contacts::set_pro_features(std::string_view session_id, ProProfileFlags features) { auto c = get_or_construct(session_id); - c.profile_bitset = features; + c.profile_flags = features; set(c); } @@ -321,7 +334,7 @@ size_t Contacts::size() const { blinded_contact_info::blinded_contact_info( std::string_view community_base_url, - std::span community_pubkey, + std::span community_pubkey, std::string_view blinded_id) : comm{community( std::move(community_base_url), blinded_id.substr(2), std::move(community_pubkey))} { @@ -331,7 +344,7 @@ blinded_contact_info::blinded_contact_info( if (prefix != session::SessionIDPrefix::community_blinded && prefix != session::SessionIDPrefix::community_blinded_legacy) throw std::invalid_argument{ - "Invalid blinded ID: Expected '15' or '25' prefix; got " + std::string{blinded_id}}; + "Invalid blinded ID: Expected '15' or '25' prefix; got {}"_format(blinded_id)}; } void blinded_contact_info::load(const dict& info_dict) { @@ -352,7 +365,7 @@ void blinded_contact_info::load(const dict& info_dict) { auto it = info_dict.find("f"); if (it != info_dict.end()) { if (auto* set = std::get_if(&it->second)) - profile_bitset.data = bitset_from_set_of_int64_or_0(*set); + profile_flags = to_flags(*set); } } @@ -374,23 +387,25 @@ void blinded_contact_info::into(contacts_blinded_contact& c) const { c.priority = priority; c.legacy_blinding = legacy_blinding; c.created = epoch_seconds(created); - c.profile_bitset.data = profile_bitset.data; + c.profile_bitset = static_cast(profile_flags); } blinded_contact_info::blinded_contact_info(const contacts_blinded_contact& c) { - comm = community(c.base_url, {c.session_id + 2, 64}, c.pubkey); + comm = community(c.base_url, {c.session_id + 2, 64}, std::as_bytes(std::span{c.pubkey})); assert(std::strlen(c.name) <= contact_info::MAX_NAME_LENGTH); name = c.name; assert(std::strlen(c.profile_pic.url) <= profile_pic::MAX_URL_LENGTH); if (std::strlen(c.profile_pic.url)) { profile_picture.url = c.profile_pic.url; - profile_picture.key.assign(c.profile_pic.key, c.profile_pic.key + 32); + profile_picture.key.assign( + reinterpret_cast(c.profile_pic.key), + reinterpret_cast(c.profile_pic.key) + 32); } profile_updated = to_sys_seconds(c.profile_updated); priority = c.priority; legacy_blinding = c.legacy_blinding; created = to_sys_seconds(c.created); - profile_bitset.data = c.profile_bitset.data; + profile_flags = static_cast(c.profile_bitset); } const std::string blinded_contact_info::session_id() const { @@ -416,7 +431,7 @@ void blinded_contact_info::set_room(std::string_view room) { comm.set_room(room); } -void blinded_contact_info::set_pubkey(std::span pubkey) { +void blinded_contact_info::set_pubkey(std::span pubkey) { comm.set_pubkey(pubkey); } @@ -425,13 +440,13 @@ void blinded_contact_info::set_pubkey(std::string_view pubkey) { } ConfigBase::DictFieldProxy Contacts::blinded_contact_field( - const blinded_contact_info& bc, std::span* get_pubkey) const { + const blinded_contact_info& bc, std::span* get_pubkey) const { auto record = data["b"][bc.comm.base_url()]; if (get_pubkey) { auto pkrec = record["#"]; if (auto pk = pkrec.string_view_or(""); pk.size() == 32) - *get_pubkey = std::span{ - reinterpret_cast(pk.data()), pk.size()}; + *get_pubkey = std::span{ + reinterpret_cast(pk.data()), pk.size()}; } return record["R"][bc.comm.room()]; // The `room` value is the blinded id without the prefix } @@ -464,8 +479,11 @@ blinded_contact_info Contacts::get_or_construct_blinded( if (auto maybe = get_blinded(blinded_id_hex)) return *std::move(maybe); + auto pk = oxenc::from_hex(community_pubkey_hex); return blinded_contact_info{ - community_base_url, to_span(oxenc::from_hex(community_pubkey_hex)), blinded_id_hex}; + community_base_url, + std::span{reinterpret_cast(pk.data()), 32}, + blinded_id_hex}; } std::vector Contacts::blinded() const { @@ -504,7 +522,7 @@ void Contacts::set_blinded(const blinded_contact_info& bc) { set_nonzero_int(info["+"], bc.priority); set_positive_int(info["y"], bc.legacy_blinding); set_ts(info["j"], bc.created); - set_int64_set_from_bitset(info["f"], bc.profile_bitset.data); + set_flags(info["f"], bc.profile_flags); } bool Contacts::erase_blinded(std::string_view base_url_, std::string_view blinded_id) { @@ -513,7 +531,7 @@ bool Contacts::erase_blinded(std::string_view base_url_, std::string_view blinde if (prefix != session::SessionIDPrefix::community_blinded && prefix != session::SessionIDPrefix::community_blinded_legacy) throw std::invalid_argument{ - "Invalid blinded ID: Expected '15' or '25' prefix; got " + std::string{blinded_id}}; + "Invalid blinded ID: Expected '15' or '25' prefix; got {}"_format(blinded_id)}; auto base_url = community::canonical_url(base_url_); auto pk = std::string(blinded_id.substr(2)); diff --git a/src/config/convo_info_volatile.cpp b/src/config/convo_info_volatile.cpp index aa47a789c..30ead1be0 100644 --- a/src/config/convo_info_volatile.cpp +++ b/src/config/convo_info_volatile.cpp @@ -3,15 +3,16 @@ #include #include #include -#include #include #include #include +#include #include #include #include "internal.hpp" +#include "session/clock.hpp" #include "session/config/convo_info_volatile.h" #include "session/config/error.h" #include "session/export.h" @@ -61,7 +62,7 @@ namespace convo { } community::community(const convo_info_volatile_community& c) : - config::community{c.base_url, c.room, std::span{c.pubkey, 32}}, + config::community{c.base_url, c.room, std::as_bytes(std::span{c.pubkey})}, base(c.last_read, c.unread) {} void community::into(convo_info_volatile_community& c) const { @@ -175,8 +176,8 @@ namespace convo { } // namespace convo ConvoInfoVolatile::ConvoInfoVolatile( - std::span ed25519_secretkey, - std::optional> dumped) { + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, std::nullopt, std::nullopt); load_key(ed25519_secretkey); } @@ -201,12 +202,12 @@ convo::one_to_one ConvoInfoVolatile::get_or_construct_1to1(std::string_view pubk } ConfigBase::DictFieldProxy ConvoInfoVolatile::community_field( - const convo::community& comm, std::span* get_pubkey) const { + const convo::community& comm, std::span* get_pubkey) const { auto record = data["o"][comm.base_url()]; if (get_pubkey) { auto pkrec = record["#"]; if (auto pk = pkrec.string_view_or(""); pk.size() == 32) - *get_pubkey = to_span(pk); + *get_pubkey = to_span(pk); } return record["R"][comm.room_norm()]; } @@ -215,11 +216,11 @@ std::optional ConvoInfoVolatile::get_community( std::string_view base_url, std::string_view room) const { convo::community og{base_url, community::canonical_room(room)}; - std::span pubkey; + std::span pubkey; if (auto* info_dict = community_field(og, &pubkey).dict()) { og.load(*info_dict); if (!pubkey.empty()) - og.set_pubkey(pubkey); + og.set_pubkey(pubkey.first<32>()); return og; } return std::nullopt; @@ -232,10 +233,8 @@ std::optional ConvoInfoVolatile::get_community( } convo::community ConvoInfoVolatile::get_or_construct_community( - std::string_view base_url, - std::string_view room, - std::span pubkey) const { - convo::community result{base_url, community::canonical_room(room), pubkey}; + std::string_view base_url, std::string_view room, std::span pubkey) const { + convo::community result{base_url, community::canonical_room(room), pubkey.first<32>()}; if (auto* info_dict = community_field(result).dict()) result.load(*info_dict); @@ -305,7 +304,7 @@ std::optional ConvoInfoVolatile::get_blinded_1to1( if (prefix != session::SessionIDPrefix::community_blinded && prefix != session::SessionIDPrefix::community_blinded_legacy) throw std::invalid_argument{ - "Invalid blinded ID: Expected '15' or '25' prefix; got " + std::string{pubkey_hex}}; + "Invalid blinded ID: Expected '15' or '25' prefix; got {}"_format(pubkey_hex)}; std::string pubkey = session_id_to_bytes(pubkey_hex, to_string(prefix)); @@ -333,7 +332,7 @@ void ConvoInfoVolatile::set(const convo::one_to_one& c) { auto pro_expiry = epoch_seconds(c.pro_expiry_at); if (pro_expiry > 0 && c.pro_revocation_tag) { set_nonzero_int(info["e"], pro_expiry); - info["g"] = *c.pro_revocation_tag; + info["g"] = to_span(*c.pro_revocation_tag); } } @@ -346,7 +345,7 @@ void ConvoInfoVolatile::set_base(const convo::base& c, DictFieldProxy& info) { r = c.last_read; else { std::chrono::system_clock::time_point last_read{std::chrono::milliseconds{c.last_read}}; - if (last_read > std::chrono::system_clock::now() - PRUNE_LOW) + if (last_read > clock_now() - PRUNE_LOW) info["r"] = c.last_read; } @@ -364,7 +363,7 @@ static bool is_stale(const C& c, std::chrono::system_clock::time_point cutoff) { } void ConvoInfoVolatile::prune_stale(std::chrono::milliseconds prune) { - const auto cutoff = std::chrono::system_clock::now() - prune; + const auto cutoff = clock_now() - prune; std::vector stale; for (auto it = begin_1to1(); it != end(); ++it) @@ -403,7 +402,7 @@ void ConvoInfoVolatile::prune_stale(std::chrono::milliseconds prune) { erase_community(base, room); } -std::tuple>, std::vector> +std::tuple>, std::vector> ConvoInfoVolatile::push() { // Prune off any conversations with last_read timestamps more than PRUNE_HIGH ago (unless they // also have a `unread` flag set, in which case we keep them indefinitely). @@ -439,7 +438,7 @@ void ConvoInfoVolatile::set(const convo::blinded_one_to_one& c) { auto pro_expiry = epoch_seconds(c.pro_expiry_at); if (pro_expiry > 0 && c.pro_revocation_tag) { set_nonzero_int(info["e"], pro_expiry); - info["g"] = *c.pro_revocation_tag; + info["g"] = to_span(*c.pro_revocation_tag); } } @@ -735,7 +734,10 @@ LIBSESSION_C_API bool convo_info_volatile_get_or_construct_community( [&] { unbox(conf) ->get_or_construct_community( - base_url, room, std::span{pubkey, 32}) + base_url, + room, + std::span{ + reinterpret_cast(pubkey), 32}) .into(*convo); return true; }, diff --git a/src/config/encrypt.cpp b/src/config/encrypt.cpp index 23a87cd9c..d5f029f59 100644 --- a/src/config/encrypt.cpp +++ b/src/config/encrypt.cpp @@ -2,12 +2,13 @@ #include #include -#include #include #include +#include "session/config/encrypt.h" #include "session/export.h" +#include "session/hash.hpp" #include "session/util.hpp" using namespace std::literals; @@ -28,7 +29,7 @@ static constexpr auto NONCE_KEY_PREFIX = "libsessionutil-config-encrypted-"sv; static_assert(NONCE_KEY_PREFIX.size() + DOMAIN_MAX_SIZE < crypto_generichash_blake2b_KEYBYTES_MAX); static std::array make_encrypt_key( - std::span key_base, uint64_t message_size, std::string_view domain) { + std::span key_base, uint64_t message_size, std::string_view domain) { if (key_base.size() != 32) throw std::invalid_argument{"encrypt called with key_base != 32 bytes"}; if (domain.size() < 1 || domain.size() > DOMAIN_MAX_SIZE) @@ -40,99 +41,100 @@ static std::array ma // nonce reuse concern so that you would not only have to hash collide but also have it happen // on messages of identical sizes and identical domain. std::array key{0}; - crypto_generichash_blake2b_state state; - crypto_generichash_blake2b_init(&state, nullptr, 0, key.size()); - crypto_generichash_blake2b_update(&state, key_base.data(), key_base.size()); oxenc::host_to_big_inplace(message_size); - crypto_generichash_blake2b_update( - &state, reinterpret_cast(&message_size), sizeof(message_size)); - crypto_generichash_blake2b_update(&state, to_unsigned(domain.data()), domain.size()); - crypto_generichash_blake2b_final(&state, key.data(), key.size()); + hash::blake2b( + key, + key_base, + std::span{reinterpret_cast(&message_size), sizeof(message_size)}, + to_span(domain)); return key; } -std::vector encrypt( - std::span message, - std::span key_base, +void encrypt_prealloced( + std::span message, + std::span key_base, std::string_view domain) { - std::vector msg; - msg.reserve(message.size() + ENCRYPT_DATA_OVERHEAD); - msg.assign(message.begin(), message.end()); - encrypt_inplace(msg, key_base, domain); - return msg; -} -void encrypt_inplace( - std::vector& message, - std::span key_base, - std::string_view domain) { - auto key = make_encrypt_key(key_base, message.size(), domain); + if (message.size() < ENCRYPT_DATA_OVERHEAD) + throw std::invalid_argument{ + "encrypt_prealloced: buffer is smaller than ENCRYPT_DATA_OVERHEAD"}; + auto plaintext = message.first(message.size() - ENCRYPT_DATA_OVERHEAD); + auto key = make_encrypt_key(key_base, plaintext.size(), domain); std::string nonce_key{NONCE_KEY_PREFIX}; nonce_key += domain; - std::array nonce; - crypto_generichash_blake2b( - nonce.data(), - nonce.size(), - message.data(), - message.size(), - to_unsigned(nonce_key.data()), - nonce_key.size()); - - size_t plaintext_len = message.size(); - message.resize(plaintext_len + ENCRYPT_DATA_OVERHEAD); + auto nonce = + hash::blake2b_key(nonce_key, plaintext); unsigned long long outlen = 0; crypto_aead_xchacha20poly1305_ietf_encrypt( - message.data(), + to_unsigned(message.data()), &outlen, - message.data(), - plaintext_len, + to_unsigned(plaintext.data()), + plaintext.size(), nullptr, 0, nullptr, - nonce.data(), + to_unsigned(nonce.data()), key.data()); - assert(outlen == message.size() - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - std::memcpy(message.data() + outlen, nonce.data(), nonce.size()); + assert(outlen == plaintext.size() + crypto_aead_xchacha20poly1305_ietf_ABYTES); + std::memcpy(to_unsigned(message.data()) + outlen, to_unsigned(nonce.data()), nonce.size()); +} + +std::vector encrypt( + std::span message, + std::span key_base, + std::string_view domain) { + std::vector out(message.size() + ENCRYPT_DATA_OVERHEAD); + std::memcpy(out.data(), message.data(), message.size()); + encrypt_prealloced(out, key_base, domain); + return out; +} + +void encrypt_inplace( + std::vector& message, + std::span key_base, + std::string_view domain) { + message.resize(message.size() + ENCRYPT_DATA_OVERHEAD); + encrypt_prealloced(message, key_base, domain); } static_assert( ENCRYPT_DATA_OVERHEAD == crypto_aead_xchacha20poly1305_IETF_ABYTES + crypto_aead_xchacha20poly1305_IETF_NPUBBYTES); -std::vector decrypt( - std::span ciphertext, - std::span key_base, +std::vector decrypt( + std::span ciphertext, + std::span key_base, std::string_view domain) { - std::vector x = session::to_vector(ciphertext); + auto x = session::to_vector(ciphertext); decrypt_inplace(x, key_base, domain); return x; } void decrypt_inplace( - std::vector& ciphertext, - std::span key_base, + std::vector& ciphertext, + std::span key_base, std::string_view domain) { size_t message_len = ciphertext.size() - ENCRYPT_DATA_OVERHEAD; if (message_len > ciphertext.size()) // overflow throw decrypt_error{"Decryption failed: ciphertext is too short"}; - std::span nonce = std::span{ciphertext}.subspan( + std::span nonce = std::span{ciphertext}.subspan( ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); auto key = make_encrypt_key(key_base, message_len, domain); unsigned long long mlen_wrote = 0; if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt( - ciphertext.data(), + to_unsigned(ciphertext.data()), &mlen_wrote, nullptr, - ciphertext.data(), + to_unsigned(ciphertext.data()), ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, nullptr, 0, - nonce.data(), + to_unsigned(nonce.data()), key.data())) throw decrypt_error{"Message decryption failed"}; @@ -140,10 +142,10 @@ void decrypt_inplace( ciphertext.resize(mlen_wrote); } -void pad_message(std::vector& data, size_t overhead) { +void pad_message(std::vector& data, size_t overhead) { size_t target_size = padded_size(data.size(), overhead); if (target_size > data.size()) - data.insert(data.begin(), target_size - data.size(), 0); + data.insert(data.begin(), target_size - data.size(), std::byte{0}); } } // namespace session::config @@ -157,9 +159,12 @@ LIBSESSION_EXPORT unsigned char* config_encrypt( const char* domain, size_t* ciphertext_size) { - std::vector ciphertext; + std::vector ciphertext; try { - ciphertext = session::config::encrypt({plaintext, len}, {key_base, 32}, domain); + ciphertext = session::config::encrypt( + std::span{reinterpret_cast(plaintext), len}, + std::span{reinterpret_cast(key_base), 32}, + domain); } catch (...) { return nullptr; } @@ -177,9 +182,12 @@ LIBSESSION_EXPORT unsigned char* config_decrypt( const char* domain, size_t* plaintext_size) { - std::vector plaintext; + std::vector plaintext; try { - plaintext = session::config::decrypt({ciphertext, clen}, {key_base, 32}, domain); + plaintext = session::config::decrypt( + std::span{reinterpret_cast(ciphertext), clen}, + std::span{reinterpret_cast(key_base), 32}, + domain); } catch (const std::exception& e) { return nullptr; } diff --git a/src/config/groups/info.cpp b/src/config/groups/info.cpp index 0ce34a036..5a567aef7 100644 --- a/src/config/groups/info.cpp +++ b/src/config/groups/info.cpp @@ -1,7 +1,6 @@ #include "session/config/groups/info.hpp" #include -#include #include @@ -17,10 +16,10 @@ using namespace std::literals; namespace session::config::groups { Info::Info( - std::span ed25519_pubkey, - std::optional> ed25519_secretkey, - std::optional> dumped) : - id{"03" + oxenc::to_hex(ed25519_pubkey.begin(), ed25519_pubkey.end())} { + std::span ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey, + std::optional> dumped) : + id{"03{:x}"_format(ed25519_pubkey)} { init(dumped, ed25519_pubkey, ed25519_secretkey); } @@ -62,12 +61,12 @@ profile_pic Info::get_profile_pic() const { pic.url = *url; if (auto* key = data["q"].string(); key && key->size() == 32) pic.key.assign( - reinterpret_cast(key->data()), - reinterpret_cast(key->data()) + 32); + reinterpret_cast(key->data()), + reinterpret_cast(key->data()) + 32); return pic; } -void Info::set_profile_pic(std::string_view url, std::span key) { +void Info::set_profile_pic(std::string_view url, std::span key) { set_pair_if(!url.empty() && key.size() == 32, data["p"], url, data["q"], key); } @@ -257,9 +256,9 @@ LIBSESSION_C_API user_profile_pic groups_info_get_pic(const config_object* conf) /// - `int` -- Returns 0 on success, non-zero on error LIBSESSION_C_API int groups_info_set_pic(config_object* conf, user_profile_pic pic) { std::string_view url{pic.url}; - std::span key; + std::span key; if (!url.empty()) - key = {pic.key, 32}; + key = {reinterpret_cast(pic.key), 32}; return wrap_exceptions( conf, diff --git a/src/config/groups/keys.cpp b/src/config/groups/keys.cpp index cc42ab8d3..161cd3649 100644 --- a/src/config/groups/keys.cpp +++ b/src/config/groups/keys.cpp @@ -3,25 +3,24 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include #include #include +#include "../../internal-util.hpp" #include "../internal.hpp" +#include "session/clock.hpp" #include "session/config/groups/info.hpp" #include "session/config/groups/keys.h" #include "session/config/groups/members.hpp" +#include "session/crypto/ed25519.hpp" +#include "session/encrypt.hpp" +#include "session/hash.hpp" #include "session/multi_encrypt.hpp" +#include "session/random.hpp" #include "session/session_encrypt.hpp" #include "session/xed25519.hpp" @@ -34,26 +33,19 @@ static auto sys_time_from_ms(int64_t milliseconds_since_epoch) { } Keys::Keys( - std::span user_ed25519_secretkey, - std::span group_ed25519_pubkey, - std::optional> group_ed25519_secretkey, - std::optional> dumped, + const ed25519::PrivKeySpan& user_ed25519_secretkey, + std::span group_ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& group_ed25519_secretkey, + std::optional> dumped, Info& info, Members& members) { if (sodium_init() == -1) throw std::runtime_error{"libsodium initialization failed!"}; - if (user_ed25519_secretkey.size() != 64) - throw std::invalid_argument{"Invalid Keys construction: invalid user ed25519 secret key"}; - if (group_ed25519_pubkey.size() != 32) - throw std::invalid_argument{"Invalid Keys construction: invalid group ed25519 public key"}; - if (group_ed25519_secretkey && group_ed25519_secretkey->size() != 64) - throw std::invalid_argument{"Invalid Keys construction: invalid group ed25519 secret key"}; - init_sig_keys(group_ed25519_pubkey, group_ed25519_secretkey); - user_ed25519_sk.load(user_ed25519_secretkey.data(), 64); + user_ed25519_sk.assign(user_ed25519_secretkey.begin(), user_ed25519_secretkey.end()); if (dumped) { load_dump(*dumped); @@ -67,14 +59,14 @@ bool Keys::needs_dump() const { return needs_dump_; } -std::vector Keys::dump() { +std::vector Keys::dump() { auto dumped = make_dump(); needs_dump_ = false; return dumped; } -std::vector Keys::make_dump() const { +std::vector Keys::make_dump() const { oxenc::bt_dict_producer d; { auto active = d.append_list("A"); @@ -108,7 +100,7 @@ std::vector Keys::make_dump() const { return to_vector(d.view()); } -void Keys::load_dump(std::span dump) { +void Keys::load_dump(std::span dump) { oxenc::bt_dict_consumer d{dump}; if (d.skip_until("A")) { @@ -138,8 +130,8 @@ void Keys::load_dump(std::span dump) { auto key_bytes = kd.consume_string_view(); if (key_bytes.size() != key.key.size()) throw config_value_error{ - "Invalid Keys dump: found key with invalid size (" + - std::to_string(key_bytes.size()) + ")"}; + "Invalid Keys dump: found key with invalid size ({})"_format( + key_bytes.size())}; std::memcpy(key.key.data(), key_bytes.data(), key.key.size()); if (!kd.skip_until("t")) @@ -172,8 +164,8 @@ void Keys::load_dump(std::span dump) { auto pk = pending.consume_string_view(); if (pk.size() != pending_key_.size()) throw config_value_error{ - "Invalid Keys dump: found pending key (k) with invalid size (" + - std::to_string(pk.size()) + ")"}; + "Invalid Keys dump: found pending key (k) with invalid size ({})"_format( + pk.size())}; std::memcpy(pending_key_.data(), pk.data(), pending_key_.size()); } } @@ -182,20 +174,20 @@ size_t Keys::size() const { return keys_.size() + !pending_key_config_.empty(); } -std::vector> Keys::group_keys() const { - std::vector> ret; +std::vector> Keys::group_keys() const { + std::vector> ret; ret.reserve(size()); if (!pending_key_config_.empty()) - ret.emplace_back(pending_key_.data(), 32); + ret.emplace_back(pending_key_); for (auto it = keys_.rbegin(); it != keys_.rend(); ++it) - ret.emplace_back(it->key.data(), 32); + ret.emplace_back(it->key); return ret; } -std::span Keys::group_enc_key() const { +std::span Keys::group_enc_key() const { if (!pending_key_config_.empty()) return {pending_key_.data(), 32}; if (keys_.empty()) @@ -205,50 +197,30 @@ std::span Keys::group_enc_key() const { return {key.data(), key.size()}; } -void Keys::load_admin_key(std::span seed, Info& info, Members& members) { +void Keys::load_admin_key(const ed25519::PrivKeySpan& secret, Info& info, Members& members) { if (admin()) return; - if (seed.size() == 64) - seed = seed.subspan(0, seed.size() - 32); - else if (seed.size() != 32) - throw std::invalid_argument{ - "Failed to load admin key: invalid secret key (expected 32 or 64 bytes)"}; - - std::array pk; - sodium_cleared> sk; - crypto_sign_ed25519_seed_keypair(pk.data(), sk.data(), seed.data()); - - if (_sign_pk.has_value() && *_sign_pk != pk) + if (_sign_pk && !std::ranges::equal(*_sign_pk, secret.pubkey())) throw std::runtime_error{ "Failed to load admin key: given secret key does not match group pubkey"}; - auto seckey = to_span(sk); - set_sig_keys(seckey); - info.set_sig_keys(seckey); - members.set_sig_keys(seckey); + set_sig_keys(secret); + info.set_sig_keys(secret); + members.set_sig_keys(secret); } namespace { - std::array compute_xpk(const unsigned char* ed25519_pk) { - std::array xpk; - if (0 != crypto_sign_ed25519_pk_to_curve25519(xpk.data(), ed25519_pk)) - throw std::runtime_error{ - "An error occured while attempting to convert Ed25519 pubkey to X25519; " - "is the pubkey valid?"}; - return xpk; - } - constexpr auto seed_hash_key = "SessionGroupKeySeed"sv; - const std::span enc_key_hash_key = to_span("SessionGroupKeyGen"); + const std::span enc_key_hash_key = to_span("SessionGroupKeyGen"); constexpr auto enc_key_admin_hash_key = "SessionGroupKeyAdminKey"sv; constexpr auto enc_key_member_hash_key = "SessionGroupKeyMemberKey"sv; - const std::span junk_seed_hash_key = to_span("SessionGroupJunkMembers"); + constexpr auto junk_seed_hash_key = "SessionGroupJunkMembers"_bytes; } // namespace -std::span Keys::rekey(Info& info, Members& members) { +std::span Keys::rekey(Info& info, Members& members) { if (!admin()) throw std::logic_error{ "Unable to issue a new group encryption key without the main group keys"}; @@ -256,10 +228,9 @@ std::span Keys::rekey(Info& info, Members& members) { // For members we calculate the outer encryption key as H(aB || A || B). But because we only // have `B` (the session id) as an x25519 pubkey, we do this in x25519 space, which means we // have to use the x25519 conversion of a/A rather than the group's ed25519 pubkey. - auto group_xpk = compute_xpk(_sign_pk->data()); + auto group_xpk = ed25519::pk_to_x25519(*_sign_pk); - sodium_cleared> group_xsk; - crypto_sign_ed25519_sk_to_curve25519(group_xsk.data(), _sign_sk.data()); + auto group_xsk = ed25519::sk_to_x25519(std::span{_sign_sk.data(), 64}); // We need quasi-randomness: full secure random would be great, except that different admins // encrypting for the same update would always create different keys, but we want it @@ -287,54 +258,33 @@ std::span Keys::rekey(Info& info, Members& members) { // member. For admins we encrypt using a 32-byte blake2b keyed hash of the group secret key // seed, just like H2, but with key "SessionGroupKeyAdminKey". - std::array h2 = seed_hash(seed_hash_key); - - std::array h1; + auto h2 = seed_hash(seed_hash_key); - crypto_generichash_blake2b_state st; - - crypto_generichash_blake2b_init( - &st, enc_key_hash_key.data(), enc_key_hash_key.size(), h1.size()); + hash::blake2b_hasher hasher{ + enc_key_hash_key, std::nullopt}; for (const auto& m : members) - crypto_generichash_blake2b_update( - &st, to_unsigned(m.session_id.data()), m.session_id.size()); + hasher.update(m.session_id); auto gen = keys_.empty() ? 0 : keys_.back().generation + 1; - auto gen_str = std::to_string(gen); - crypto_generichash_blake2b_update(&st, to_unsigned(gen_str.data()), gen_str.size()); - - crypto_generichash_blake2b_update(&st, h2.data(), 32); + hasher.update("{}"_format(gen), h2); - crypto_generichash_blake2b_final(&st, h1.data(), h1.size()); + auto h1 = hasher.finalize(); - std::span enc_key{h1.data(), 32}; - std::span nonce{h1.data() + 32, 24}; + std::span enc_key = + std::span{h1}.first(); + std::span nonce = + std::span{h1}.last(); oxenc::bt_dict_producer d{}; d.append("#", to_string_view(nonce)); - static_assert(crypto_aead_xchacha20poly1305_ietf_KEYBYTES == 32); - static_assert(crypto_aead_xchacha20poly1305_ietf_ABYTES == 16); - std::array< - unsigned char, - crypto_aead_xchacha20poly1305_ietf_KEYBYTES + crypto_aead_xchacha20poly1305_ietf_ABYTES> - encrypted; + std::array encrypted; std::string_view enc_sv = to_string_view(encrypted); // Shared key for admins auto member_k = seed_hash(enc_key_admin_hash_key); - static_assert(member_k.size() == crypto_aead_xchacha20poly1305_ietf_KEYBYTES); - crypto_aead_xchacha20poly1305_ietf_encrypt( - encrypted.data(), - nullptr, - enc_key.data(), - enc_key.size(), - nullptr, - 0, - nullptr, - nonce.data(), - member_k.data()); + encryption::xchacha20poly1305_encrypt(encrypted, enc_key, nonce, member_k); d.append("G", gen); d.append("K", enc_sv); @@ -342,8 +292,8 @@ std::span Keys::rekey(Info& info, Members& members) { { auto member_keys = d.append_list("k"); int member_count = 0; - std::vector> member_xpk_raw; - std::vector> member_xpks; + std::vector member_xpk_raw; + std::vector> member_xpks; member_xpk_raw.reserve(members.size()); member_xpks.reserve(members.size()); for (const auto& m : members) { @@ -358,7 +308,7 @@ std::span Keys::rekey(Info& info, Members& members) { to_span(group_xsk), to_span(group_xpk), enc_key_member_hash_key, - [&](std::span enc_sv) { + [&](std::span enc_sv) { member_keys.append(enc_sv); member_count++; }, @@ -368,17 +318,12 @@ std::span Keys::rekey(Info& info, Members& members) { // Pad it out with junk entries to the next MESSAGE_KEY_MULTIPLE if (member_count % MESSAGE_KEY_MULTIPLE) { int n_junk = MESSAGE_KEY_MULTIPLE - (member_count % MESSAGE_KEY_MULTIPLE); - std::vector junk_data; + std::vector junk_data; junk_data.resize(encrypted.size() * n_junk); - std::array rng_seed; - crypto_generichash_blake2b_init( - &st, junk_seed_hash_key.data(), junk_seed_hash_key.size(), rng_seed.size()); - crypto_generichash_blake2b_update(&st, h1.data(), h1.size()); - crypto_generichash_blake2b_update(&st, _sign_sk.data(), _sign_sk.size()); - crypto_generichash_blake2b_final(&st, rng_seed.data(), rng_seed.size()); + auto rng_seed = hash::blake2b_key<32>(junk_seed_hash_key, h1, _sign_sk); - randombytes_buf_deterministic(junk_data.data(), junk_data.size(), rng_seed.data()); + random::fill_deterministic(junk_data, rng_seed); std::string_view junk_view = to_string_view(junk_data); while (!junk_view.empty()) { member_keys.append(junk_view.substr(0, encrypted.size())); @@ -389,8 +334,7 @@ std::span Keys::rekey(Info& info, Members& members) { // Finally we sign the message at put it as the ~ key (which is 0x7e, and thus comes later than // any other printable ascii key). - d.append_signature( - "~", [this](std::span to_sign) { return sign(to_sign); }); + d.append_signature("~", [this](std::span to_sign) { return sign(to_sign); }); // Load this key/config/gen into our pending variables pending_gen_ = gen; @@ -408,17 +352,17 @@ std::span Keys::rekey(Info& info, Members& members) { needs_dump_ = true; - return std::span{pending_key_config_.data(), pending_key_config_.size()}; + return std::span{pending_key_config_.data(), pending_key_config_.size()}; } -std::vector Keys::sign(std::span data) const { +std::vector Keys::sign(std::span data) const { auto sig = signer_(data); if (sig.size() != 64) throw std::logic_error{"Invalid signature: signing function did not return 64 bytes"}; return sig; } -std::vector Keys::key_supplement(const std::vector& sids) const { +std::vector Keys::key_supplement(const std::vector& sids) const { if (!admin()) throw std::logic_error{ "Unable to issue supplemental group encryption keys without the main group keys"}; @@ -430,10 +374,8 @@ std::vector Keys::key_supplement(const std::vector& // For members we calculate the outer encryption key as H(aB || A || B). But because we only // have `B` (the session id) as an x25519 pubkey, we do this in x25519 space, which means we // have to use the x25519 conversion of a/A rather than the group's ed25519 pubkey. - auto group_xpk = compute_xpk(_sign_pk->data()); - - sodium_cleared> group_xsk; - crypto_sign_ed25519_sk_to_curve25519(group_xsk.data(), _sign_sk.data()); + auto group_xpk = ed25519::pk_to_x25519(*_sign_pk); + auto group_xsk = ed25519::sk_to_x25519(std::span{_sign_sk.data(), 64}); // We need quasi-randomness here for the nonce: full secure random would be great, except that // different admins encrypting for the same update would always create different keys, but we @@ -463,24 +405,15 @@ std::vector Keys::key_supplement(const std::vector& supp_keys = std::move(supp).str(); } - std::array h1; - - crypto_generichash_blake2b_state st; - - crypto_generichash_blake2b_init( - &st, enc_key_hash_key.data(), enc_key_hash_key.size(), h1.size()); - + hash::blake2b_hasher nonce_hasher{ + enc_key_hash_key, std::nullopt}; for (const auto& sid : sids) - crypto_generichash_blake2b_update(&st, to_unsigned(sid.data()), sid.size()); - - crypto_generichash_blake2b_update(&st, to_unsigned(supp_keys.data()), supp_keys.size()); + nonce_hasher.update(sid); - std::array h2 = seed_hash(seed_hash_key); - crypto_generichash_blake2b_update(&st, h2.data(), h2.size()); + auto h2 = seed_hash(seed_hash_key); + nonce_hasher.update(supp_keys, h2); - crypto_generichash_blake2b_final(&st, h1.data(), h1.size()); - - std::span nonce{h1.data(), h1.size()}; + auto nonce = nonce_hasher.finalize(); oxenc::bt_dict_producer d{}; @@ -488,13 +421,13 @@ std::vector Keys::key_supplement(const std::vector& { auto list = d.append_list("+"); - std::vector encrypted; - encrypted.resize(supp_keys.size() + crypto_aead_xchacha20poly1305_ietf_ABYTES); + std::vector encrypted; + encrypted.resize(supp_keys.size() + encryption::XCHACHA20_ABYTES); size_t member_count = 0; - std::vector> member_xpk_raw; - std::vector> member_xpks; + std::vector member_xpk_raw; + std::vector> member_xpks; member_xpk_raw.reserve(sids.size()); member_xpks.reserve(sids.size()); for (const auto& sid : sids) { @@ -509,7 +442,7 @@ std::vector Keys::key_supplement(const std::vector& to_span(group_xsk), to_span(group_xpk), enc_key_member_hash_key, - [&](std::span encrypted) { + [&](std::span encrypted) { list.append(encrypted); member_count++; }, @@ -525,49 +458,38 @@ std::vector Keys::key_supplement(const std::vector& // Finally we sign the message at put it as the ~ key (which is 0x7e, and thus comes later than // any other printable ascii key). - d.append_signature( - "~", [this](std::span to_sign) { return sign(to_sign); }); + d.append_signature("~", [this](std::span to_sign) { return sign(to_sign); }); return to_vector(d.view()); } // Blinding factor for subaccounts: H(sessionid || groupid) mod L, where H is 64-byte blake2b, using // a hash key derived from the group's seed. -std::array Keys::subaccount_blind_factor( - const std::array& session_xpk) const { +b32 Keys::subaccount_blind_factor(std::span session_xpk) const { auto mask = seed_hash("SessionGroupSubaccountMask"); - static_assert(mask.size() == crypto_generichash_blake2b_KEYBYTES); - - std::array h; - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, mask.data(), mask.size(), h.size()); - crypto_generichash_blake2b_update(&st, to_unsigned("\x05"), 1); - crypto_generichash_blake2b_update(&st, session_xpk.data(), session_xpk.size()); - crypto_generichash_blake2b_update(&st, to_unsigned("\x03"), 1); - crypto_generichash_blake2b_update(&st, _sign_pk->data(), _sign_pk->size()); - crypto_generichash_blake2b_final(&st, h.data(), h.size()); - - std::array out; - crypto_core_ed25519_scalar_reduce(out.data(), h.data()); - return out; + + auto h = hash::blake2b_key<64>(mask, std::byte{0x05}, session_xpk, std::byte{0x03}, *_sign_pk); + + return ed25519::scalar_reduce(h); } namespace { // These constants are defined and explains in more detail in oxen-storage-server - constexpr unsigned char SUBACC_FLAG_READ = 0b0001; - constexpr unsigned char SUBACC_FLAG_WRITE = 0b0010; - constexpr unsigned char SUBACC_FLAG_DEL = 0b0100; - constexpr unsigned char SUBACC_FLAG_ANY_PREFIX = 0b1000; - - constexpr unsigned char subacc_flags(bool write, bool del) { - return SUBACC_FLAG_READ | (write ? SUBACC_FLAG_WRITE : 0) | (del ? SUBACC_FLAG_DEL : 0); + constexpr std::byte SUBACC_FLAG_READ{0b0001}; + constexpr std::byte SUBACC_FLAG_WRITE{0b0010}; + constexpr std::byte SUBACC_FLAG_DEL{0b0100}; + constexpr std::byte SUBACC_FLAG_ANY_PREFIX{0b1000}; + + constexpr std::byte subacc_flags(bool write, bool del) { + return SUBACC_FLAG_READ | (write ? SUBACC_FLAG_WRITE : std::byte{0}) | + (del ? SUBACC_FLAG_DEL : std::byte{0}); } } // namespace -std::vector Keys::swarm_make_subaccount( +std::vector Keys::swarm_make_subaccount( std::string_view session_id, bool write, bool del) const { if (!admin()) throw std::logic_error{"Cannot make subaccount signature: admin keys required"}; @@ -599,30 +521,30 @@ std::vector Keys::swarm_make_subaccount( auto T = xed25519::pubkey(X); // kT is the user's Ed25519 blinded pubkey: - std::array kT; - - if (0 != crypto_scalarmult_ed25519_noclamp(kT.data(), k.data(), T.data())) - throw std::runtime_error{"scalarmult failed: perhaps an invalid session id?"}; + auto kT = ed25519::scalarmult_noclamp(k, T); - std::vector out; + std::vector out; out.resize(4 + 32 + 64); - out[0] = 0x03; // network prefix + out[0] = std::byte{0x03}; // network prefix out[1] = subacc_flags(write, del); // permission flags - out[2] = 0; // reserved 1 - out[3] = 0; // reserved 2 + out[2] = std::byte{0}; // reserved 1 + out[3] = std::byte{0}; // reserved 2 // The next 32 bytes are k (NOT kT; the user can go make kT themselves): std::memcpy(&out[4], k.data(), k.size()); // And then finally, we append a group signature of: p || f || 0 || 0 || kT - std::array to_sign; + std::array to_sign; std::memcpy(&to_sign[0], out.data(), 4); // first 4 bytes are the same as out std::memcpy(&to_sign[4], kT.data(), 32); // but then we have kT instead of k - crypto_sign_ed25519_detached(&out[36], nullptr, to_sign.data(), to_sign.size(), c.data()); + ed25519::sign( + std::span{out.data() + 36, 64}, + ed25519::PrivKeySpan{std::span{c.data(), 64}}, + to_sign); return out; } -std::vector Keys::swarm_subaccount_token( +std::vector Keys::swarm_subaccount_token( std::string_view session_id, bool write, bool del) const { if (!admin()) throw std::logic_error{"Cannot make subaccount signature: admin keys required"}; @@ -635,21 +557,20 @@ std::vector Keys::swarm_subaccount_token( // T = |S| auto T = xed25519::pubkey(X); - std::vector out; + auto kT = ed25519::scalarmult_noclamp(k, T); + + std::vector out; out.resize(4 + 32); - out[0] = 0x03; // network prefix + out[0] = std::byte{0x03}; // network prefix out[1] = subacc_flags(write, del); // permission flags - out[2] = 0; // reserved 1 - out[3] = 0; // reserved 2 - if (0 != crypto_scalarmult_ed25519_noclamp(&out[4], k.data(), T.data())) - throw std::runtime_error{"scalarmult failed: perhaps an invalid session id?"}; + out[2] = std::byte{0}; // reserved 1 + out[3] = std::byte{0}; // reserved 2 + std::memcpy(&out[4], kT.data(), 32); return out; } Keys::swarm_auth Keys::swarm_subaccount_sign( - std::span msg, - std::span sign_val, - bool binary) const { + std::span msg, std::span sign_val, bool binary) const { if (sign_val.size() != 100) throw std::logic_error{"Invalid signing value: size is wrong"}; @@ -661,7 +582,7 @@ Keys::swarm_auth Keys::swarm_subaccount_sign( // (see above for variable/crypto notation) - std::span k = sign_val.subspan(4, 32); + auto k = sign_val.subspan<4, 32>(); // our token is the first 4 bytes of `sign_val` (flags, etc.), followed by kT which we have to // compute: @@ -669,30 +590,27 @@ Keys::swarm_auth Keys::swarm_subaccount_sign( std::memcpy(token.data(), sign_val.data(), 4); // T = |S|, i.e. we have to clear the sign bit from our pubkey - std::array T; - crypto_sign_ed25519_sk_to_pk(T.data(), user_ed25519_sk.data()); - bool neg = T[31] & 0x80; - T[31] &= 0x7f; - if (0 != crypto_scalarmult_ed25519_noclamp(to_unsigned(token.data() + 4), k.data(), T.data())) - throw std::runtime_error{"scalarmult failed: perhaps an invalid session id or seed?"}; + ed25519::PrivKeySpan user_sk{std::span{user_ed25519_sk.data(), 64}}; + b32 T; + std::ranges::copy(user_sk.pubkey(), T.begin()); + bool neg = (T[31] & std::byte{0x80}) != std::byte{0}; + T[31] &= std::byte{0x7f}; - // token is now set: flags || kT - std::span kT{to_unsigned(token.data() + 4), 32}; + auto kT = ed25519::scalarmult_noclamp(k, T); + std::memcpy(token.data() + 4, kT.data(), 32); // sub_sig is just the admin's signature, sitting at the end of sign_val (after 4f || k): sub_sig = to_string_view(sign_val.subspan(36)); // Our signing private scalar is kt, where t = ±s according to whether we had to negate S to // make T - std::array s, s_neg; - crypto_sign_ed25519_sk_to_curve25519(s.data(), user_ed25519_sk.data()); - crypto_core_ed25519_scalar_negate(s_neg.data(), s.data()); + auto s = ed25519::sk_to_x25519(user_sk); + auto s_neg = ed25519::scalar_negate(s); xed25519::constant_time_conditional_assign(s, s_neg, neg); auto& t = s; - std::array kt; - crypto_core_ed25519_scalar_mul(kt.data(), k.data(), t.data()); + auto kt = ed25519::scalar_mul(k, t); // We now have kt, kT, our privkey/public. (Note that kt is a scalar, not a seed). @@ -716,46 +634,28 @@ Keys::swarm_auth Keys::swarm_subaccount_sign( // // (using the standard Ed25519 SHA-512 here for H) - constexpr auto seed_hash_key = "SubaccountSeed"sv; - constexpr auto r_hash_key = "SubaccountSig"sv; - std::array hseed; - crypto_generichash_blake2b( - hseed.data(), - hseed.size(), - user_ed25519_sk.data(), - 32, - to_unsigned(seed_hash_key.data()), - seed_hash_key.size()); - - std::array tmp; - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init( - &st, to_unsigned(r_hash_key.data()), r_hash_key.size(), tmp.size()); - crypto_generichash_blake2b_update(&st, hseed.data(), hseed.size()); - crypto_generichash_blake2b_update(&st, kT.data(), kT.size()); - crypto_generichash_blake2b_update(&st, msg.data(), msg.size()); - crypto_generichash_blake2b_final(&st, tmp.data(), tmp.size()); - - std::array r; - crypto_core_ed25519_scalar_reduce(r.data(), tmp.data()); + constexpr auto subacc_seed_key = "SubaccountSeed"_bytes; + constexpr auto subacc_sig_key = "SubaccountSig"_bytes; + b32 hseed; + hash::blake2b_key(hseed, subacc_seed_key, user_sk.seed()); + + b64 tmp; + hash::blake2b_key(tmp, subacc_sig_key, hseed, kT, msg); + + auto r = ed25519::scalar_reduce(tmp); sig.resize(64); - unsigned char* R = to_unsigned(sig.data()); - unsigned char* S = to_unsigned(sig.data() + 32); + auto R = std::span{to_bytes(sig.data()), 32}; + auto S = std::span{to_bytes(sig.data()) + 32, 32}; // R = rB - crypto_scalarmult_ed25519_base_noclamp(R, r.data()); + ed25519::scalarmult_base_noclamp(R, r); // Compute S = r + H(R || A || M) a mod L: (with A = kT, a = kt) - crypto_hash_sha512_state shast; - crypto_hash_sha512_init(&shast); - crypto_hash_sha512_update(&shast, R, 32); - crypto_hash_sha512_update(&shast, kT.data(), kT.size()); // A = pubkey, that is, kT - crypto_hash_sha512_update(&shast, msg.data(), msg.size()); - std::array hram; - crypto_hash_sha512_final(&shast, hram.data()); // S = H(R||A||M) - crypto_core_ed25519_scalar_reduce(S, hram.data()); // S %= L - crypto_core_ed25519_scalar_mul(S, S, kt.data()); // S *= a - crypto_core_ed25519_scalar_add(S, S, r.data()); // S += r + b64 hram; + hash::sha512(hram, R, kT, msg); + ed25519::scalar_reduce(S, hram); // S = H(R||A||M) % L + ed25519::scalar_mul(S, S, kt); // S *= a + ed25519::scalar_add(S, S, r); // S += r // sig is now set to the desired R || S, with S = r + H(R || A || M)a (all mod L) @@ -769,12 +669,12 @@ Keys::swarm_auth Keys::swarm_subaccount_sign( } bool Keys::swarm_verify_subaccount( - std::span sign_val, bool write, bool del) const { + std::span sign_val, bool write, bool del) const { if (!_sign_pk) return false; return swarm_verify_subaccount( - "03" + oxenc::to_hex(_sign_pk->begin(), _sign_pk->end()), - std::span{user_ed25519_sk.data(), user_ed25519_sk.size()}, + "03{:x}"_format(*_sign_pk), + ed25519::PrivKeySpan::from(user_ed25519_sk), sign_val, write, del); @@ -782,8 +682,8 @@ bool Keys::swarm_verify_subaccount( bool Keys::swarm_verify_subaccount( std::string group_id, - std::span user_ed_sk, - std::span sign_val, + const ed25519::PrivKeySpan& session_ed25519_secretkey, + std::span sign_val, bool write, bool del) { auto group_pk = session_id_pk(group_id, "03"); @@ -791,46 +691,43 @@ bool Keys::swarm_verify_subaccount( if (sign_val.size() != 100) return false; - std::span prefix = sign_val.subspan(0, 4); - if (prefix[0] != 0x03 && !(prefix[1] & SUBACC_FLAG_ANY_PREFIX)) + auto prefix = sign_val.subspan<0, 4>(); + if (prefix[0] != std::byte{0x03} && (prefix[1] & SUBACC_FLAG_ANY_PREFIX) == std::byte{0}) return false; // require either 03 prefix match, or the "any prefix" flag - if (!(prefix[1] & SUBACC_FLAG_READ)) + if ((prefix[1] & SUBACC_FLAG_READ) == std::byte{0}) return false; // missing the read flag - if (write && !(prefix[1] & SUBACC_FLAG_WRITE)) + if (write && (prefix[1] & SUBACC_FLAG_WRITE) == std::byte{0}) return false; // we require write, but it isn't set - // - if (del && !(prefix[1] & SUBACC_FLAG_DEL)) + + if (del && (prefix[1] & SUBACC_FLAG_DEL) == std::byte{0}) return false; // we require delete, but it isn't set - std::span k = sign_val.subspan(4, 32); - std::span sig = sign_val.subspan(36); + auto k = sign_val.subspan<4, 32>(); + auto sig = sign_val.subspan<36, 64>(); // T = |S|, i.e. we have to clear the sign bit from our pubkey - std::array T; - crypto_sign_ed25519_sk_to_pk(T.data(), user_ed_sk.data()); - T[31] &= 0x7f; + b32 T; + std::ranges::copy(session_ed25519_secretkey.pubkey(), T.begin()); + T[31] &= std::byte{0x7f}; // Compute kT, then reconstruct the `flags || kT` value the admin should have provided a // signature for - std::array kT; - if (0 != crypto_scalarmult_ed25519_noclamp(kT.data(), k.data(), T.data())) - throw std::runtime_error{"scalarmult failed: perhaps an invalid session id or seed?"}; + auto kT = ed25519::scalarmult_noclamp(k, T); - std::array to_verify; + std::array to_verify; std::memcpy(&to_verify[0], sign_val.data(), 4); // prefix, flags, 2x future use bytes std::memcpy(&to_verify[4], kT.data(), 32); // Verify it! - return 0 == crypto_sign_ed25519_verify_detached( - sig.data(), to_verify.data(), to_verify.size(), group_pk.data()); + return ed25519::verify(sig, group_pk, to_verify); } -std::optional> Keys::pending_config() const { +std::optional> Keys::pending_config() const { if (pending_key_config_.empty()) return std::nullopt; - return std::span{pending_key_config_.data(), pending_key_config_.size()}; + return std::span{pending_key_config_.data(), pending_key_config_.size()}; } void Keys::insert_key(std::string_view msg_hash, key_info&& new_key) { @@ -864,10 +761,10 @@ void Keys::insert_key(std::string_view msg_hash, key_info&& new_key) { // Attempts xchacha20 decryption. // // Preconditions: -// - `ciphertext` must be at least 16 [crypto_aead_xchacha20poly1305_ietf_ABYTES] +// - `ciphertext` must be at least 16 [encryption::XCHACHA20_ABYTES] // - `out` must have enough space (ciphertext.size() - 16 -// [crypto_aead_xchacha20poly1305_ietf_ABYTES]) -// - `nonce` must be 24 bytes [crypto_aead_xchacha20poly1305_ietf_NPUBBYTES] +// [encryption::XCHACHA20_ABYTES]) +// - `nonce` must be 24 bytes [encryption::XCHACHA20_NONCEBYTES] // - `key` must be 32 bytes [crypto_aead_xchacha20poly1305_ietf_KEYBYTES] // // The latter two are asserted in a debug build, but not otherwise checked. @@ -875,39 +772,17 @@ void Keys::insert_key(std::string_view msg_hash, key_info&& new_key) { // Returns true (after writing to `out`) if decryption succeeds, false if it fails. namespace { bool try_decrypting( - unsigned char* out, - std::span encrypted, - std::span nonce, - std::span key) { - assert(encrypted.size() >= crypto_aead_xchacha20poly1305_ietf_ABYTES); - assert(nonce.size() == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - assert(key.size() == crypto_aead_xchacha20poly1305_ietf_KEYBYTES); - - return 0 == crypto_aead_xchacha20poly1305_ietf_decrypt( - out, - nullptr, - nullptr, - encrypted.data(), - encrypted.size(), - nullptr, - 0, - nonce.data(), - key.data()); - } - bool try_decrypting( - unsigned char* out, - std::span encrypted, - std::span nonce, - - const std::array& key) { - return try_decrypting( - out, encrypted, nonce, std::span{key.data(), key.size()}); + std::span out, + std::span encrypted, + std::span nonce, + std::span key) { + return encryption::xchacha20poly1305_decrypt(out, encrypted, nonce, key); } } // namespace bool Keys::load_key_message( std::string_view hash, - std::span data, + std::span data, int64_t timestamp_ms, Info& info, Members& members) { @@ -917,22 +792,26 @@ bool Keys::load_key_message( if (!_sign_pk || !verifier_) throw std::logic_error{"Group pubkey is not set; unable to load config message"}; - auto group_xpk = compute_xpk(_sign_pk->data()); + auto group_xpk = ed25519::pk_to_x25519(*_sign_pk); if (!d.skip_until("#")) throw config_value_error{"Key message has no nonce"}; - auto nonce = to_span(d.consume_string_view()); + auto nonce_dyn = d.consume_span(); + if (nonce_dyn.size() != encryption::XCHACHA20_NONCEBYTES) + throw config_value_error{"Key message has invalid nonce size"}; + auto nonce = nonce_dyn.first(); sodium_vector new_keys; std::optional max_gen; // If set then associate the message with this generation // value, even if we didn't find a key for us. - sodium_cleared> member_dec_key; - sodium_cleared> member_xsk; - std::array member_xpk; + sodium_cleared member_dec_key; + sodium_cleared member_xsk; + b32 member_xpk; if (!admin()) { - crypto_sign_ed25519_sk_to_curve25519(member_xsk.data(), user_ed25519_sk.data()); - member_xpk = compute_xpk(user_ed25519_sk.data() + 32); + ed25519::PrivKeySpan user_sk{std::span{user_ed25519_sk.data(), 64}}; + member_xsk = ed25519::sk_to_x25519(user_sk); + member_xpk = ed25519::pk_to_x25519(user_sk.pubkey()); } if (d.skip_until("+")) { @@ -941,7 +820,7 @@ bool Keys::load_key_message( int member_key_pos = -1; - auto next_ciphertext = [&]() -> std::optional> { + auto next_ciphertext = [&]() -> std::optional> { while (!supp.is_finished()) { member_key_pos++; auto encrypted = to_span(supp.consume_string_view()); @@ -954,11 +833,10 @@ bool Keys::load_key_message( // e + 1 // --- // 52 - if (encrypted.size() < 52 + crypto_aead_xchacha20poly1305_ietf_ABYTES) + if (encrypted.size() < 52 + encryption::XCHACHA20_ABYTES) throw config_value_error{ - "Supplemental key message has invalid key info size at " - "index " + - std::to_string(member_key_pos)}; + "Supplemental key message has invalid key info size at index {}"_format( + member_key_pos)}; if (!new_keys.empty() || admin()) continue; // Keep parsing, to ensure validity of the whole message @@ -1026,13 +904,13 @@ bool Keys::load_key_message( "Non-supplemental key message is missing required admin key (K)"}; auto admin_key = to_span(d.consume_string_view()); - if (admin_key.size() != 32 + crypto_aead_xchacha20poly1305_ietf_ABYTES) + if (admin_key.size() != 32 + encryption::XCHACHA20_ABYTES) throw config_value_error{"Key message has invalid admin key length"}; if (admin()) { auto k = seed_hash(enc_key_admin_hash_key); - if (!try_decrypting(new_key.key.data(), admin_key, nonce, k)) + if (!try_decrypting(new_key.key, admin_key, nonce, k)) throw config_value_error{"Failed to decrypt admin key from key message"}; found_key = true; @@ -1046,14 +924,14 @@ bool Keys::load_key_message( auto key_list = d.consume_list_consumer(); int member_key_pos = -1; - auto next_ciphertext = [&]() -> std::optional> { + auto next_ciphertext = [&]() -> std::optional> { while (!key_list.is_finished()) { member_key_pos++; auto member_key = to_span(key_list.consume_string_view()); - if (member_key.size() != 32 + crypto_aead_xchacha20poly1305_ietf_ABYTES) + if (member_key.size() != 32 + encryption::XCHACHA20_ABYTES) throw config_value_error{ - "Key message has invalid member key length at index " + - std::to_string(member_key_pos)}; + "Key message has invalid member key length at index {}"_format( + member_key_pos)}; if (found_key) continue; @@ -1183,60 +1061,46 @@ bool Keys::needs_rekey() const { return last_it->generation == second_it->generation; } -std::optional> Keys::pending_key() const { +std::optional> Keys::pending_key() const { if (!pending_key_config_.empty()) - return std::span{pending_key_.data(), pending_key_.size()}; + return std::span{pending_key_}; return std::nullopt; } static constexpr size_t ENCRYPT_OVERHEAD = - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + crypto_aead_xchacha20poly1305_ietf_ABYTES; + encryption::XCHACHA20_NONCEBYTES + encryption::XCHACHA20_ABYTES; -std::vector Keys::encrypt_message( - std::span plaintext, bool compress, size_t padding) const { +std::vector Keys::encrypt_message( + std::span plaintext, bool compress, size_t padding) const { assert(_sign_pk); - std::vector ciphertext = encrypt_for_group( - user_ed25519_sk, *_sign_pk, group_enc_key(), plaintext, compress, padding); + std::vector ciphertext = encrypt_for_group( + ed25519::PrivKeySpan::from(user_ed25519_sk), + *_sign_pk, + group_enc_key(), + plaintext, + compress, + padding); return ciphertext; } -std::pair> Keys::decrypt_message( - std::span ciphertext) const { +std::pair> Keys::decrypt_message( + std::span ciphertext) const { assert(_sign_pk); // // Decrypt, using all the possible keys, starting with a pending one (if we have one) // - DecryptGroupMessage decrypt = {}; - bool decrypt_success = false; - if (auto pending = pending_key()) { - try { - std::span> key_list = {&(*pending), 1}; - decrypt = decrypt_group_message(key_list, *_sign_pk, ciphertext); - decrypt_success = true; - } catch (const std::exception&) { - } - } - - if (!decrypt_success) { - for (auto& k : keys_) { - try { - std::span key = {k.key.data(), k.key.size()}; - std::span> key_list = {&key, 1}; - decrypt = decrypt_group_message(key_list, *_sign_pk, ciphertext); - decrypt_success = true; - break; - } catch (const std::exception&) { - } - } - } + // Build the list of candidate keys: pending key (if any) first, then all active keys. + std::vector> key_list; + key_list.reserve(keys_.size() + 1); + if (auto pending = pending_key()) + key_list.push_back(*pending); + for (auto& k : keys_) + key_list.emplace_back(k.key); - if (!decrypt_success) // none of the keys worked - throw std::runtime_error{fmt::format( - "unable to decrypt ciphertext with any current group keys; tried {}", - keys_.size() + (pending_key() ? 1 : 0))}; + auto decrypt = decrypt_group_message(key_list, *_sign_pk, ciphertext); - std::pair> result; + std::pair> result; result.first = std::move(decrypt.session_id); result.second = std::move(decrypt.plaintext); return result; @@ -1306,14 +1170,13 @@ LIBSESSION_C_API int groups_keys_init( assert(user_ed25519_secretkey && group_ed25519_pubkey && cinfo && cmembers); - std::span user_sk{user_ed25519_secretkey, 64}; - std::span group_pk{group_ed25519_pubkey, 32}; - std::optional> group_sk; - if (group_ed25519_secretkey) - group_sk.emplace(group_ed25519_secretkey, 64); - std::optional> dumped; + ed25519::PrivKeySpan user_sk{user_ed25519_secretkey, 64}; + auto group_pk = to_byte_span<32>(group_ed25519_pubkey); + ed25519::OptionalPrivKeySpan group_sk{ + group_ed25519_secretkey, group_ed25519_secretkey ? 64u : 0u}; + std::optional> dumped; if (dump && dumplen) - dumped.emplace(dump, dumplen); + dumped.emplace(to_byte_span(dump, dumplen)); auto& info = *unbox(cinfo); auto& members = *unbox(cmembers); @@ -1353,7 +1216,7 @@ LIBSESSION_C_API const unsigned char* groups_keys_get_key(const config_group_key auto keys = unbox(conf).group_keys(); if (N >= keys.size()) return nullptr; - return keys[N].data(); + return to_unsigned(keys[N].data()); } LIBSESSION_C_API size_t groups_keys_get_keys( @@ -1363,9 +1226,9 @@ LIBSESSION_C_API size_t groups_keys_get_keys( auto keys = unbox(conf).group_keys(); size_t clamped_offset = std::min(keys.size(), offset); for (size_t index = clamped_offset; index < keys.size() && result < dest_size; index++) { - const std::span& src_key = keys[index]; + const auto& src_key = keys[index]; span_u8* dest_key = dest + result++; - dest_key->data = const_cast(src_key.data()); + dest_key->data = const_cast(to_unsigned(src_key.data())); dest_key->size = src_key.size(); } } @@ -1375,8 +1238,8 @@ LIBSESSION_C_API size_t groups_keys_get_keys( LIBSESSION_C_API const span_u8 groups_keys_group_enc_key(const config_group_keys* conf) { span_u8 result = {}; try { - std::span key = unbox(conf).group_enc_key(); - result.data = const_cast(key.data()); + auto key = unbox(conf).group_enc_key(); + result.data = const_cast(to_unsigned(key.data())); result.size = key.size(); assert(result.size == 32); } catch (const std::exception& e) { @@ -1397,7 +1260,7 @@ LIBSESSION_C_API bool groups_keys_load_admin_key( conf, [&] { unbox(conf).load_admin_key( - std::span{secret, 32}, + ed25519::PrivKeySpan{secret, 32}, *unbox(info), *unbox(members)); return true; @@ -1413,14 +1276,14 @@ LIBSESSION_C_API bool groups_keys_rekey( size_t* outlen) { assert(info && members); auto& keys = unbox(conf); - std::span to_push; + std::span to_push; return wrap_exceptions( conf, [&] { to_push = keys.rekey(*unbox(info), *unbox(members)); if (out && outlen) { - *out = to_push.data(); + *out = to_unsigned(to_push.data()); *outlen = to_push.size(); } return true; @@ -1432,7 +1295,7 @@ LIBSESSION_C_API bool groups_keys_pending_config( const config_group_keys* conf, const unsigned char** out, size_t* outlen) { assert(out && outlen); if (auto pending = unbox(conf).pending_config()) { - *out = pending->data(); + *out = to_unsigned(pending->data()); *outlen = pending->size(); return true; } @@ -1453,7 +1316,7 @@ LIBSESSION_C_API bool groups_keys_load_message( [&] { unbox(conf).load_key_message( msg_hash, - std::span{data, datalen}, + to_byte_span(data, datalen), timestamp_ms, *unbox(info), *unbox(members)); @@ -1491,10 +1354,9 @@ LIBSESSION_C_API void groups_keys_encrypt_message( size_t* ciphertext_len) { assert(plaintext_in && ciphertext_out && ciphertext_len); - std::vector ciphertext; + std::vector ciphertext; try { - ciphertext = unbox(conf).encrypt_message( - std::span{plaintext_in, plaintext_len}); + ciphertext = unbox(conf).encrypt_message(to_byte_span(plaintext_in, plaintext_len)); *ciphertext_out = static_cast(std::malloc(ciphertext.size())); std::memcpy(*ciphertext_out, ciphertext.data(), ciphertext.size()); *ciphertext_len = ciphertext.size(); @@ -1516,8 +1378,8 @@ LIBSESSION_C_API bool groups_keys_decrypt_message( return wrap_exceptions( conf, [&] { - auto [sid, plaintext] = unbox(conf).decrypt_message( - std::span{ciphertext_in, ciphertext_len}); + auto [sid, plaintext] = + unbox(conf).decrypt_message(to_byte_span(ciphertext_in, ciphertext_len)); std::memcpy(session_id, sid.c_str(), sid.size() + 1); *plaintext_out = static_cast(std::malloc(plaintext.size())); std::memcpy(*plaintext_out, plaintext.data(), plaintext.size()); @@ -1587,8 +1449,8 @@ LIBSESSION_C_API bool groups_keys_swarm_verify_subaccount_flags( try { return groups::Keys::swarm_verify_subaccount( group_id, - std::span{session_ed25519_secretkey, 64}, - std::span{signing_value, 100}, + ed25519::PrivKeySpan{session_ed25519_secretkey, 64}, + to_byte_span(signing_value, 100), write, del); } catch (...) { @@ -1603,8 +1465,8 @@ LIBSESSION_C_API bool groups_keys_swarm_verify_subaccount( try { return groups::Keys::swarm_verify_subaccount( group_id, - std::span{session_ed25519_secretkey, 64}, - std::span{signing_value, 100}); + ed25519::PrivKeySpan{session_ed25519_secretkey, 64}, + to_byte_span(signing_value, 100)); } catch (...) { return false; } @@ -1624,8 +1486,7 @@ LIBSESSION_C_API bool groups_keys_swarm_subaccount_sign( conf, [&] { auto auth = unbox(conf).swarm_subaccount_sign( - std::span{msg, msg_len}, - std::span{signing_value, 100}); + to_byte_span(msg, msg_len), to_byte_span(signing_value, 100)); assert(auth.subaccount.size() == 48); assert(auth.subaccount_sig.size() == 88); assert(auth.signature.size() == 88); @@ -1654,9 +1515,7 @@ LIBSESSION_C_API bool groups_keys_swarm_subaccount_sign_binary( conf, [&] { auto auth = unbox(conf).swarm_subaccount_sign( - std::span{msg, msg_len}, - std::span{signing_value, 100}, - true); + to_byte_span(msg, msg_len), to_byte_span(signing_value, 100), true); assert(auth.subaccount.size() == 36); assert(auth.subaccount_sig.size() == 64); assert(auth.signature.size() == 64); diff --git a/src/config/groups/members.cpp b/src/config/groups/members.cpp index 5d40d5fea..bd1c086b5 100644 --- a/src/config/groups/members.cpp +++ b/src/config/groups/members.cpp @@ -8,9 +8,9 @@ namespace session::config::groups { Members::Members( - std::span ed25519_pubkey, - std::optional> ed25519_secretkey, - std::optional> dumped) { + std::span ed25519_pubkey, + const ed25519::OptionalPrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, ed25519_pubkey, ed25519_secretkey); } @@ -186,7 +186,9 @@ member::member(const config_group_member& m) : session_id{m.session_id, 66} { assert(std::strlen(m.profile_pic.url) <= profile_pic::MAX_URL_LENGTH); if (std::strlen(m.profile_pic.url)) { profile_picture.url = m.profile_pic.url; - profile_picture.key.assign(m.profile_pic.key, m.profile_pic.key + 32); + profile_picture.key.assign( + reinterpret_cast(m.profile_pic.key), + reinterpret_cast(m.profile_pic.key) + 32); } profile_updated = to_sys_seconds(m.profile_updated); admin = m.admin; diff --git a/src/config/internal.cpp b/src/config/internal.cpp index 6125f7ba6..d556b48da 100644 --- a/src/config/internal.cpp +++ b/src/config/internal.cpp @@ -30,8 +30,8 @@ void check_session_id(std::string_view session_id, std::string_view prefix) { if (!(session_id.size() == 64 + prefix.size() && oxenc::is_hex(session_id) && session_id.substr(0, prefix.size()) == prefix)) throw std::invalid_argument{ - "Invalid session ID: expected 66 hex digits starting with " + std::string{prefix} + - "; got " + std::string{session_id}}; + "Invalid session ID: expected 66 hex digits starting with {}; got {}"_format( + prefix, session_id)}; } SessionIDPrefix get_session_id_prefix(std::string_view id) { @@ -57,9 +57,9 @@ std::string session_id_to_bytes(std::string_view session_id, std::string_view pr return oxenc::from_hex(session_id); } -std::array session_id_pk(std::string_view session_id, std::string_view prefix) { +b32 session_id_pk(std::string_view session_id, std::string_view prefix) { check_session_id(session_id, prefix); - std::array pk; + b32 pk; session_id.remove_prefix(2); oxenc::from_hex(session_id.begin(), session_id.end(), pk.begin()); return pk; @@ -72,8 +72,8 @@ void check_encoded_pubkey(std::string_view pk) { throw std::invalid_argument{"Invalid encoded pubkey: expected hex, base32z or base64"}; } -std::vector decode_pubkey(std::string_view pk) { - std::vector pubkey; +std::vector decode_pubkey(std::string_view pk) { + std::vector pubkey; pubkey.reserve(32); if (pk.size() == 64 && oxenc::is_hex(pk)) oxenc::from_hex(pk.begin(), pk.end(), std::back_inserter(pubkey)); @@ -147,28 +147,6 @@ std::optional maybe_string(const session::config::dict& d, const ch return std::nullopt; } -uint64_t bitset_from_set_of_int64_or_0(const session::config::set& s) { - uint64_t result = 0; - constexpr size_t bits_available = sizeof(result) * 8; - for (auto& v : s) { - auto* val = std::get_if(&v); - if (val && (*val >= 0 && *val < bits_available)) - result |= (1ULL << *val); - } - return result; -} - -void set_int64_set_from_bitset(ConfigBase::DictFieldProxy&& field, uint64_t bitset) { - constexpr size_t bits_available = sizeof(bitset) * 8; - for (size_t index = 0; index < bits_available; index++) { - uint64_t bit = bitset & (1ULL << index); - if (bit) - field.set_insert(index); - else - field.set_erase(index); - } -} - std::string string_or_empty(const session::config::dict& d, const char* key) { if (auto* s = maybe_scalar(d, key)) return *s; @@ -187,21 +165,19 @@ std::string_view sv_or_empty(const session::config::dict& d, const char* key) { return ""sv; } -std::optional> maybe_span( +std::optional> maybe_span( const session::config::dict& d, const char* key) { - std::optional> ret; + std::optional> ret; if (auto* s = maybe_scalar(d, key)) - ret.emplace(reinterpret_cast(s->data()), s->size()); + ret.emplace(reinterpret_cast(s->data()), s->size()); return ret; } -std::optional> maybe_vector( +std::optional> maybe_vector( const session::config::dict& d, const char* key) { - std::optional> result; + std::optional> result; if (auto* s = maybe_scalar(d, key)) - result.emplace( - reinterpret_cast(s->data()), - reinterpret_cast(s->data()) + s->size()); + result = to_vector(*s); return result; } diff --git a/src/config/internal.hpp b/src/config/internal.hpp index 499e2eaa0..5baba6d1a 100644 --- a/src/config/internal.hpp +++ b/src/config/internal.hpp @@ -7,10 +7,14 @@ #include #include #include +#include +#include "../internal-util.hpp" +#include "session/clock.hpp" #include "session/config/base.h" #include "session/config/base.hpp" #include "session/config/error.h" +#include "session/session_protocol.hpp" #include "session/types.hpp" namespace session { @@ -63,10 +67,10 @@ template size_t dumplen, char* error) { assert(ed25519_secretkey_bytes); - std::span ed25519_secretkey{ed25519_secretkey_bytes, 64}; - std::optional> dump; + ed25519::PrivKeySpan ed25519_secretkey{ed25519_secretkey_bytes, 64}; + std::optional> dump; if (dumpstr && dumplen) - dump.emplace(dumpstr, dumplen); + dump.emplace(reinterpret_cast(dumpstr), dumplen); return c_wrapper_init_generic(conf, error, ed25519_secretkey, dump); } @@ -81,13 +85,13 @@ template assert(ed25519_pubkey_bytes); - std::span ed25519_pubkey{ed25519_pubkey_bytes, 32}; - std::optional> ed25519_secretkey; - if (ed25519_secretkey_bytes) - ed25519_secretkey.emplace(ed25519_secretkey_bytes, 64); - std::optional> dump; + std::span ed25519_pubkey{ + reinterpret_cast(ed25519_pubkey_bytes), 32}; + ed25519::OptionalPrivKeySpan ed25519_secretkey{ + ed25519_secretkey_bytes, ed25519_secretkey_bytes ? 64u : 0u}; + std::optional> dump; if (dump_bytes && dumplen) - dump.emplace(dump_bytes, dumplen); + dump.emplace(reinterpret_cast(dump_bytes), dumplen); return c_wrapper_init_generic(conf, error, ed25519_pubkey, ed25519_secretkey, dump); } @@ -148,7 +152,7 @@ std::string session_id_to_bytes(std::string_view session_id, std::string_view pr // Checks the session_id (throwing if invalid) then returns it as bytes, omitting the 05 (or // whatever) prefix, which is a pubkey (x25519 for 05 session_ids, ed25519 for other prefixes). -std::array session_id_pk( +std::array session_id_pk( std::string_view session_id, std::string_view prefix = "05"); // Validates a community pubkey; we accept it in hex, base32z, or base64 (padded or unpadded). @@ -157,7 +161,7 @@ void check_encoded_pubkey(std::string_view pk); // Takes a 32-byte pubkey value encoded as hex, base32z, or base64 and returns the decoded 32 bytes. // Throws if invalid. -std::vector decode_pubkey(std::string_view pk); +std::vector decode_pubkey(std::string_view pk); // Modifies a string to be (ascii) lowercase. void make_lc(std::string& s); @@ -172,12 +176,6 @@ std::optional maybe_int(const session::config::dict& d, const char* key // int. Equivalent to `maybe_int(d, key).value_or(0)`. int64_t int_or_0(const session::config::dict& d, const char* key); -// Returns std::chrono::system_clock::now(), with the given precision (seconds, if unspecified). -template -std::chrono::sys_time ts_now() { - return std::chrono::floor(std::chrono::system_clock::now()); -} - // Digs into a config `dict` to get out an int64_t containing unix timestamp seconds, returns it // wrapped in a std::chrono::sys_seconds. Returns nullopt if not there (or not int). std::optional maybe_ts(const session::config::dict& d, const char* key); @@ -196,11 +194,34 @@ std::chrono::sys_seconds ts_or_epoch(const session::config::dict& d, const char* // Digs into a config `dict` to get out a string; nullopt if not there (or not string) std::optional maybe_string(const session::config::dict& d, const char* key); -// Extract a U64 bitset from a set of i64's -uint64_t bitset_from_set_of_int64_or_0(const session::config::set& s); +// A feature flag enum (ProProfileFlags/ProMessageFlags) is stored in a config as the *set of bit +// positions* that are set (e.g. the flags 0b101 are stored as the set {0, 2}) rather than as a +// single packed integer, so that the CRDT merges concurrent per-flag changes as a set union instead +// of clobbering the whole value. These two helpers convert between that stored set and the enum. + +// Converts a config set of bit positions into flags of type `E`. Set elements that aren't int64s or +// fall outside [0, 64) are ignored; an empty or absent set yields no flags. +template +E to_flags(const session::config::set& positions) { + uint64_t mask = 0; + for (const auto& v : positions) + if (auto* pos = std::get_if(&v); pos && *pos >= 0 && *pos < 64) + mask |= 1ULL << *pos; + return static_cast(mask); +} -// Individually write each bit from bitset into a set consisting of int64's -void set_int64_set_from_bitset(ConfigBase::DictFieldProxy&& field, uint64_t bitset); +// Inverse of to_flags(): writes `f` into `field` as the set of its set bit positions (inserting the +// position of each set flag, erasing that of each unset one). +template +void set_flags(ConfigBase::DictFieldProxy&& field, E f) { + auto mask = static_cast(f); + for (int64_t pos = 0; pos < 64; pos++) { + if (mask & (1ULL << pos)) + field.set_insert(pos); + else + field.set_erase(pos); + } +} // Digs into a config `dict` to get out a string; ""s if not there (or not string) std::string string_or_empty(const session::config::dict& d, const char* key); @@ -213,15 +234,14 @@ std::optional maybe_sv(const session::config::dict& d, const c // string view is only valid as long as the dict stays unchanged. std::string_view sv_or_empty(const session::config::dict& d, const char* key); -// Digs into a config `dict` to get out a std::span; nullopt if not there (or +// Digs into a config `dict` to get out a std::span; nullopt if not there (or // not string) -std::optional> maybe_span( +std::optional> maybe_span( const session::config::dict& d, const char* key); -// Digs into a config `dict` to get out a std::vector; nullopt if not there (or not +// Digs into a config `dict` to get out a std::vector; nullopt if not there (or not // string) -std::optional> maybe_vector( - const session::config::dict& d, const char* key); +std::optional> maybe_vector(const session::config::dict& d, const char* key); /// Sets a value to 1 if true, removes it if false. void set_flag(ConfigBase::DictFieldProxy&& field, bool val); @@ -271,6 +291,47 @@ void load_unknowns( oxenc::bt_dict_consumer& in, std::string_view previous, std::string_view until); +template , int> = 0> +inline internals& unbox(config_object* conf) { + return *static_cast*>(conf->internals); +} +template , int> = 0> +inline const internals& unbox(const config_object* conf) { + return *static_cast*>(conf->internals); +} + +// Wraps a lambda and, if an exception is thrown, sets an error message in the config_object's +// error buffer and updates the last_error pointer. +template +decltype(auto) wrap_exceptions(config_object* conf, Call&& f) { + using Ret = std::invoke_result_t; + + try { + conf->last_error = nullptr; + return std::invoke(std::forward(f)); + } catch (const std::exception& e) { + session::copy_c_str(conf->_error_buf, e.what()); + conf->last_error = conf->_error_buf; + } + if constexpr (std::is_pointer_v) + return static_cast(nullptr); + else + static_assert(std::is_void_v, "Don't know how to return an error value!"); +} + +// Same as above but accepts callbacks with value returns on errors +template +Ret wrap_exceptions(config_object* conf, Call&& f, Ret error_return) { + try { + conf->last_error = nullptr; + return std::invoke(std::forward(f)); + } catch (const std::exception& e) { + session::copy_c_str(conf->_error_buf, e.what()); + conf->last_error = conf->_error_buf; + } + return error_return; +} + } // namespace session::config namespace fmt { diff --git a/src/config/local.cpp b/src/config/local.cpp index 3b6575d33..4f8c79edf 100644 --- a/src/config/local.cpp +++ b/src/config/local.cpp @@ -1,7 +1,5 @@ #include "session/config/local.h" -#include - #include "internal.hpp" #include "session/config/error.h" #include "session/config/local.hpp" @@ -11,8 +9,8 @@ using namespace session::config; Local::Local( - std::span ed25519_secretkey, - std::optional> dumped) { + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, std::nullopt, std::nullopt); load_key(ed25519_secretkey); } diff --git a/src/config/pro.cpp b/src/config/pro.cpp index 8526e878d..f30c64d21 100644 --- a/src/config/pro.cpp +++ b/src/config/pro.cpp @@ -4,9 +4,10 @@ #include #include +#include #include +#include #include -#include namespace session::config { @@ -20,9 +21,9 @@ bool ProConfig::load(std::string_view bt_encoded) { // next proof fetch. oxenc::bt_dict_consumer d{bt_encoded}; auto expiry = d.require("e"); - auto tag = d.require_span("g"); - auto seed = d.require_span("r"); - auto sig = d.require_span("s"); + auto tag = d.require_span("g"); + auto seed = d.require_span("r"); + auto sig = d.require_span("s"); // A future proof format would take a new config key rather than an in-dict version marker // (an opaque per-key-merged value can't carry a version describing itself), so there is @@ -32,8 +33,7 @@ bool ProConfig::load(std::string_view bt_encoded) { std::memcpy(proof.sig.data(), sig.data(), proof.sig.size()); // Derive the rotating public key + full private key from the stored seed. - crypto_sign_ed25519_seed_keypair( - proof.rotating_pubkey.data(), rotating_privkey.data(), seed.data()); + ed25519::seed_keypair(proof.rotating_pubkey, rotating_privkey, seed); return true; } catch (const std::exception&) { return false; diff --git a/src/config/protos.cpp b/src/config/protos.cpp index bfd5d7af1..1a3d6a76b 100644 --- a/src/config/protos.cpp +++ b/src/config/protos.cpp @@ -1,15 +1,17 @@ #include "session/config/protos.hpp" -#include -#include +#include #include +#include #include #include +#include "../internal-util.hpp" #include "SessionProtos.pb.h" #include "WebSocketResources.pb.h" #include "session/session_encrypt.hpp" +#include "session/util.hpp" namespace session::config::protos { @@ -33,24 +35,12 @@ namespace { } // namespace -std::vector wrap_config( - std::span ed25519_sk, - std::span data, +std::vector wrap_config( + const ed25519::PrivKeySpan& ed25519_sk, + std::span data, int64_t seqno, config::Namespace t) { - std::array tmp_sk; - if (ed25519_sk.size() == 32) { - std::array ignore_pk; - crypto_sign_ed25519_seed_keypair(ignore_pk.data(), tmp_sk.data(), ed25519_sk.data()); - ed25519_sk = {tmp_sk.data(), 64}; - } else if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "Error: ed25519_sk is not the expected 64-byte Ed25519 secret key"}; - - std::array my_xpk; - if (0 != crypto_sign_ed25519_pk_to_curve25519(my_xpk.data(), ed25519_sk.data() + 32)) - throw std::invalid_argument{ - "Failed to convert Ed25519 pubkey to X25519; invalid secret key?"}; + auto my_xpk = ed25519::pk_to_x25519(ed25519_sk.pubkey()); if (static_cast(t) > 5) throw std::invalid_argument{"Error: received invalid outgoing SharedConfigMessage type"}; @@ -91,7 +81,7 @@ std::vector wrap_config( // derived from our private key, but old Session clients expect this. // NOTE: This is dumb. auto enc_shared_conf = encrypt_for_recipient_deterministic( - ed25519_sk, {my_xpk.data(), my_xpk.size()}, to_span(shared_conf)); + ed25519_sk, my_xpk, to_span(shared_conf)); // This is the point in session client code where this value got base64-encoded, passed to // another function, which then base64-decoded that value to put into the envelope. We're going @@ -121,24 +111,16 @@ std::vector wrap_config( msg.set_type(WebSocketProtos::WebSocketMessage_Type_REQUEST); *msg.mutable_request() = webreq; - return to_vector(msg.SerializeAsString()); + return to_vector(msg.SerializeAsString()); } -std::vector unwrap_config( - std::span ed25519_sk, - std::span data, +std::vector unwrap_config( + const ed25519::PrivKeySpan& ed25519_sk, + std::span data, config::Namespace ns) { // Hurray, we get to undo everything from the above! - std::array tmp_sk; - if (ed25519_sk.size() == 32) { - std::array ignore_pk; - crypto_sign_ed25519_seed_keypair(ignore_pk.data(), tmp_sk.data(), ed25519_sk.data()); - ed25519_sk = {tmp_sk.data(), 64}; - } else if (ed25519_sk.size() != 64) - throw std::invalid_argument{ - "Error: ed25519_sk is not the expected 64-byte Ed25519 secret key"}; - auto ed25519_pk = ed25519_sk.subspan(32); + auto ed25519_pk = ed25519_sk.pubkey(); WebSocketProtos::WebSocketMessage req{}; @@ -152,22 +134,20 @@ std::vector unwrap_config( if (!envelope.ParseFromString(req.request().body())) throw std::runtime_error{"Failed to parse Envelope"}; - auto [content, sender] = decrypt_incoming(ed25519_sk, to_span(envelope.content())); + auto [content, sender] = decrypt_incoming(ed25519_sk, to_span(envelope.content())); if (to_string_view(sender) != to_string_view(ed25519_pk)) throw std::runtime_error{"Incoming config data was not from us; ignoring"}; if (content.empty()) throw std::runtime_error{"Incoming config data decrypted to empty string"}; - if (!(content.back() == 0x00 || content.back() == 0x80)) + if (!(content.back() == std::byte{0x00} || content.back() == std::byte{0x80})) throw std::runtime_error{"Incoming config data doesn't have required padding"}; - if (auto it = std::find_if( - content.rbegin(), content.rend(), [](unsigned char c) { return c != 0; }); - it != content.rend() && *it == 0x80) - content.resize(content.size() - std::distance(content.rbegin(), it) - 1); - else + trim_trailing(content); + if (content.empty() || content.back() != std::byte{0x80}) throw std::runtime_error{"Incoming config data has invalid padding"}; + content.pop_back(); // the 0x80 padding terminator itself SessionProtos::Content config{}; if (!config.ParseFromArray(content.data(), content.size())) @@ -180,7 +160,7 @@ std::vector unwrap_config( throw std::runtime_error{"SharedConfig has wrong kind for config namespace"}; // if ParseFromString fails, we have a raw (not protobuf encoded) message - return to_vector(shconf.data()); + return to_vector(shconf.data()); } } // namespace session::config::protos diff --git a/src/config/user_groups.cpp b/src/config/user_groups.cpp index aa9257b4a..a3bb32324 100644 --- a/src/config/user_groups.cpp +++ b/src/config/user_groups.cpp @@ -3,8 +3,6 @@ #include #include #include -#include -#include #include #include @@ -58,7 +56,7 @@ legacy_group_info::legacy_group_info(std::string sid) : session_id{std::move(sid } community_info::community_info(const ugroups_community_info& c) : - community_info{c.base_url, c.room, std::span{c.pubkey, 32}} { + community_info{c.base_url, c.room, std::as_bytes(std::span{c.pubkey})} { base_from(*this, c); } @@ -80,8 +78,8 @@ legacy_group_info::legacy_group_info(const ugroups_legacy_group_info& c, impl_t) assert(name.size() <= NAME_MAX_LENGTH); // Otherwise the caller messed up base_from(*this, c); if (c.have_enc_keys) { - enc_pubkey.assign(c.enc_pubkey, c.enc_pubkey + 32); - enc_seckey.assign(c.enc_seckey, c.enc_seckey + 32); + enc_pubkey = to_vector(to_byte_span(c.enc_pubkey)); + enc_seckey = to_vector(to_byte_span(c.enc_seckey)); } } @@ -205,9 +203,9 @@ group_info::group_info(const ugroups_group_info& c) : id{c.id, 66} { assert(name.size() <= NAME_MAX_LENGTH); // Otherwise the caller messed up if (c.have_secretkey) - secretkey.assign(c.secretkey, c.secretkey + 64); + secretkey = to_vector(to_byte_span(c.secretkey)); if (c.have_auth_data) - auth_data.assign(c.auth_data, c.auth_data + sizeof(c.auth_data)); + auth_data = to_vector(to_byte_span(c.auth_data)); } void group_info::into(ugroups_group_info& c) const { @@ -231,12 +229,11 @@ void group_info::load(const dict& info_dict) { name.clear(); if (auto seed = maybe_span(info_dict, "K"); seed && seed->size() == 32) { - std::array pk; - pk[0] = 0x03; - secretkey.resize(64); - crypto_sign_seed_keypair(pk.data() + 1, secretkey.data(), seed->data()); - if (id != oxenc::to_hex(pk.begin(), pk.end())) + auto [pk, sk] = ed25519::keypair(std::span{*seed}.first<32>()); + if (id != "03{:x}"_format(pk)) secretkey.clear(); + else + secretkey.assign(sk.begin(), sk.end()); } if (auto sig = maybe_vector(info_dict, "s"); sig && sig->size() == 100) auth_data = std::move(*sig); @@ -280,20 +277,20 @@ void community_info::load(const dict& info_dict) { } UserGroups::UserGroups( - std::span ed25519_secretkey, - std::optional> dumped) { + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, std::nullopt, std::nullopt); load_key(ed25519_secretkey); } ConfigBase::DictFieldProxy UserGroups::community_field( - const community_info& og, std::span* get_pubkey) const { + const community_info& og, std::span* get_pubkey) const { auto record = data["o"][og.base_url()]; if (get_pubkey) { auto pkrec = record["#"]; if (auto pk = pkrec.string_view_or(""); pk.size() == 32) - *get_pubkey = std::span{ - reinterpret_cast(pk.data()), pk.size()}; + *get_pubkey = std::span{ + reinterpret_cast(pk.data()), pk.size()}; } return record["R"][og.room_norm()]; } @@ -302,11 +299,11 @@ std::optional UserGroups::get_community( std::string_view base_url, std::string_view room) const { community_info og{base_url, room}; - std::span pubkey; + std::span pubkey; if (auto* info_dict = community_field(og, &pubkey).dict()) { og.load(*info_dict); if (!pubkey.empty()) - og.set_pubkey(pubkey); + og.set_pubkey(pubkey.first<32>()); return og; } return std::nullopt; @@ -318,10 +315,8 @@ std::optional UserGroups::get_community(std::string_view partial } community_info UserGroups::get_or_construct_community( - std::string_view base_url, - std::string_view room, - std::span pubkey) const { - community_info result{base_url, room, pubkey}; + std::string_view base_url, std::string_view room, std::span pubkey) const { + community_info result{base_url, room, pubkey.first<32>()}; if (auto* info_dict = community_field(result).dict()) result.load(*info_dict); @@ -383,17 +378,10 @@ group_info UserGroups::get_or_construct_group(std::string_view pubkey_hex) const } group_info UserGroups::create_group() const { - std::array pk; - std::vector sk; - sk.resize(64); - crypto_sign_keypair(pk.data(), sk.data()); - std::string pk_hex; - pk_hex.reserve(66); - pk_hex += "03"; - oxenc::to_hex(pk.begin(), pk.end(), std::back_inserter(pk_hex)); - - group_info gr{std::move(pk_hex)}; - gr.secretkey = std::move(sk); + auto [pk, sk] = ed25519::keypair(); + + group_info gr{"03{:x}"_format(pk)}; + gr.secretkey.assign(sk.begin(), sk.end()); return gr; } @@ -446,13 +434,13 @@ void UserGroups::set(const group_info& g) { if (g.secretkey.size() == 64 && // Make sure the secretkey's embedded pubkey matches the group id: - to_string_view(std::span{g.secretkey.data() + 32, 32}) == - to_string_view(std::span{ - reinterpret_cast(pk_bytes.data() + 1), + to_string_view(std::span{g.secretkey.data() + 32, 32}) == + to_string_view(std::span{ + reinterpret_cast(pk_bytes.data() + 1), pk_bytes.size() - 1})) - info["K"] = std::span{g.secretkey.data(), 32}; + info["K"] = std::span{g.secretkey.data(), 32}; else { - info["K"] = std::span{}; + info["K"] = std::span{}; if (g.auth_data.size() == 100) info["s"] = g.auth_data; else @@ -668,7 +656,10 @@ LIBSESSION_C_API bool user_groups_get_or_construct_community( [&] { unbox(conf) ->get_or_construct_community( - base_url, room, std::span{pubkey, 32}) + base_url, + room, + std::span{ + reinterpret_cast(pubkey), 32}) .into(*comm); return true; }, diff --git a/src/config/user_profile.cpp b/src/config/user_profile.cpp index 4669d3459..860ccce24 100644 --- a/src/config/user_profile.cpp +++ b/src/config/user_profile.cpp @@ -1,8 +1,9 @@ #include "session/config/user_profile.h" -#include #include +#include + #include "internal.hpp" #include "session/config/contacts.hpp" #include "session/config/error.h" @@ -15,8 +16,8 @@ using namespace session::config; UserProfile::UserProfile( - std::span ed25519_secretkey, - std::optional> dumped) { + const ed25519::PrivKeySpan& ed25519_secretkey, + std::optional> dumped) { init(dumped, std::nullopt, std::nullopt); load_key(ed25519_secretkey); } @@ -38,7 +39,7 @@ void UserProfile::set_name(std::string_view new_name) { set_nonempty_str(data["n"], new_name); const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } void UserProfile::set_name_truncated(std::string new_name) { set_name(utf8_truncate(std::move(new_name), contact_info::MAX_NAME_LENGTH)); @@ -55,12 +56,12 @@ profile_pic UserProfile::get_profile_pic() const { pic.url = *url; if (auto* key = data[key_key].string(); key && key->size() == 32) pic.key.assign( - reinterpret_cast(key->data()), - reinterpret_cast(key->data()) + 32); + reinterpret_cast(key->data()), + reinterpret_cast(key->data()) + 32); return pic; } -void UserProfile::set_profile_pic(std::string_view url, std::span key) { +void UserProfile::set_profile_pic(std::string_view url, std::span key) { auto current_url = data["p"].string_view_or(""); auto current_key_str = data["q"].string_view_or(""); std::string_view new_key_str{reinterpret_cast(key.data()), key.size()}; @@ -75,15 +76,14 @@ void UserProfile::set_profile_pic(std::string_view url, std::span key) { +void UserProfile::set_reupload_profile_pic(std::string_view url, std::span key) { auto current_url = data["P"].string_view_or(""); auto current_key_str = data["Q"].string_view_or(""); std::string_view new_key_str{reinterpret_cast(key.data()), key.size()}; @@ -93,7 +93,7 @@ void UserProfile::set_reupload_profile_pic( return; set_pair_if(!url.empty() && key.size() == 32, data["P"], url, data["Q"], key); - data["T"] = ts_now(); + data["T"] = clock_now_s(); } void UserProfile::set_reupload_profile_pic(profile_pic pic) { @@ -118,6 +118,33 @@ std::optional UserProfile::get_nts_expiry() const { return std::nullopt; } +void UserProfile::set_nts_delete_before(std::chrono::sys_seconds before) { + set_ts(data["d"], before); + + // Deleting the messages takes their attachments with them, so an attachment instruction at or + // before this point no longer says anything and is cleared rather than left to linger. + if (get_nts_delete_attach_before() <= before) + set_ts(data["D"], std::chrono::sys_seconds{}); +} + +std::chrono::sys_seconds UserProfile::get_nts_delete_before() const { + if (auto* d = data["d"].integer(); d && *d > 0) + return as_sys_seconds(*d); + return {}; +} + +void UserProfile::set_nts_delete_attach_before(std::chrono::sys_seconds before) { + // The same redundancy from the other side: an attachment instruction already covered by the + // message one adds nothing, so it is not recorded. + set_ts(data["D"], before <= get_nts_delete_before() ? std::chrono::sys_seconds{} : before); +} + +std::chrono::sys_seconds UserProfile::get_nts_delete_attach_before() const { + if (auto* d = data["D"].integer(); d && *d > 0) + return as_sys_seconds(*d); + return {}; +} + void UserProfile::set_blinded_msgreqs(std::optional value) { std::optional current_value; if (data["M"].exists()) @@ -132,7 +159,7 @@ void UserProfile::set_blinded_msgreqs(std::optional value) { data["M"] = static_cast(*value); const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } std::optional UserProfile::get_blinded_msgreqs() const { @@ -141,6 +168,16 @@ std::optional UserProfile::get_blinded_msgreqs() const { return std::nullopt; } +// `x` is inverted -- it means *do not* notify -- so that the default is an absent key rather than a +// value in every push. +bool UserProfile::get_notify_media_saved() const { + return data["x"].integer_or(0) == 0; +} + +void UserProfile::set_notify_media_saved(bool notify) { + set_flag(data["x"], !notify); +} + std::chrono::sys_seconds UserProfile::get_profile_updated() const { if (auto t = data["t"].sys_seconds()) { if (auto T = data["T"].sys_seconds(); T && *T > *t) @@ -170,11 +207,11 @@ void UserProfile::set_pro_config(const ProConfig& pro) { const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } // A live proof means any in-flight purchase has resolved: clear the prepaid marker. - if (pro.proof.expiry_at > ts_now() && data["I"].exists()) + if (pro.proof.expiry_at > clock_now_s() && data["I"].exists()) data["I"].erase(); } @@ -184,31 +221,31 @@ bool UserProfile::remove_pro_config() { return result; } -session::ProProfileBitset UserProfile::get_profile_bitset() const { - ProProfileBitset result = {}; +session::ProProfileFlags UserProfile::get_profile_flags() const { + ProProfileFlags result = ProProfileFlags::None; if (const config::set* set = data["f"].set()) - result.data = bitset_from_set_of_int64_or_0(*set); + result = to_flags(*set); return result; } -void UserProfile::set_pro_badge(bool enabled) { - auto feature = SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE; - bool dirtied = enabled ? data["f"].set_insert(feature) : data["f"].set_erase(feature); +void UserProfile::set_profile_feature(ProProfileFlags flag, bool enabled) { + // The "f" set stores feature bit *positions*, so deflate the single-bit mask to its position. + assert(std::has_single_bit(static_cast(flag))); + auto position = std::countr_zero(static_cast(flag)); + bool dirtied = enabled ? data["f"].set_insert(position) : data["f"].set_erase(position); if (dirtied) { const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } } +void UserProfile::set_pro_badge(bool enabled) { + set_profile_feature(ProProfileFlags::ProBadge, enabled); +} + void UserProfile::set_animated_avatar(bool enabled) { - auto feature = SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR; - bool dirtied = enabled ? data["f"].set_insert(feature) : data["f"].set_erase(feature); - if (dirtied) { - const auto target_timestamp = - (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); - } + set_profile_feature(ProProfileFlags::AnimatedAvatar, enabled); } std::optional UserProfile::get_pro_access_expiry() const { @@ -251,11 +288,11 @@ void UserProfile::set_pro_access_expiry(std::optional // Confirming a live entitlement means any in-flight purchase resolved, and any long-stale // refund request is moot -- opportunistically clear both (we're already writing E anyway). - if (access_expiry_ts && *access_expiry_ts > ts_now()) { + if (access_expiry_ts && *access_expiry_ts > clock_now_s()) { if (data["I"].exists()) data["I"].erase(); if (auto* R = data["R"].integer(); R && std::chrono::sys_seconds{std::chrono::seconds{*R}} < - ts_now() - std::chrono::weeks{1}) + clock_now_s() - std::chrono::weeks{1}) data["R"].erase(); } } @@ -287,7 +324,7 @@ std::optional UserProfile::get_refund_requested() cons std::chrono::sys_seconds when{std::chrono::seconds{*R}}; // Ignore stale values: a request more than a week old is treated as absent, so a flag some // client forgot to clear cannot linger indefinitely across the account's devices. - if (when >= ts_now() - std::chrono::weeks{1}) + if (when >= clock_now_s() - std::chrono::weeks{1}) return when; } return std::nullopt; @@ -301,14 +338,14 @@ void UserProfile::set_refund_requested(std::optional w // Stamp the profile-updated timestamp so the change is time-ordered across devices. const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } std::optional UserProfile::get_pro_prepaid() const { if (auto* I = data["I"].integer()) { std::chrono::sys_seconds when{std::chrono::seconds{*I}}; // Ignore a stale marker (a purchase that never propagated) so devices don't poll forever. - if (when >= ts_now() - std::chrono::weeks{1}) + if (when >= clock_now_s() - std::chrono::weeks{1}) return when; } return std::nullopt; @@ -326,7 +363,7 @@ void UserProfile::set_pro_prepaid(std::optional when) // or a still-future access expiry); otherwise there's nothing to poll for. bool already_pro = get_pro_config().has_value(); if (!already_pro) - if (auto e = get_pro_access_expiry(); e && *e > ts_now()) + if (auto e = get_pro_access_expiry(); e && *e > clock_now_s()) already_pro = true; if (!already_pro) { data["I"] = epoch_seconds(*when); @@ -336,7 +373,7 @@ void UserProfile::set_pro_prepaid(std::optional when) if (changed) { const auto target_timestamp = (data["t"].integer_or(0) >= data["T"].integer_or(0) ? "t" : "T"); - data[target_timestamp] = ts_now(); + data[target_timestamp] = clock_now_s(); } } @@ -378,6 +415,14 @@ std::optional UserProfile::pro_renewal_target( // The nudges below are best-effort: they only make it *less likely* that two devices near a // rotating-seed period boundary race on the same renewal. A genuine collision is still resolved // by config resolution, so none of this needs to be airtight. + // + // TODO: investigate adding renewal-time jitter here, skewed per device using the account's + // device count (the same multi-device-account information that drives PFS key rotation) so that + // the first-order statistic -- the earliest device's renewal time, which is what an observer + // actually sees -- is uniformly distributed regardless of N. Naive per-device i.i.d. jitter + // would instead publicly leak N, since the min of N jitters has an N-dependent distribution. + // dev cannot do this (device count unknown there) and deliberately accepts the lesser, + // backend-only leak instead of a public one. auto near_boundary = [](std::chrono::sys_seconds t) { auto off = t.time_since_epoch() % PRO_ROTATING_SEED_PERIOD; return off <= 15s || off >= PRO_ROTATING_SEED_PERIOD - 15s; @@ -445,9 +490,9 @@ LIBSESSION_C_API user_profile_pic user_profile_get_pic(const config_object* conf LIBSESSION_C_API int user_profile_set_pic(config_object* conf, user_profile_pic pic) { std::string_view url{pic.url}; - std::span key; + std::span key; if (!url.empty()) - key = {pic.key, 32}; + key = {reinterpret_cast(pic.key), 32}; return wrap_exceptions( conf, @@ -460,9 +505,9 @@ LIBSESSION_C_API int user_profile_set_pic(config_object* conf, user_profile_pic LIBSESSION_C_API int user_profile_set_reupload_pic(config_object* conf, user_profile_pic pic) { std::string_view url{pic.url}; - std::span key; + std::span key; if (!url.empty()) - key = {pic.key, 32}; + key = {reinterpret_cast(pic.key), 32}; return wrap_exceptions( conf, @@ -489,6 +534,23 @@ LIBSESSION_C_API void user_profile_set_nts_expiry(config_object* conf, int expir unbox(conf)->set_nts_expiry(std::max(0, expiry) * 1s); } +LIBSESSION_C_API int64_t user_profile_get_nts_delete_before(const config_object* conf) { + return epoch_seconds(unbox(conf)->get_nts_delete_before()); +} + +LIBSESSION_C_API void user_profile_set_nts_delete_before(config_object* conf, int64_t before) { + unbox(conf)->set_nts_delete_before(to_sys_seconds(before)); +} + +LIBSESSION_C_API int64_t user_profile_get_nts_delete_attach_before(const config_object* conf) { + return epoch_seconds(unbox(conf)->get_nts_delete_attach_before()); +} + +LIBSESSION_C_API void user_profile_set_nts_delete_attach_before( + config_object* conf, int64_t before) { + unbox(conf)->set_nts_delete_attach_before(to_sys_seconds(before)); +} + LIBSESSION_C_API int user_profile_get_blinded_msgreqs(const config_object* conf) { if (auto opt = unbox(conf)->get_blinded_msgreqs()) return static_cast(*opt); @@ -502,6 +564,14 @@ LIBSESSION_C_API void user_profile_set_blinded_msgreqs(config_object* conf, int unbox(conf)->set_blinded_msgreqs(std::move(val)); } +LIBSESSION_C_API bool user_profile_get_notify_media_saved(const config_object* conf) { + return unbox(conf)->get_notify_media_saved(); +} + +LIBSESSION_C_API void user_profile_set_notify_media_saved(config_object* conf, bool notify) { + unbox(conf)->set_notify_media_saved(notify); +} + LIBSESSION_C_API int64_t user_profile_get_profile_updated(config_object* conf) { return epoch_seconds(unbox(conf)->get_profile_updated()); } @@ -551,11 +621,8 @@ LIBSESSION_C_API bool user_profile_remove_pro_config(config_object* conf) { return unbox(conf)->remove_pro_config(); } -LIBSESSION_C_API session_protocol_pro_profile_bitset -user_profile_get_pro_features(const config_object* conf) { - session_protocol_pro_profile_bitset result = {}; - result.data = unbox(conf)->get_profile_bitset().data; - return result; +LIBSESSION_C_API uint64_t user_profile_get_pro_features(const config_object* conf) { + return static_cast(unbox(conf)->get_profile_flags()); } LIBSESSION_C_API void user_profile_set_pro_badge(config_object* conf, bool enabled) { diff --git a/src/core.cpp b/src/core.cpp new file mode 100644 index 000000000..666159544 --- /dev/null +++ b/src/core.cpp @@ -0,0 +1,1382 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/swarm_request.hpp" +#include "session/config/namespaces.hpp" +#include "session/core/component.hpp" + +namespace session::core { + +namespace log = oxen::log; +using namespace session::sqlite; +using namespace oxen::log::literals; +static auto cat = log::Cat("core"); + +// How many recent hashes to keep per namespace per swarm node, as a position to fall back to when +// the newest ones are deleted from the swarm. Deleting more than this in a run costs one full +// retrieve from that node, which is the same thing that happens today for any other reason. +static constexpr int SWARM_HASH_HISTORY = 100; + +static cleared_b32 seed_from_words( + std::span words, const mnemonics::Mnemonics& lang) { + auto n = words.size(); + if (n != 12 && n != 13 && n != 24 && n != 25) + throw std::invalid_argument{ + "Seed phrase must be 12, 13, 24, or 25 words (got {})"_format(n)}; + + cleared_b32 result; + if (n <= 13) { + // 12 or 13 words → 16-byte seed in the lower half; upper 16 bytes are zeroed + mnemonics::words_to_bytes(words, lang, std::span(result.data(), 16)); + std::memset(result.data() + 16, 0, 16); + } else { + // 24 or 25 words → full 32-byte seed + mnemonics::words_to_bytes(words, lang, std::span(result.data(), 32)); + } + return result; +} + +predefined_seed::predefined_seed( + std::span words, const mnemonics::Mnemonics& lang) : + predefined_seed{seed_from_words(words, lang)} {} + +predefined_seed::predefined_seed( + std::span words, std::string_view lang_name) : + predefined_seed{words, mnemonics::get_language(lang_name)} {} + +void Core::NetworkDeleter::operator()(network::Network* p) const { + delete p; +} + +void Core::init() { + if (sodium_init() < 0) + throw std::runtime_error{"libsodium initialization failed!"}; + + apply_migrations(); + + for (auto* component : _comp_init) + component->init(); + + _comp_init.clear(); + + _update_polling(); +} + +void Core::register_comp_init(detail::CoreComponent* c) { + _comp_init.push_back(c); +} + +quic::Loop& Core::loop() { + return _loop; +} + +void Core::set_network(std::unique_ptr network) { + // Polling signs its retrieve requests with the account key, so attaching a network before the + // account has an identity would fail inside a background poll rather than here. Refuse at the + // call site, where the ordering mistake actually is. + if (network && !globals.have_account()) + throw no_account{}; + + // Ownership moves in via release() because the two pointer types differ deliberately: the + // parameter is a plain unique_ptr so callers can hand over a std::make_unique, while the + // member's deleter (which is just `delete`) is what keeps Network an incomplete type in + // core.hpp -- including session_network.hpp there costs ~6x the compile time per file. + _network.reset(network.release()); + _update_polling(); +} + +void Core::_update_polling() { + if (_network && !_poll_ticker) { + _poll_ticker = _loop.call_every(_poll_interval, [this] { _poll(); }); + } else if (!_network && _poll_ticker) { + _poll_ticker->stop(); + _poll_ticker.reset(); + } +} + +void Core::set_poll_interval(std::chrono::milliseconds interval) { + // Marshalled onto the loop rather than done here: this replaces the ticker, and creating or + // stopping a libevent event from a thread that is not the loop's races the loop itself. (Both + // `_poll_interval` and `_poll_ticker` are otherwise only touched there.) `Loop::call` runs it + // inline when we are already on the loop thread, so this costs nothing in that case. + _loop.call([this, interval] { + _poll_interval = interval; + if (_poll_ticker) { + _poll_ticker->stop(); + _poll_ticker.reset(); + } + _update_polling(); + }); +} + +// Order matters: the batch's results are handled in this order, so a namespace whose contents +// another one's depend on has to come first. The configs lead because a message arriving in the +// same poll may be from a contact those configs are what tells us about; among themselves, +// ConvoInfoVolatile comes last because it refers to conversations that Contacts and UserGroups +// are what establish. +static constexpr std::array POLL_NAMESPACES = { + config::Namespace::UserProfile, + config::Namespace::Contacts, + config::Namespace::UserGroups, + config::Namespace::ConvoInfoVolatile, + config::Namespace::Default, + config::Namespace::Devices, + config::Namespace::AccountPubkeys}; + +// Ceiling on continuation rounds within one poll. A well-behaved node exhausts a namespace in far +// fewer; this exists so that a node whose `more` never goes false cannot poll indefinitely. +static constexpr int POLL_MAX_ROUNDS = 20; + +void Core::_poll() { + // Non-owning: the Network is ours alone, and callbacks below must not keep it alive -- doing so + // could make the loop thread the last owner and run ~Network there. + auto* net = _network.get(); + if (!net) { + log::debug(cat, "Not polling: no network attached"); + return; + } + + log::debug(cat, "Polling swarm for {}", globals.session_id_hex()); + + net->get_swarm(globals.pubkey_x25519(), false, [this, net](auto, auto swarm) { + if (swarm.empty()) { + log::warning(cat, "Cannot poll: no swarm nodes available"); + return; + } + + _send_poll(net, swarm.front(), {POLL_NAMESPACES.begin(), POLL_NAMESPACES.end()}, 0); + }); +} + +void Core::_send_poll( + network::Network* net, + network::service_node node, + std::vector namespaces, + int round) { + + auto now_ms = epoch_ms(clock_now_ms()); + auto ed25519_hex = globals.pubkey_ed25519().hex(); + + // Build per-namespace signatures for namespaces that require authentication; index-aligned with + // `namespaces`. Empty string means no auth needed for that namespace. Signed here rather than + // once per poll because the signature covers a timestamp the storage server checks for + // freshness, so a continuation round cannot reuse the first round's. + std::vector ns_sig(namespaces.size()); + { + auto seed = globals.account_seed(); + for (size_t i = 0; i < namespaces.size(); ++i) { + auto ns_val = static_cast(namespaces[i]); + if (!retrieve_requires_auth(ns_val)) + continue; + auto to_sign = ns_signature_value("retrieve", ns_val, now_ms); + auto sig = ed25519::sign(seed.ed25519_secret(), to_span(to_sign)); + ns_sig[i] = "{:b}"_format(sig); + } + } + + // Build one batch subrequest per namespace. + nlohmann::json requests = nlohmann::json::array(); + { + auto conn = db.conn(); + + for (size_t i = 0; i < namespaces.size(); ++i) { + auto ns_val = static_cast(namespaces[i]); + nlohmann::json params = { + {"pubkey", globals.session_id_hex()}, + {"namespace", ns_val}, + }; + + if (!ns_sig[i].empty()) { + params["pubkey_ed25519"] = ed25519_hex; + params["timestamp"] = now_ms; + params["signature"] = ns_sig[i]; + } + + // The newest hash this node handed us that it still holds. Derived rather than stored + // so that deleting a hash from the swarm moves the cursor back to its predecessor on + // its own, with nothing to remember to update. A continuation round therefore picks up + // the hashes the previous round recorded, with no separate cursor to thread through. + // + // A NULL expiry is a node that did not tell us when it would drop the message, which is + // unknown rather than expired: refusing to use it would throw away a working cursor + // over a missing field. + auto last_hash = conn.prepared_maybe_get( + R"( +SELECT h.hash FROM swarm_hashes h JOIN swarm_nodes n ON n.id = h.node + WHERE h.namespace = ? AND n.pubkey = ? AND (h.expiry IS NULL OR h.expiry > ?) + ORDER BY h.id DESC LIMIT 1 +)", + ns_val, + node.remote_pubkey, + epoch_ms(clock_now_ms())); + if (last_hash) + params["last_hash"] = *last_hash; + + requests.push_back({{"method", "retrieve"}, {"params", std::move(params)}}); + } + } + + auto body_str = nlohmann::json{{"requests", std::move(requests)}}.dump(); + + log::debug( + cat, + "Retrieving {} namespaces from {} (round {}): {}", + namespaces.size(), + node.remote_pubkey.hex(), + round, + body_str); + + net->send_request( + swarm_request(node, globals.pubkey_x25519(), "batch", to_vector(body_str)), + [this, node, namespaces = std::move(namespaces), round]( + bool success, + bool timeout, + int16_t /*status_code*/, + std::vector> /*headers*/, + std::optional body) mutable { + if (!success || !body) { + log::warning( + cat, + "Swarm poll request failed: {}", + timeout ? "timed out" + : body ? *body + : "request failed"); + return; + } + + _handle_poll_response( + std::move(node), std::move(namespaces), std::move(*body), round); + }); +} + +void Core::_handle_poll_response( + network::service_node node, + std::vector namespaces, + std::string body, + int round) { + + const auto& sn_pubkey = node.remote_pubkey; + + // Namespaces the node says it has more of, to continue in another round. Collected rather than + // continued in place because the cursor each one resumes from is written below. + std::vector unfinished; + + try { + auto json = nlohmann::json::parse(body); + auto it = json.find("results"); + if (it == json.end() || !it->is_array()) + return; + + // One poll can carry several config namespaces, and each merge would otherwise dump on its + // way out. Nothing reads those intermediate states, so hold them until the whole response + // is handled. (An external caller feeding receive_messages() directly can take its own.) + auto configs_held = configs.batch(); + + auto& results = *it; + auto conn = db.conn(); + for (size_t i = 0; i < namespaces.size() && i < results.size(); ++i) { + auto ns = namespaces[i]; + auto ns_val = static_cast(ns); + + const auto& res = results[i]; + auto code_it = res.find("code"); + if (code_it == res.end() || code_it->get() != 200) { + log::warning(cat, "Retrieve of namespace {} failed: {}", ns_val, res.dump()); + continue; + } + auto body_it = res.find("body"); + if (body_it == res.end()) + continue; + auto msgs_it = body_it->find("messages"); + if (msgs_it == body_it->end() || !msgs_it->is_array()) + continue; + + // A retrieve is capped, so this says whether the node is holding more past what it + // returned. Everything above `continue`s instead, which is the distinction that + // matters: a namespace that failed or answered malformedly is not reported to its + // handler at all, while one that answered with nothing is -- "we asked and there is + // nothing" is an answer, and some handlers act on it. + bool more = false; + if (auto m = body_it->find("more"); m != body_it->end() && m->is_boolean()) + more = m->get(); + + log::debug(cat, "Retrieved {} message(s) from namespace {}", msgs_it->size(), ns_val); + + // Decode each message; keep the decoded bytes alive until after + // receive_messages() returns, since SwarmMessage::data spans + // into them. + std::vector> messages_data; + std::vector swarm_messages; + + for (const auto& msg : *msgs_it) { + auto data_it = msg.find("data"); + if (data_it == msg.end() || !data_it->is_string()) + continue; + auto& decoded = messages_data.emplace_back(); + auto b64 = data_it->get(); + decoded.reserve(oxenc::from_base64_size(b64.size())); + oxenc::from_base64(b64.begin(), b64.end(), std::back_inserter(decoded)); + + SwarmMessage swarm_msg; + swarm_msg.data = {decoded.data(), decoded.size()}; + + if (auto h = msg.find("hash"); h != msg.end() && h->is_string()) + swarm_msg.hash = h->get(); + + if (auto t = msg.find("timestamp"); t != msg.end() && t->is_number_integer()) + swarm_msg.timestamp = from_epoch_ms(t->get()); + + if (auto e = msg.find("expiry"); e != msg.end() && e->is_number_integer()) + swarm_msg.expiry = from_epoch_ms(e->get()); + + swarm_messages.push_back(std::move(swarm_msg)); + } + + // A node claiming more while returning nothing cannot be continued: there is no new + // hash to move the cursor to, so another round would ask the same question and get the + // same answer. Treat the namespace as finished instead, or a handler waiting on + // `is_final` would wait for one that never comes. + more = more && !swarm_messages.empty(); + + receive_messages(swarm_messages, ns, !more); + if (more) + unfinished.push_back(ns); + + if (!swarm_messages.empty()) { + // Only advance the cursor once the batch has been handled: the swarm filters on + // last_hash, so advancing past messages that threw would drop them permanently. + // Handling then dying before this point re-delivers the batch instead, so message + // handlers must tolerate seeing a message twice. + // + // Every hash goes in, not just the ones that produced something we kept: the cursor + // is a position in what this node returned, so leaving out what we ignored would + // park it behind those and fetch them again on every poll. Insertion order is the + // order the node returned them, which is what `id DESC` reads back. + conn.prepared_exec( + "INSERT INTO swarm_nodes (pubkey) VALUES (?) ON CONFLICT DO NOTHING", + sn_pubkey); + auto node_id = conn.prepared_get( + "SELECT id FROM swarm_nodes WHERE pubkey = ?", sn_pubkey); + + for (const auto& m : swarm_messages) { + if (m.hash.empty()) + continue; + conn.prepared_exec( + R"( +INSERT INTO swarm_hashes (namespace, node, hash, expiry) VALUES (?, ?, ?, ?) +ON CONFLICT(namespace, node, hash) DO UPDATE SET expiry = max(expiry, excluded.expiry) +)", + ns_val, + node_id, + m.hash, + m.expiry.time_since_epoch().count() > 0 + ? std::optional{epoch_ms(m.expiry)} + : std::nullopt); + } + + // An expired hash is not a cursor: the node no longer holds the message to measure + // from. + conn.prepared_exec( + "DELETE FROM swarm_hashes WHERE expiry IS NOT NULL AND expiry <= ?", + epoch_ms(clock_now_ms())); + + // And a cap on top of that, because expiry alone bounds this at every message in + // the retention window. Only the newest entry is ever read; the rest exist solely + // to walk back past hashes deleted from the swarm, so keeping more than a run of + // deletions could plausibly cover buys nothing but disk. + conn.prepared_exec( + R"( +DELETE FROM swarm_hashes + WHERE namespace = ?1 AND node = ?2 + AND id NOT IN (SELECT id FROM swarm_hashes + WHERE namespace = ?1 AND node = ?2 + ORDER BY id DESC LIMIT ?3) +)", + ns_val, + node_id, + SWARM_HASH_HISTORY); + } + } + } catch (const std::exception& e) { + log::warning(cat, "Failed to parse poll response: {}", e.what()); + return; + } + + if (unfinished.empty()) + return; + + if (round + 1 >= POLL_MAX_ROUNDS) { + log::warning( + cat, + "Stopping poll of {} after {} rounds with {} namespace(s) still reporting more", + sn_pubkey.hex(), + POLL_MAX_ROUNDS, + unfinished.size()); + return; + } + + // Deliberately not re-fetching the swarm: the cursor these resume from is this node's, so the + // continuation has to go back to the same one. + if (auto* net = _network.get()) + _send_poll(net, std::move(node), std::move(unfinished), round + 1); +} + +PfsKeyStatus Core::prefetch_pfs_keys(std::span session_id) { + // Non-owning: the Network is ours alone, and callbacks below must not keep it alive -- doing so + // could make the loop thread the last owner and run ~Network there. + auto* net = _network.get(); + if (!net) + throw std::logic_error{"prefetch_pfs_keys called without a network object"}; + + // One copy of session_id for async use; subsequently moved into lambdas. + b33 sid; + std::ranges::copy(session_id, sid.begin()); + + // Skip the fetch if the cached entry is still fresh, or a recent NAK suppresses retrying. + // Otherwise determine whether we have a stale (but usable) key or no key at all. + auto status = PfsKeyStatus::fetching; + { + auto conn = db.conn(); + if (auto row = conn.prepared_maybe_get, std::optional>( + "SELECT fetched_at, nak_at FROM pfs_key_cache WHERE session_id = ?", sid)) { + auto [fetched_at, nak_at] = *row; + if (fetched_at) { + auto age = clock_now_s() - from_epoch_s(*fetched_at); + if (age < PFS_KEY_FRESH_DURATION) { + log::debug( + cat, + "prefetch_pfs_keys: cached key for {} is still fresh ({} old), " + "skipping", + session_id, + age); + return PfsKeyStatus::fresh; + } + log::debug( + cat, + "prefetch_pfs_keys: cached key for {} is stale ({} old), re-fetching", + session_id, + age); + status = PfsKeyStatus::stale; + } else if (nak_at) { + auto age = clock_now_s() - from_epoch_s(*nak_at); + if (age < PFS_KEY_NAK_DURATION) { + log::debug( + cat, + "prefetch_pfs_keys: recent NAK for {} ({} old), skipping", + session_id, + age); + return PfsKeyStatus::nak; + } + log::debug( + cat, + "prefetch_pfs_keys: expired NAK for {} ({} old), re-fetching", + session_id, + age); + } + } else { + log::debug(cat, "prefetch_pfs_keys: no cached key for {}, fetching", session_id); + } + } + + // The swarm is indexed by the x25519 pubkey — the session_id without its 0x05 prefix. + network::x25519_pubkey x25519_pub; + std::ranges::copy(session_id.subspan<1>(), x25519_pub.begin()); + + auto now_ms = epoch_ms(clock_now_ms()); + + // AccountPubkeys (-21) allows unauthenticated retrieve: no signature needed. + nlohmann::json params = { + {"pubkey", oxenc::to_hex(session_id)}, + {"namespace", static_cast(config::Namespace::AccountPubkeys)}, + }; + + net->get_swarm( + x25519_pub, + false, + [this, net, sid = std::move(sid), params, x25519_pub](auto, auto swarm) { + if (swarm.empty()) { + log::debug(cat, "prefetch_pfs_keys: get_swarm returned empty swarm"); + _pfs_fetch_done(sid, PfsKeyFetch::failed); + return; + } + + auto body_str = params.dump(); + net->send_request( + swarm_request(swarm.front(), x25519_pub, "retrieve", to_vector(body_str)), + [this, sid = std::move(sid)]( + bool success, + bool timeout, + int16_t /*status_code*/, + std::vector> /*headers*/, + std::optional body) { + if (!success || !body) { + log::warning( + cat, + "Failed to fetch PFS keys for {}: {}", + sid, + timeout ? "timed out" + : body ? *body + : "request failed"); + _pfs_fetch_done(sid, PfsKeyFetch::failed); + return; + } + + return _handle_pfs_response(sid, std::move(*body)); + }); + }); + return status; +} + +bool Core::_store_pfs_keys( + std::span session_id, + std::span x25519_pub, + std::span mlkem768_pub) { + auto now_s = epoch_seconds(clock_now_s()); + auto conn = db.conn(); + SQLite::Transaction tx{conn.sql}; + + bool is_unchanged = conn.prepared_maybe_get( + R"( +SELECT 1 FROM pfs_key_cache +WHERE session_id = ? AND pubkey_x25519 = ? AND pubkey_mlkem768 = ? +)", + session_id, + x25519_pub, + mlkem768_pub) + .has_value(); + + conn.prepared_exec( + R"( +INSERT INTO pfs_key_cache (session_id, fetched_at, nak_at, pubkey_x25519, pubkey_mlkem768) +VALUES (?, ?, NULL, ?, ?) +ON CONFLICT(session_id) DO UPDATE SET + fetched_at = excluded.fetched_at, + pubkey_x25519 = excluded.pubkey_x25519, + pubkey_mlkem768 = excluded.pubkey_mlkem768 +)", + session_id, + now_s, + x25519_pub, + mlkem768_pub); + tx.commit(); + return !is_unchanged; +} + +void Core::_store_pfs_nak(std::span session_id) { + auto now_s = epoch_seconds(clock_now_s()); + db.conn().prepared_exec( + R"( +INSERT INTO pfs_key_cache (session_id, fetched_at, nak_at, pubkey_x25519, pubkey_mlkem768) +VALUES (?, NULL, ?, NULL, NULL) +ON CONFLICT(session_id) DO UPDATE SET nak_at = excluded.nak_at +)", + session_id, + now_s); +} + +void Core::_handle_pfs_response(std::span sid, std::string body) { + try { + auto json = nlohmann::json::parse(body); + auto msgs_it = json.find("messages"); + if (msgs_it == json.end() || !msgs_it->is_array()) { + log::warning( + cat, + "prefetch_pfs_keys: response missing or invalid " + "'messages' array"); + return; + } + + // Strip the 0x05 prefix to get the x25519 pubkey for + // signature verification. + auto x25519_pub = sid.subspan<1>(); + + // Track the most recently valid pubkeys seen across all messages. + std::optional> pk_x25519; + std::optional> pk_mlkem768; + + for (const auto& msg : *msgs_it) { + auto data_it = msg.find("data"); + if (data_it == msg.end() || !data_it->is_string()) { + log::warning( + cat, + "prefetch_pfs_keys: message missing or " + "non-string 'data' field"); + continue; + } + auto b64 = data_it->get(); + std::vector decoded; + decoded.reserve(oxenc::from_base64_size(b64.size())); + oxenc::from_base64(b64.begin(), b64.end(), std::back_inserter(decoded)); + try { + oxenc::bt_dict_consumer in{decoded}; + auto M = in.require_span("M"); + auto X = in.require_span("X"); + in.require_signature( + "~", + [&x25519_pub]( + std::span b, std::span sig) { + if (sig.size() != 64 || + !xed25519::verify(sig.first<64>(), x25519_pub, b)) + throw std::runtime_error{"signature verification failed"}; + }); + std::ranges::copy(X, pk_x25519.emplace().begin()); + std::ranges::copy(M, pk_mlkem768.emplace().begin()); + } catch (const std::exception& e) { + log::warning( + cat, + "Ignoring malformed remote account pubkey " + "message: {}", + e.what()); + } + } + + if (!pk_x25519 || !pk_mlkem768) { + log::debug( + cat, + "prefetch_pfs_keys: no valid account pubkey message " + "found in response"); + _store_pfs_nak(sid); + _pfs_fetch_done(sid, PfsKeyFetch::not_found); + return; + } + + bool changed = _store_pfs_keys(sid, *pk_x25519, *pk_mlkem768); + _pfs_fetch_done(sid, changed ? PfsKeyFetch::new_key : PfsKeyFetch::unchanged); + } catch (const std::exception& e) { + log::warning(cat, "Failed to process PFS key fetch response: {}", e.what()); + } +} + +void Core::delete_from_swarm( + std::vector hashes, std::function on_complete) { + if (hashes.empty()) { + if (on_complete) + on_complete(true); + return; + } + + // Non-owning: the Network is ours alone, and callbacks below must not keep it alive -- doing so + // could make the loop thread the last owner and run ~Network there. + auto* net = _network.get(); + if (!net) + throw std::logic_error{"delete_from_swarm: no network object"}; + + b64 sig; + { + auto seed = globals.account_seed(); + // Signed over the hashes in the order they are sent, so the two must not be reordered + // independently. + auto to_sign = delete_signature_value(hashes); + ed25519::sign(sig, seed.ed25519_secret(), std::as_bytes(std::span{to_sign})); + } + + nlohmann::json params = { + {"pubkey", globals.session_id_hex()}, + {"pubkey_ed25519", globals.pubkey_ed25519().hex()}, + {"messages", hashes}, + {"signature", "{:b}"_format(sig)}, + }; + auto body = to_vector(params.dump()); + + net->get_swarm( + globals.pubkey_x25519(), + false, + [this, net, hashes = std::move(hashes), body = std::move(body), on_complete]( + auto, auto swarm) mutable { + if (swarm.empty()) { + log::warning(cat, "Cannot delete from swarm: no swarm nodes available"); + if (on_complete) + on_complete(false); + return; + } + + net->send_request( + swarm_request( + swarm.front(), globals.pubkey_x25519(), "delete", std::move(body)), + [this, hashes = std::move(hashes), on_complete]( + bool success, + bool timeout, + int16_t status, + auto, + std::optional resp) { + if (!success) { + log::warning( + cat, + "Swarm delete failed ({}): {}", + timeout ? "timed out" : "status {}"_format(status), + resp.value_or("no response body")); + if (on_complete) + on_complete(false); + return; + } + + // Forget the cursors naming what we just deleted, so the next retrieve + // measures from the newest hash the node still holds. Done on success + // only: a failed delete leaves the messages there, and dropping the + // cursor would replay the retention window for nothing. + { + auto conn = db.conn(); + for (const auto& h : hashes) + conn.prepared_exec( + "DELETE FROM swarm_hashes WHERE hash = ?", h); + } + + if (on_complete) + on_complete(true); + }); + }); +} + +void Core::_send_to_swarm( + std::span dest_pubkey, + config::Namespace ns, + std::vector payload, + std::chrono::milliseconds ttl, + std::function swarm_hash)> on_complete) { + // Non-owning: the Network is ours alone, and callbacks below must not keep it alive -- doing so + // could make the loop thread the last owner and run ~Network there. + auto* net = _network.get(); + if (!net) + throw std::logic_error{"_send_to_swarm: no network object"}; + + auto ns_val = static_cast(ns); + auto now_ms = epoch_ms(clock_now_ms()); + + // The pubkey in the body and the swarm the request goes to must be the same account: a storage + // server answers a store for a pubkey outside its own swarm with a 421, and the retry that + // provokes cannot recover, because every node of the swarm we picked says the same thing. + nlohmann::json params = { + {"pubkey", oxenc::to_hex(dest_pubkey.begin(), dest_pubkey.end())}, + {"namespace", ns_val}, + {"data", "{:b}"_format(payload)}, + {"timestamp", now_ms}, + {"ttl", ttl.count()}, + }; + + // Signed only where the storage server actually requires it. Signing anyway would not merely + // be redundant -- a public inbox store skips signature checking entirely, so the server never + // reads it -- it would identify us as the one doing the storing. For a message deposited in + // our own swarm that distinguishes a copy we sent from one we were sent, which is precisely + // what a storage server should not be able to tell. + // + // TODO: this signs with the account key, which is only right when the destination swarm is our + // own. Storing to a group swarm (namespaces 11-14) also requires authentication, but we do not + // hold the group's key: a non-admin member signs with the subaccount token the admins issued + // them and sends it alongside as `subaccount` + `subaccount_sig`, which the server checks + // carries subaccount_access::Write (and, for a public outbox namespace, Delete as well). Until + // that exists, a group store signed here will be rejected with a 401. + bool signed_store = store_requires_auth(ns_val); + if (signed_store) { + auto to_sign = ns_signature_value("store", ns_val, now_ms); + b64 sig; + { + auto seed = globals.account_seed(); + ed25519::sign(sig, seed.ed25519_secret(), std::as_bytes(std::span{to_sign})); + } + params["pubkey_ed25519"] = globals.pubkey_ed25519().hex(); + params["sig_timestamp"] = now_ms; + params["signature"] = "{:b}"_format(sig); + } + + auto body = to_vector(params.dump()); + + log::debug( + cat, + "Storing {}B to namespace {} of {}{}", + payload.size(), + ns_val, + params["pubkey"].get(), + signed_store ? ", signed" : ""); + + // Resolve the recipient's swarm and send. + network::x25519_pubkey x25519_pub; + std::memcpy(x25519_pub.data(), dest_pubkey.data() + 1, 32); + + net->get_swarm( + x25519_pub, + false, + [net, body = std::move(body), on_complete = std::move(on_complete), x25519_pub]( + auto, auto swarm) mutable { + if (swarm.empty()) { + log::warning(cat, "Cannot store: no swarm nodes available"); + if (on_complete) + on_complete(false, std::nullopt); + return; + } + // The two values a 421 turns on: which swarm we resolved, and which of its nodes we + // picked. Read this against the "Storing ... of " line above -- a store + // rejected as misdirected means those two pubkeys are not the same account. + log::debug( + cat, + "Storing to swarm of {} via {} ({} nodes)", + x25519_pub.hex(), + swarm.front().to_string(), + swarm.size()); + + net->send_request( + swarm_request(swarm.front(), x25519_pub, "store", std::move(body)), + [on_complete = std::move(on_complete)]( + bool success, + bool timeout, + int16_t status, + auto, + std::optional resp) { + if (!success) + log::warning( + cat, + "Store request failed ({}): {}", + timeout ? "timed out" : "status {}"_format(status), + resp.value_or("no response body")); + if (!on_complete) + return; + + std::optional hash; + if (success && resp) { + try { + auto json = nlohmann::json::parse(*resp); + if (auto h = json.find("hash"); + h != json.end() && h->is_string()) + hash = h->get(); + } catch (const std::exception& e) { + log::warning( + cat, + "Could not read stored message hash: {}", + e.what()); + } + } + on_complete( + success, + hash ? std::optional{*hash} : std::nullopt); + }); + }); +} + +void Core::_do_send_dm( + int64_t message_id, + std::span recipient, + std::span content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey, + std::chrono::milliseconds ttl, + bool force_v2) { + auto fire_status = [&](MessageSendStatus status) { + if (callbacks.message_send_status) { + try { + callbacks.message_send_status(message_id, status, std::nullopt); + } catch (const std::exception& e) { + log::error(cat, "message_send_status callback threw: {}", e.what()); + } + } + }; + + // Look up cached PFS keys for the recipient. + using X = sqlite::blob_guts; + using M = sqlite::blob_guts>; + auto row = db.conn() + .prepared_maybe_get< + std::optional, + std::optional, + std::optional, + std::optional>( + "SELECT fetched_at, nak_at, pubkey_x25519, pubkey_mlkem768" + " FROM pfs_key_cache WHERE session_id = ?", + recipient); + + const b32* pfs_x25519 = nullptr; + const std::array* pfs_mlkem768 = nullptr; + if (row) { + auto& [fetched_at, nak_at, pk_x, pk_m] = *row; + if (fetched_at && pk_x && pk_m) { + pfs_x25519 = &static_cast(*pk_x); + pfs_mlkem768 = &static_cast&>(*pk_m); + } + } + + // Encrypt the message. v2 (PFS or nopfs) produces the complete wire format directly + // (0x00 0x02 | ki | E | mlkem_ct | encrypted_inner) — no protobuf wrapping. v1 uses + // encode_dm_v1 which wraps in Envelope + WebSocketMessage protobufs. + std::vector payload; + try { + auto seed = globals.account_seed(); + auto ed_sec = seed.ed25519_secret(); + + std::string_view version; + if (pfs_x25519) { + payload = encrypt_for_recipient_v2( + ed_sec, recipient, *pfs_x25519, *pfs_mlkem768, content, pro_privkey); + version = "v2 PFS"; + } else if (force_v2) { + payload = encrypt_for_recipient_v2_nopfs(ed_sec, recipient, content, pro_privkey); + version = "v2 nopfs"; + } else { + payload = encode_dm_v1(content, ed_sec, sent_timestamp, recipient, pro_privkey); + version = "v1"; + } + + log::debug( + cat, + "send_dm: message {} encrypted for {} as {} ({}B)", + message_id, + oxenc::to_hex(recipient), + version, + payload.size()); + } catch (const std::exception& e) { + log::warning(cat, "send_dm: encryption failed for message {}: {}", message_id, e.what()); + fire_status(MessageSendStatus::encrypt_failed); + return; + } + + // Dispatch to swarm. + fire_status(MessageSendStatus::sending); + try { + _send_to_swarm( + recipient, + config::Namespace::Default, + std::move(payload), + ttl, + [this, message_id](bool success, std::optional swarm_hash) { + if (callbacks.message_send_status) { + try { + callbacks.message_send_status( + message_id, + success ? MessageSendStatus::success + : MessageSendStatus::network_error, + swarm_hash); + } catch (const std::exception& e) { + log::error(cat, "message_send_status callback threw: {}", e.what()); + } + } + }); + } catch (const std::logic_error&) { + fire_status(MessageSendStatus::no_network); + } +} + +void Core::_pfs_fetch_done(std::span session_id, PfsKeyFetch result) { + if (callbacks.pfs_keys_fetched) { + try { + callbacks.pfs_keys_fetched(session_id, result); + } catch (const std::exception& e) { + // Contained so that a misbehaving callback cannot strand the queued sends below. + log::error(cat, "pfs_keys_fetched callback threw: {}", e.what()); + } + } + _flush_pending_sends(session_id); +} + +void Core::_flush_pending_sends(std::span session_id) { + auto it = _pending_sends.begin(); + while (it != _pending_sends.end()) { + if (std::ranges::equal(it->recipient, session_id)) { + auto pending = std::move(*it); + it = _pending_sends.erase(it); + _do_send_dm( + pending.id, + pending.recipient, + pending.content, + pending.sent_timestamp, + pending.pro_privkey ? ed25519::OptionalPrivKeySpan{*pending.pro_privkey} + : ed25519::OptionalPrivKeySpan{}, + pending.ttl, + pending.force_v2); + } else { + ++it; + } + } +} + +int64_t Core::send_dm( + std::span recipient_session_id, + std::span content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey, + std::chrono::milliseconds ttl, + bool force_v2) { + auto id = _next_message_id++; + + log::debug( + cat, + "send_dm: message {} to {} ({}B content)", + id, + oxenc::to_hex(recipient_session_id), + content.size()); + + // Check cache state to decide whether we can send immediately or must queue. + auto conn = db.conn(); + auto row = conn.prepared_maybe_get, std::optional>( + "SELECT fetched_at, nak_at FROM pfs_key_cache WHERE session_id = ?", + recipient_session_id); + + bool have_cached_key = false; + bool is_nak = false; + + if (row) { + auto& [fetched_at, nak_at] = *row; + if (fetched_at) + have_cached_key = true; + else if (nak_at) + is_nak = true; + } + + if (have_cached_key || is_nak) { + // Can send immediately: either we have keys (use v2 PFS) or it's a NAK (use v1 or v2 + // nopfs). + _do_send_dm(id, recipient_session_id, content, sent_timestamp, pro_privkey, ttl, force_v2); + } else if (_network) { + // No cache entry at all: need to fetch keys first. Queue the send and initiate a + // prefetch; _pfs_fetch_done() releases it when the fetch settles, whatever the outcome. + PendingSend pending; + pending.id = id; + std::ranges::copy(recipient_session_id, pending.recipient.begin()); + pending.content.assign(content.begin(), content.end()); + pending.sent_timestamp = sent_timestamp; + if (pro_privkey) { + auto& stored = pending.pro_privkey.emplace(); + std::memcpy(stored.data(), pro_privkey->data(), 64); + } + pending.ttl = ttl; + pending.force_v2 = force_v2; + _pending_sends.push_back(std::move(pending)); + + if (callbacks.message_send_status) + callbacks.message_send_status(id, MessageSendStatus::awaiting_keys, std::nullopt); + + prefetch_pfs_keys(recipient_session_id); + } else { + // No cache and no network: fire immediate failure. + if (callbacks.message_send_status) + callbacks.message_send_status(id, MessageSendStatus::no_network, std::nullopt); + } + + return id; +} + +int64_t Core::send_dm( + std::span recipient_session_id, + const SessionProtos::Content& content, + sys_ms sent_timestamp, + const ed25519::OptionalPrivKeySpan& pro_privkey, + std::chrono::milliseconds ttl, + bool force_v2) { + + auto ts = static_cast(sent_timestamp.time_since_epoch().count()); + + std::string serialized; + if (!content.has_sigtimestamp()) { + auto stamped = content; + stamped.set_sigtimestamp(ts); + serialized = stamped.SerializeAsString(); + } else { + if (content.sigtimestamp() != ts) + throw std::invalid_argument{fmt::format( + "send_dm: Content sigTimestamp ({}) disagrees with sent_timestamp ({})", + content.sigtimestamp(), + ts)}; + serialized = content.SerializeAsString(); + } + + return send_dm( + recipient_session_id, + to_span(serialized), + sent_timestamp, + pro_privkey, + ttl, + force_v2); +} + +void Core::_handle_direct_messages(std::span messages) { + if (!callbacks.message_received && !callbacks.message_decrypt_failed) + return; + + auto seed = globals.account_seed(); + auto session_id = globals.session_id(); + // Long-term X25519 pub/sec used for v2 key-indicator prefix decryption. + std::span x25519_pub{session_id.data() + 1, 32}; + auto x25519_sec = seed.x25519_key(); + + // Ed25519 secret key used for v1 envelope decryption. + auto ed_sec = seed.ed25519_secret(); + + auto fire_received = [&](ReceivedMessage out) { + if (!callbacks.message_received) + return; + try { + callbacks.message_received(std::move(out)); + } catch (const std::exception& e) { + log::error(cat, "message_received callback threw: {}", e.what()); + } + }; + + auto fire_fail = [&](const SwarmMessage& msg, MessageDecryptFailure reason) { + if (!callbacks.message_decrypt_failed) + return; + try { + callbacks.message_decrypt_failed(msg, reason); + } catch (const std::exception& e) { + log::error(cat, "message_decrypt_failed callback threw: {}", e.what()); + } + }; + + for (const auto& msg : messages) { + auto data = msg.data; + if (data.empty()) { + fire_fail(msg, MessageDecryptFailure::bad_format); + continue; + } + + if (data[0] == std::byte{0x00}) { + // Version 2 (PFS+PQ) or an unrecognised future version. + if (data.size() < 2 || data[1] != std::byte{0x02}) { + fire_fail(msg, MessageDecryptFailure::unknown_version); + continue; + } + + // Extract the 2-byte ML-KEM key indicator, then look up matching account keys. + std::array ki; + try { + ki = decrypt_incoming_v2_prefix(x25519_sec, x25519_pub, data); + } catch (const std::exception&) { + // Ciphertext is too short or otherwise structurally malformed. + fire_fail(msg, MessageDecryptFailure::bad_format); + continue; + } + + auto keys = devices.active_account_keys(ki); + + bool decrypted = false; + for (auto& key : keys) { + try { + auto result = decrypt_incoming_v2( + session_id, key.x25519_sec, key.x25519_pub, key.mlkem768_sec, data); + ReceivedMessage out; + out.hash = msg.hash; + out.timestamp = msg.timestamp; + out.expiry = msg.expiry; + out.sender_session_id = result.sender_session_id; + out.version = 2; + out.content = std::move(result.content); + out.pro_signature = result.pro_signature; + out.pfs_encrypted = true; + fire_received(std::move(out)); + decrypted = true; + break; + } catch (const DecryptV2Error&) { + // This key didn't work; try the next candidate. + } catch (const std::exception& e) { + // Unrecoverable structural error in the message itself. + log::warning(cat, "v2 direct message format error: {}", e.what()); + fire_fail(msg, MessageDecryptFailure::bad_format); + decrypted = true; // Prevent the non-PFS fallback attempt. + break; + } + } + if (!decrypted) { + // No PFS key matched; try the non-PFS fallback (sender had no PFS keys). + try { + auto result = + decrypt_incoming_v2_nopfs(session_id, x25519_sec, x25519_pub, data); + ReceivedMessage out; + out.hash = msg.hash; + out.timestamp = msg.timestamp; + out.expiry = msg.expiry; + out.sender_session_id = result.sender_session_id; + out.version = 2; + out.content = std::move(result.content); + out.pro_signature = result.pro_signature; + // pfs_encrypted remains false (default) + fire_received(std::move(out)); + } catch (const DecryptV2Error&) { + // Non-PFS fallback also failed: message cannot be read. + fire_fail(msg, MessageDecryptFailure::no_pfs_key); + } catch (const std::exception& e) { + log::warning(cat, "v2 direct message format error: {}", e.what()); + fire_fail(msg, MessageDecryptFailure::bad_format); + } + } + + } else { + // Version 1: protobuf WebSocketMessage → Envelope wire format. + try { + auto decoded = decode_dm_envelope(ed_sec, data, pro_backend::PUBKEY); + + ReceivedMessage out; + out.hash = msg.hash; + out.timestamp = msg.timestamp; + out.expiry = msg.expiry; + out.version = 1; + // Reconstruct the 33-byte (0x05-prefixed) session ID from the x25519 pubkey. + out.sender_session_id[0] = std::byte{0x05}; + std::ranges::copy(decoded.sender_x25519_pubkey, out.sender_session_id.begin() + 1); + out.content = std::move(decoded.content_plaintext); + if (decoded.envelope.flags & SESSION_PROTOCOL_ENVELOPE_FLAGS_PRO_SIG) + out.pro_signature = decoded.envelope.pro_sig; + fire_received(std::move(out)); + } catch (const std::exception& e) { + log::warning(cat, "v1 direct message decryption error: {}", e.what()); + fire_fail(msg, MessageDecryptFailure::decrypt_failed); + } + } + } +} + +void Core::receive_messages( + std::span messages, config::Namespace ns, bool is_final) { + using config::Namespace; + switch (ns) { + case Namespace::Default: _handle_direct_messages(messages); break; + case Namespace::Devices: devices.parse_device_messages(messages, is_final); break; + case Namespace::AccountPubkeys: devices.parse_account_pubkeys(messages, is_final); break; + case Namespace::UserProfile: + case Namespace::Contacts: + case Namespace::ConvoInfoVolatile: + case Namespace::UserGroups: configs.merge(ns, messages); break; + default: + log::warning( + cat, + "receive_messages: ignoring unhandled namespace {}", + static_cast(ns)); + } +} + +void Core::apply_migrations() { + auto cat = log::Cat("schema"); + + auto conn = db.conn(); + exec_query(conn.sql, R"( +CREATE TABLE IF NOT EXISTS migrations_applied ( + name TEXT PRIMARY KEY NOT NULL +) STRICT +)"); + + std::unordered_set applied; + { + SQLite::Statement st{conn.sql, "SELECT name FROM migrations_applied"}; + while (st.executeStep()) + applied.insert(get(st)); + } + + log::debug(cat, "Checking schema migrations"); + + // Core's own migrations record their bare name; an extension's are recorded as "owner:name" so + // that two sets cannot collide. A collision would not error, it would silently mark the second + // migration as already applied. Core's names deliberately stay unprefixed: prefixing them now + // would re-run every migration on every existing database. + auto apply_set = [&](std::string_view owner, + std::span migrations, + std::string_view full_schema) { + auto key_for = [&owner](std::string_view name) { + return owner.empty() ? std::string{name} : "{}:{}"_format(owner, name); + }; + + // full_schema.sql creates the schema outright; the migrations beside it are deltas that + // upgrade a database built from an *older* full_schema. Nothing builds the schema from + // nothing, and that is the point: a CREATE lives in one place rather than being duplicated + // into an initial migration that then never changes. + // + // So a database with no record of this owner is built from the full schema, with the + // migrations recorded as applied without running. Keyed on the owner rather than on the + // database being new, so an extension added to an existing database takes this path too. + // + // The marker is a row of its own rather than being inferred from the migration list, + // because that list is legitimately empty until the first delta is written -- "none of this + // owner's migrations are applied" would then be vacuously true on every open, re-running + // the full schema against tables that already exist. Migration names all begin with a + // digit, so this cannot collide with one. + auto created_key = owner.empty() ? std::string{"@created"} : "{}:@created"_format(owner); + + if (!full_schema.empty() && !applied.count(created_key) && + std::ranges::none_of(migrations, [&](const auto& m) { + return applied.count(key_for(m.name)) > 0; + })) { + try { + log::info( + cat, "Creating {} schema from full_schema", owner.empty() ? "core" : owner); + + SQLite::Transaction tx{conn.sql}; + + conn.sql.exec(std::string{full_schema}); + conn.prepared_exec("INSERT INTO migrations_applied (name) VALUES (?)", created_key); + for (const auto& m : migrations) + conn.prepared_exec( + "INSERT INTO migrations_applied (name) VALUES (?)", key_for(m.name)); + + tx.commit(); + } catch (const std::exception& e) { + log::critical( + cat, + "Creating {} schema from full_schema failed: {}", + owner.empty() ? "core" : owner, + e.what()); + throw; + } + return; + } + + for (const auto& [name, apply] : migrations) { + auto key = key_for(name); + if (applied.count(key)) { + log::debug(cat, "Schema migration {} already applied", key); + continue; + } + + try { + log::info(cat, "Applying database schema migration {}", key); + + SQLite::Transaction tx{conn.sql}; + + apply(conn, *this); + conn.prepared_exec("INSERT INTO migrations_applied (name) VALUES (?)", key); + + tx.commit(); + } catch (const std::exception& e) { + log::critical(cat, "Database schema migration '{}' failed: {}", key, e.what()); + throw; + } + } + }; + + apply_set("", schema::MIGRATIONS, schema::FULL_SCHEMA); + + std::unordered_set owners; + for (const auto& ext : _schema_extensions) { + if (ext.owner.empty() || ext.owner.find(':') != std::string_view::npos) + throw std::invalid_argument{ + "schema_extension owner must be non-empty and must not contain ':' (got '{}')"_format( + ext.owner)}; + if (!owners.insert(ext.owner).second) + throw std::invalid_argument{ + "duplicate schema_extension owner '{}': migration names would collide"_format( + ext.owner)}; + apply_set(ext.owner, ext.migrations, ext.full_schema); + } + _schema_extensions.clear(); + + log::debug(cat, "All schema migrations are applied"); +} + +} // namespace session::core diff --git a/src/core/component.cpp b/src/core/component.cpp new file mode 100644 index 000000000..e5828340a --- /dev/null +++ b/src/core/component.cpp @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +namespace session::core::detail { + +sqlite::Connection CoreComponent::conn() { + return core.db.conn(); +} + +core::callbacks& CoreComponent::cb() { + return core.callbacks; +} + +quic::Loop& CoreComponent::loop() { + return core._loop; +} + +CoreComponent::CoreComponent(Core& core) : core{core} { + core.register_comp_init(this); +} + +} // namespace session::core::detail diff --git a/src/core/configs.cpp b/src/core/configs.cpp new file mode 100644 index 000000000..5aa930be6 --- /dev/null +++ b/src/core/configs.cpp @@ -0,0 +1,497 @@ +#include "session/core/configs.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swarm_request.hpp" + +namespace session::core { + +static auto cat = oxen::log::Cat("configs"); + +namespace log = oxen::log; + +Configs::Configs(Core& core) : CoreComponent{core} {} + +Configs::~Configs() = default; + +void Configs::_load() { + if (_loaded) + return; + + auto seed = core.globals.account_seed(); + auto key = seed.ed25519_secret(); + + // Copied out rather than referenced: a sqlite::blob spans the statement's own memory, which is + // reused as the iteration advances. + std::unordered_map> dumps; + for (auto [type, data] : conn().prepared_results( + "SELECT type, data FROM config_dumps WHERE pubkey = ?", core.globals.session_id())) + dumps.emplace(std::move(type), std::vector{data.begin(), data.end()}); + + auto stored = [&dumps](std::string_view type) -> std::optional> { + if (auto it = dumps.find(std::string{type}); it != dumps.end()) + return std::span{it->second}; + return std::nullopt; + }; + + _user_profile = std::make_unique(key, stored("UserProfile")); + _contacts = std::make_unique(key, stored("Contacts")); + _convo_info_volatile = + std::make_unique(key, stored("ConvoInfoVolatile")); + _user_groups = std::make_unique(key, stored("UserGroups")); + _local = std::make_unique(key, stored("Local")); + + _loaded = true; + + // The names above are literals because a dump has to be handed to the constructor, so there is + // no object to ask for its domain until after it exists. Anything left unclaimed is therefore + // either a typo in one of them -- which would silently discard a config and resync it from the + // swarm -- or a dump written by a version that knows a config this one does not. + for (auto* conf : all()) + dumps.erase(std::string{conf->encryption_domain()}); + for (const auto& [type, _] : dumps) + log::warning(cat, "Ignoring stored config dump of unrecognised type {}", type); + + log::debug(cat, "Loaded {} config(s)", all().size()); +} + +std::vector Configs::all() { + _load(); + return {_user_profile.get(), + _contacts.get(), + _convo_info_volatile.get(), + _user_groups.get(), + _local.get()}; +} + +config::UserProfile& Configs::user_profile() { + _load(); + return *_user_profile; +} + +config::Contacts& Configs::contacts() { + _load(); + return *_contacts; +} + +config::ConvoInfoVolatile& Configs::convo_info_volatile() { + _load(); + return *_convo_info_volatile; +} + +config::UserGroups& Configs::user_groups() { + _load(); + return *_user_groups; +} + +config::Local& Configs::local() { + _load(); + return *_local; +} + +config::ConfigBase* Configs::for_namespace(config::Namespace ns) { + _load(); + switch (ns) { + case config::Namespace::UserProfile: return _user_profile.get(); + case config::Namespace::Contacts: return _contacts.get(); + case config::Namespace::ConvoInfoVolatile: return _convo_info_volatile.get(); + case config::Namespace::UserGroups: return _user_groups.get(); + default: return nullptr; + } +} + +void Configs::_store(config::ConfigBase& conf) { + if (!conf.needs_dump()) + return; + + conn().prepared_exec( + R"( +INSERT INTO config_dumps (pubkey, type, data) VALUES (?, ?, ?) +ON CONFLICT (pubkey, type) DO UPDATE SET data = excluded.data +)", + core.globals.session_id(), + conf.encryption_domain(), + conf.dump()); +} + +void Configs::store_dumps() { + for (auto* conf : all()) + _store(*conf); +} + +Configs::Batch::Batch(Configs& configs) : _configs{configs} { + _configs._batch_depth++; +} + +Configs::Batch::~Batch() { + if (--_configs._batch_depth == 0) + _configs._flush(); +} + +void Configs::_flush() { + if (_batch_depth > 0) + return; + + store_dumps(); + + // Unconditional rather than only after a merge, so that the batch a poll holds doubles as a + // sweep: a config changed locally without one gets noticed here rather than sitting unpushed. + if (needs_push()) + _schedule_push(); + + // After the dumps, so a handler never reads state that is not yet on disk. A throwing handler + // must not take the merge down with it, and the change is not redelivered -- the next merge of + // that config reports it again, and reconciliation compares rather than replays regardless. + if (!_changed.empty()) { + auto changed = std::move(_changed); + _changed.clear(); + if (cb().configs_changed) { + try { + cb().configs_changed(changed); + } catch (const std::exception& e) { + log::warning(cat, "configs_changed callback threw: {}", e.what()); + } + } + } +} + +void Configs::merge(config::Namespace ns, std::span messages) { + // A poll reports every namespace it successfully fetched, including ones that returned nothing, + // because some handlers need to know the fetch happened. Configs are not among them: there is + // no state here that a completed-but-empty poll settles, so an empty batch would only run a + // merge of nothing and a flush behind it, on every namespace, on every poll. + if (messages.empty()) + return; + + auto held = batch(); + + auto* conf = for_namespace(ns); + if (!conf) { + log::warning( + cat, + "Ignoring {} config message(s) for namespace {}, which holds no config", + messages.size(), + static_cast(ns)); + return; + } + + std::vector>> incoming; + incoming.reserve(messages.size()); + for (const auto& m : messages) + incoming.emplace_back(m.hash, m.data); + + // The returned hash set says which messages parsed, not whether any of them mattered -- a stale + // config counts as parsed. The seqno is what actually moves when a merge changes something. + auto before = conf->seqno(); + auto accepted = conf->merge(incoming); + if (conf->seqno() != before && std::ranges::find(_changed, ns) == _changed.end()) + _changed.push_back(ns); + + log::debug( + cat, + "Merged {} of {} {} config message(s); {}, {}", + accepted.size(), + incoming.size(), + conf->encryption_domain(), + conf->needs_push() ? "needs push" : "up to date", + conf->needs_dump() ? "changed" : "unchanged"); + + _flush(); +} + +void Configs::initialise_new_account() { + // Note to self starts with no conversation, which UserProfile can only say by giving it a + // negative priority. A contact's conversation exists because there is an entry for it in the + // Contacts config; UserProfile has no entry to be absent, since it exists from the moment the + // account does, so priority carries existence as well as visibility here. An account that has + // never written a note is indistinguishable from one that set 0 deliberately unless this is + // written, because the getter reports an unset value as 0. + // + // It matters that this is not a local display decision: nts_priority lives in the shared + // UserProfile config, so leaving it at the default 0 would not merely show the conversation + // here, it would make it appear on every device on the account the moment they synced. + user_profile().set_nts_priority(-1); + _flush(); +} + +std::vector Configs::_pushable() { + _load(); + return {_user_profile.get(), _contacts.get(), _convo_info_volatile.get(), _user_groups.get()}; +} + +bool Configs::needs_push() { + for (auto* conf : _pushable()) + if (conf->needs_push()) + return true; + return false; +} + +// The longest a storage server will hold a message in a namespace only its owner may write +// (oxenss TTL_MAXIMUM_PRIVATE; the limit for public namespaces is half of it). This is the ceiling +// rather than a chosen figure: a config is what a device that has been away comes back to, so there +// is nothing to be gained by expiring it sooner. +static constexpr auto CONFIG_TTL = 30 * 24h; + +void Configs::_schedule_push() { + auto now = std::chrono::steady_clock::now(); + _last_change = now; + if (_burst_started == std::chrono::steady_clock::time_point{}) + _burst_started = now; + + if (_push_scheduled) + return; + _push_scheduled = true; + _arm_push_timer(push_debounce); +} + +void Configs::_arm_push_timer(std::chrono::milliseconds delay) { + loop().call_later(delay, [this, alive = std::weak_ptr{_alive}] { + if (alive.expired()) + return; + _push_if_due(); + }); +} + +void Configs::_push_if_due() { + auto now = std::chrono::steady_clock::now(); + auto quiet = now - _last_change; + auto waited = now - _burst_started; + + if (quiet >= push_debounce || waited >= push_max_delay) { + _push_scheduled = false; + _burst_started = {}; + push_now(); + return; + } + + // Changes are still arriving, so wait for them -- but no further than the cap allows. Both + // bounds are recomputed rather than tracked, so a re-arm cannot drift past the deadline the + // first change set. + using std::chrono::duration_cast; + using std::chrono::milliseconds; + _arm_push_timer(std::min( + duration_cast(push_debounce - quiet), + duration_cast(push_max_delay - waited))); +} + +void Configs::push_now() { + if (_push_in_flight) + return; + _send_push(); +} + +void Configs::_send_push() { + // Checked here rather than at the scheduling end so that everything up to the wire still + // happens: changes are held and dumped, and needs_push() keeps reporting them, so the state + // reads as unpublished rather than as settled. + if (!push_enabled) { + log::warning(cat, "Not pushing configs: pushing is disabled"); + return; + } + + auto net = core.network(); + if (!net) { + log::debug(cat, "Not pushing configs: no network attached"); + return; + } + + // Which subrequests belong to which config, so that a result can be matched back to the config + // whose push produced it. A sequence answers positionally, so this is the only link. + struct Pending { + config::ConfigBase* conf; + config::seqno_t seqno; + size_t first; + size_t count; + }; + + auto now_ms = epoch_ms(clock_now_ms()); + auto pubkey_hex = core.globals.session_id_hex(); + auto ed25519_hex = core.globals.pubkey_ed25519().hex(); + + auto sign = [this](std::string_view value) { + b64 sig; + auto seed = core.globals.account_seed(); + ed25519::sign(sig, seed.ed25519_secret(), std::as_bytes(std::span{value})); + return "{:b}"_format(sig); + }; + + std::vector pending; + std::vector obsolete; + auto requests = nlohmann::json::array(); + + for (auto* conf : _pushable()) { + if (!conf->needs_push()) + continue; + + auto ns_val = static_cast(conf->storage_namespace()); + auto [seqno, messages, superseded] = conf->push(); + + pending.push_back({conf, seqno, requests.size(), messages.size()}); + + for (const auto& msg : messages) { + nlohmann::json params = { + {"pubkey", pubkey_hex}, + {"namespace", ns_val}, + {"data", "{:b}"_format(msg)}, + {"timestamp", now_ms}, + {"ttl", std::chrono::milliseconds{CONFIG_TTL}.count()}, + {"pubkey_ed25519", ed25519_hex}, + {"sig_timestamp", now_ms}, + {"signature", sign(ns_signature_value("store", ns_val, now_ms))}, + }; + requests.push_back({{"method", "store"}, {"params", std::move(params)}}); + } + + obsolete.insert(obsolete.end(), superseded.begin(), superseded.end()); + } + + if (pending.empty()) + return; + + // One delete for every config's obsolete hashes rather than one each: they go to the same + // pubkey's swarm, so a single delete is the same information in fewer requests. It goes last + // so that nothing is dropped before its replacement has been stored -- which is why this is a + // sequence rather than a batch, since a sequence stops at the first failure. + if (!obsolete.empty()) { + nlohmann::json params = { + {"pubkey", pubkey_hex}, + {"pubkey_ed25519", ed25519_hex}, + {"messages", obsolete}, + // Signed over the hashes in the order they are sent, so the two must not be + // reordered independently. + {"signature", sign(delete_signature_value(obsolete))}, + }; + requests.push_back({{"method", "delete"}, {"params", std::move(params)}}); + } + + auto body = to_vector(nlohmann::json{{"requests", std::move(requests)}}.dump()); + + log::debug( + cat, + "Pushing {} config(s) in {} subrequest(s), obsoleting {} message(s)", + pending.size(), + requests.size(), + obsolete.size()); + + _push_in_flight = true; + + net->get_swarm( + core.globals.pubkey_x25519(), + false, + [this, + net, + alive = std::weak_ptr{_alive}, + pending = std::move(pending), + body = std::move(body)](auto, auto swarm) mutable { + if (alive.expired()) + return; + if (swarm.empty()) { + log::warning(cat, "Cannot push configs: no swarm nodes available"); + _push_in_flight = false; + return; + } + + net->send_request( + swarm_request( + swarm.front(), + core.globals.pubkey_x25519(), + "sequence", + std::move(body)), + [this, alive, pending = std::move(pending)]( + bool success, + bool timeout, + int16_t status, + auto, + std::optional resp) { + if (alive.expired()) + return; + _push_in_flight = false; + + if (!success || !resp) { + log::warning( + cat, + "Config push failed ({}): {}", + timeout ? "timed out" : "status {}"_format(status), + resp.value_or("no response body")); + return; + } + + // A config is confirmed only if *every* message it split into was + // stored. Confirming a partial push would drop the parts that did + // land from the obsolete list while leaving the config believing it + // is clean, so the missing part would never be sent again. + try { + auto json = nlohmann::json::parse(*resp); + auto results = json.find("results"); + if (results == json.end() || !results->is_array()) { + log::warning(cat, "Config push response carried no results"); + return; + } + + for (const auto& p : pending) { + std::unordered_set hashes; + bool stored = true; + for (size_t i = p.first; stored && i < p.first + p.count; i++) { + if (i >= results->size()) { + stored = false; + break; + } + const auto& r = (*results)[i]; + auto code = r.find("code"); + auto b = r.find("body"); + if (code == r.end() || code->get() != 200 || + b == r.end()) { + stored = false; + break; + } + auto h = b->find("hash"); + if (h == b->end() || !h->is_string()) { + stored = false; + break; + } + hashes.insert(h->get()); + } + + if (!stored) { + log::warning( + cat, + "Config push: {} was not stored, leaving it dirty", + p.conf->encryption_domain()); + continue; + } + p.conf->confirm_pushed(p.seqno, std::move(hashes)); + } + } catch (const std::exception& e) { + log::warning( + cat, "Could not read config push response: {}", e.what()); + return; + } + + // Confirming changes the configs' state, and a change that arrived + // while this was in flight has re-dirtied them. + store_dumps(); + if (needs_push()) + _schedule_push(); + }); + }); +} + +} // namespace session::core diff --git a/src/core/devices.cpp b/src/core/devices.cpp new file mode 100644 index 000000000..493d80db2 --- /dev/null +++ b/src/core/devices.cpp @@ -0,0 +1,1709 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../internal-util.hpp" + +namespace session::core { + +using namespace fmt::literals; +using namespace oxen::log::literals; +using namespace session::literals; +using namespace std::literals; + +namespace log = oxen::log; +static auto cat = log::Cat("core.dev"); + +static constexpr auto dev_key = "device_unique_id"sv; + +// Set by Globals when it *generates* an account, and cleared once the device group exists. A +// restored account never sets it: its group, if it has one, belongs to devices we have not met yet. +static constexpr auto establish_key = "devices_establish_group"sv; + +void Devices::init() { + if (core.globals.get_blob_to(dev_key, self_id)) + log::info(cat, "Loaded existing unique device id: {}", self_id); + else { + random::fill(self_id); + core.globals.set(dev_key, self_id); + log::info(cat, "Generated new unique device id: {}", self_id); + } + + // Here rather than where the account is created, for two reasons: this component initialises + // after Globals, so `self_id` does not exist yet at that point, and the flag is persisted, so + // an account created by a run that died before reaching this still gets its group. + establish_group(); +} + +void Devices::_mark_group_owed() { + core.globals.set(establish_key, int64_t{1}); +} + +void Devices::establish_group() { + if (!core.globals.have_account()) + return; + if (!core.globals.get_integer(establish_key).value_or(0)) + return; + + // Not inside a transaction of our own: both of these open one. `active_device_keys` also + // generates this device's keys if it has none, which is the case being bootstrapped here. + auto keys = active_device_keys(); + auto& key = keys.front(); + active_account_keys(); // Mints the account's first shared seed if there is not one yet. + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + // `broadcast_needed` rather than a bumped seqno is what marks this for pushing: seqno tracks + // changes to our *info*, and nothing about our info has changed -- we have gone from being no + // device at all to being a registered one, which is a state transition. + // + // The descriptive fields are left empty deliberately. An application sets them through + // update_info() whenever it gets around to it, and that bumps the seqno normally; a group whose + // sole device has no description is honest about what we know, where inventing one would not + // be. + c.prepared_exec( + R"(INSERT INTO devices + (unique_id, state, seqno, timestamp, device_type, description, version, + pubkey_mlkem768, pubkey_x25519, broadcast_needed) + VALUES (?1, ?2, 1, ?3, '', '', 0, ?4, ?5, 1) + ON CONFLICT(unique_id) DO UPDATE SET + state = ?2, + broadcast_needed = 1, + pubkey_mlkem768 = ?4, + pubkey_x25519 = ?5)", + self_id, + static_cast(device::State::Registered), + epoch_seconds(clock_now_s()), + std::as_bytes(std::span{key.mlkem768_pub}), + std::as_bytes(std::span{key.x25519_pub})); + + core.globals.set(establish_key, int64_t{0}); + + tx.commit(); + + log::info(cat, "Established device group with this device ({}) as its only member", self_id); +} + +std::string Devices::device_id() const { + return oxenc::to_hex(self_id); +} + +template +consteval auto KEY_DOMAIN() = delete; +template <> +consteval auto KEY_DOMAIN() { + return "SessionDeviceKeys"_bytes; +} +template <> +consteval auto KEY_DOMAIN() { + return "SessionAccountKeys"_bytes; +} + +template Keys> +static Keys keys_from_seed(std::span seed) { + Keys keys; + auto& [x_sec, x_pub, ml_sec, ml_pub] = static_cast(keys); + + static_assert(mlkem768::PUBLICKEYBYTES == sizeof(ml_pub)); + static_assert(mlkem768::SECRETKEYBYTES == sizeof(ml_sec)); + + // Use SHAKE256 to expand the seed into separate X25519 and MLKEM-768 seeds. Domain + // separation is achieved by prepending the domain string before the seed. + cleared_array ml_seed; + hash::shake256(KEY_DOMAIN(), seed)(x_sec, ml_seed); + x25519::scalarmult_base(x_pub, x_sec); + + mlkem768::keygen(ml_pub, ml_sec, ml_seed); + + return keys; +} + +namespace { + +} // namespace + +// format_as for XWingKeys-derived types (DeviceKeys, AccountKeys), defined in session::core so +// that fmtlib's ADL-based lookup can find it when logging these types. +template Keys> +std::string format_as(const Keys& k) { + return "X25519[{:9.4}], MLKEM768[{:9.4}]"_format(k.x25519_pub, k.mlkem768_pub); +} + +Devices::DeviceKeys Devices::rotate_device_keys() { + // We store just one single seed value, then use SHAKE256 to expand it into separate X25519 + // (32B) and MLKEM-768 (64B) seeds. + cleared_b32 seed; + random::fill(seed); + + // Call this mainly to ensure that we can successfully produce keys from this seed. + auto keys = keys_from_seed(seed); + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + auto now = epoch_seconds(clock_now_s()); + c.prepared_exec("INSERT INTO device_privkeys (created, seed) VALUES (?, ?)", now, seed); + + // Update our own device row with the new pubkeys and bump seqno so the change gets broadcast. + // If no row exists yet, this is a no-op; the new pubkeys will be read from the active device + // keys when the row is first created. + c.prepared_exec( + "UPDATE devices" + " SET pubkey_mlkem768 = ?, pubkey_x25519 = ?, seqno = seqno + 1, timestamp = ?" + " WHERE unique_id = ?", + keys.mlkem768_pub, + keys.x25519_pub, + now, + self_id); + + tx.commit(); + + log::info(cat, "New rotating device keys generated: {}", keys); + + return keys; +} + +void Devices::rotate_account_keys() { + cleared_b32 seed; + random::fill(seed); + auto keys = keys_from_seed(seed); + + auto c = conn(); + c.prepared_exec( + "INSERT INTO device_account_keys (created, seed, pubkey_mlkem768, pubkey_x25519)" + " VALUES (?, ?, ?, ?)", + epoch_seconds(clock_now_s()), + seed, + keys.mlkem768_pub, + keys.x25519_pub); + + log::info(cat, "New account keys generated: {}", keys); +} + +std::vector Devices::active_device_keys() { + std::vector keys; + auto c = conn(); + bool have_active = false; + for (auto [seed, rotated] : c.prepared_results, std::optional>( + "SELECT seed, rotated FROM device_privkeys" + " ORDER BY rotated DESC NULLS FIRST, created DESC")) { + auto& k = keys.emplace_back(keys_from_seed(seed)); + if (rotated) + k.rotated.emplace(std::chrono::seconds{*rotated}); + else + have_active = true; + } + + if (!have_active) { + log::info(cat, "No currently active device keys; generating a new one"); + keys.insert(keys.begin(), rotate_device_keys()); + } + + return keys; +} + +std::vector Devices::active_account_keys( + std::optional> key_indicator) { + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + c.prepared_exec( + "DELETE FROM device_account_keys WHERE rotated < ?", + epoch_seconds(clock_now_s() - ACCOUNT_KEY_RETENTION)); + + std::vector keys; + bool have_active = false; + + auto query_all = + "SELECT id, created, rotated, seed, pubkey_mlkem768, pubkey_x25519" + " FROM device_account_keys" + " ORDER BY rotated DESC NULLS FIRST, created DESC"; + auto query_ki = + "SELECT id, created, rotated, seed, pubkey_mlkem768, pubkey_x25519" + " FROM device_account_keys" + " WHERE key_indicator = ?" + " ORDER BY rotated DESC NULLS FIRST, created DESC"; + + using cols_t = sqlite::IterableStatementWrapper< + int64_t, + int64_t, + std::optional, + sqlite::blobn<32>, + sqlite::blobn, + sqlite::blobn<32>>; + + for (auto [id, created, rotated, seed, pk_ml, pk_x] : + key_indicator ? cols_t{c.prepared_bind(query_ki, *key_indicator)} + : cols_t{c.prepared_bind(query_all)}) { + auto& k = keys.emplace_back(keys_from_seed(seed)); + k.created = std::chrono::sys_seconds{std::chrono::seconds{created}}; + if (rotated) + k.rotated.emplace(std::chrono::seconds{*rotated}); + if (!rotated) + have_active = true; + if (std::memcmp(k.mlkem768_pub.data(), pk_ml.data(), pk_ml.size()) != 0 || + std::memcmp(k.x25519_pub.data(), pk_x.data(), pk_x.size()) != 0) { + log::warning( + cat, + "device_account_keys row with id={} ignored: row contains invalid precomputed " + "pubkeys", + id); + keys.pop_back(); + } + } + + tx.commit(); + + if (!key_indicator && !have_active) { + log::info(cat, "No currently active account keys; generating a new one"); + rotate_account_keys(); + return active_account_keys(); + } + + return keys; +} + +namespace { + + // Builds a device::Info from the fields of a devices table row (excluding the row id, changes, + // and kicked_timestamp columns, which are not part of device::Info). + device::Info fill_device_info( + std::span devid, + int state, + int seqno, + int64_t timestamp, + std::string type, + std::string desc, + int64_t ver, + const sqlite::blobn& pk_ml, + const sqlite::blobn<32>& pk_x) { + device::Info info; + std::memcpy(info.id.data(), devid.data(), info.id.size()); + info.seqno = seqno; + info.timestamp = std::chrono::sys_seconds{std::chrono::seconds{timestamp}}; + info.type = device::type_from_encoded(type); + if (info.type == device::Type::Unknown) + info.other_device = std::move(type); + info.description = std::move(desc); + info.state = static_cast(state); + info.version[2] = ver % 1000; + info.version[1] = ver / 1000 % 1000; + info.version[0] = ver / 1000000; + std::memcpy(info.pk_x25519.data(), pk_x.data(), info.pk_x25519.size()); + std::memcpy(info.pk_mlkem768.data(), pk_ml.data(), info.pk_mlkem768.size()); + return info; + } + + void load_device_extras(sqlite::Connection& c, int64_t row_id, device::Info& info) { + for (auto [key, value] : c.prepared_results( + "SELECT key, bt_value FROM device_unknown WHERE device = ? ORDER BY key", + row_id)) { + try { + info.extra[key] = oxenc::bt_deserialize(value); + } catch (const std::exception& e) { + log::warning(cat, "Failed to deserialize extra device data: {}", e.what()); + } + } + } + + // Upserts a device into the devices table and updates device_unknown extras. Returns the row + // id if the record was applied, nullopt if the guard rejected it as not newer. info.id must be + // set to the 32-byte device id. + // + // The guard is `(state, seqno)` as a row value, not the seqno alone. A state change is + // invisible to the seqno -- state never goes on the wire, and is inferred from which message + // the record arrived in -- so a seqno-only guard discards exactly the transitions it is there + // to decide: an applicant is stored Pending at seqno 1, the accepting device pushes the + // identical record as Registered at seqno 1, and `1 > 1` rejects it, leaving every device + // Pending forever. + // + // Ranking the states makes the comparison decide both questions at once, and subsumes the + // special cases: equal rank falls back to the seqno, an acceptance outranks a newer link + // request, and a kick outranks everything so its tombstone -- which carries no seqno at all -- + // no longer needs an ungated update of its own. + std::optional upsert_device_info(sqlite::Connection& c, const device::Info& info) { + auto ver = info.version[0] * 1000000 + info.version[1] * 1000 + info.version[2]; + auto dev_id = c.prepared_maybe_get( + R"(INSERT INTO devices + (unique_id, state, seqno, timestamp, device_type, description, version, + pubkey_mlkem768, pubkey_x25519) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(unique_id) DO UPDATE SET + state = excluded.state, + seqno = excluded.seqno, + timestamp = excluded.timestamp, + device_type = excluded.device_type, + description = excluded.description, + version = excluded.version, + pubkey_mlkem768 = excluded.pubkey_mlkem768, + pubkey_x25519 = excluded.pubkey_x25519 + WHERE (excluded.state, excluded.seqno) > (state, seqno) + RETURNING id)", + info.id, + static_cast(info.state), + info.seqno, + info.timestamp.time_since_epoch().count(), + info.encoded_type(), + info.description, + ver, + info.pk_mlkem768, + info.pk_x25519); + + if (!dev_id) + return std::nullopt; + + c.prepared_exec("DELETE FROM device_unknown WHERE device = ?", *dev_id); + for (const auto& [key, val] : info.extra) { + auto encoded = std::visit([](const auto& v) { return oxenc::bt_serialize(v); }, val); + c.prepared_exec( + "INSERT INTO device_unknown (device, key, bt_value) VALUES (?, ?, ?)", + *dev_id, + key, + to_span(encoded)); + } + + return dev_id; + } + +} // namespace + +device::map Devices::devices( + bool include_registered, + bool include_pending, + bool include_unregistered, + std::span only_device) { + + // Encode included states as a bitmask, one bit per State value, so the query string is stable + // regardless of which states are selected. + // + // `include_unregistered` covers Kicked as well as Unregistered: the two were one state until + // the merge rules needed them apart, and a caller asking for devices that are not in the group + // means both. Separating them here is a caller-visible change worth making on its own. + int state_mask = (include_registered ? 1 << static_cast(device::State::Registered) : 0) | + (include_pending ? 1 << static_cast(device::State::Pending) : 0) | + (include_unregistered ? (1 << static_cast(device::State::Unregistered)) | + (1 << static_cast(device::State::Kicked)) + : 0); + if (state_mask == 0) + return {}; + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + device::map devs; + + std::string query = + "SELECT id, unique_id, state, seqno, timestamp, device_type, description," + " version, pubkey_mlkem768, pubkey_x25519, kicked_timestamp" + " FROM devices WHERE ((1 << state) & ?) != 0"; + if (!only_device.empty()) + query += " AND unique_id = ?"; + query += " ORDER BY unique_id"; + + auto st = c.prepared_st(query); + if (only_device.empty()) + bind_oneshot(st, state_mask); + else + bind_oneshot(st, state_mask, only_device); + + for (auto [id, devid, state, seqno, timestamp, type, desc, ver, pk_ml, pk_x, kicked] : + sqlite::IterableStatementWrapper< + int64_t, + sqlite::blob_guts>, + int, + int, + int64_t, + std::string, + std::string, + int64_t, + sqlite::blobn, + sqlite::blobn<32>, + std::optional>{std::move(st)}) { + auto& info = devs[devid]; + info = fill_device_info( + devid, state, seqno, timestamp, std::move(type), std::move(desc), ver, pk_ml, pk_x); + if (kicked) + info.kicked.emplace(std::chrono::seconds{*kicked}); + load_device_extras(c, id, info); + } + + return devs; +} + +std::pair Devices::device_info() { + auto devs = devices(true, true, true, self_id); + if (auto it = devs.find(self_id); it != devs.end()) { + // Read the state out before the move: the elements of a braced-init-list are evaluated in + // order, so testing `it->second` in the second element is testing a moved-from Info. + bool registered = it->second.state == device::State::Registered; + return {std::move(it->second), registered}; + } + return {device::Info{.id = self_id}, false}; +} + +bool device::Info::same_user_fields(const Info& other) const { + auto fields = [](const Info& i) { + return std::tie(i.type, i.other_device, i.description, i.version, i.extra); + }; + return fields(*this) == fields(other); +} + +void Devices::update_info(const device::Info& info) { + auto [current, is_registered] = device_info(); + + // Early-exit if nothing changed: no seqno bump, no push triggered. + // current.seqno == 0 means no row exists yet (default-init sentinel; real rows have seqno >= + // 1). + if (current.seqno > 0 && current.same_user_fields(info)) + return; + + auto keys = active_device_keys(); + auto& front_key = keys.front(); + auto now = clock_now_s(); + auto ver = info.version[0] * 1000000 + info.version[1] * 1000 + info.version[2]; + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + auto dev_id = c.prepared_get( + R"(INSERT INTO devices + (unique_id, state, seqno, timestamp, device_type, description, version, + pubkey_mlkem768, pubkey_x25519) + VALUES (?, ?, 1, ?, ?, ?, ?, ?, ?) + ON CONFLICT(unique_id) DO UPDATE SET + seqno = seqno + 1, + timestamp = excluded.timestamp, + device_type = excluded.device_type, + description = excluded.description, + version = excluded.version + RETURNING id)", + self_id, + static_cast(device::State::Unregistered), + now.time_since_epoch().count(), + info.encoded_type(), + info.description, + ver, + std::as_bytes(std::span{front_key.mlkem768_pub}), + std::as_bytes(std::span{front_key.x25519_pub})); + + c.prepared_exec("DELETE FROM device_unknown WHERE device = ?", dev_id); + for (const auto& [key, val] : info.extra) { + auto encoded = std::visit([](const auto& v) { return oxenc::bt_serialize(v); }, val); + c.prepared_exec( + "INSERT INTO device_unknown (device, key, bt_value) VALUES (?, ?, ?)", + dev_id, + key, + to_span(encoded)); + } + + tx.commit(); +} + +namespace { + + // Plain-old-data representation of a single account key seed entry as read from or written to + // the "K" list in the device group plaintext payload. + struct AccountKeySeed { + cleared_b32 seed; + int64_t created; + std::optional rotated; + }; + + struct GroupPayload { + device::map devices; + std::vector account_keys; + }; + + // Called while building a bt dict to pull out any unknown intermediate keys immediately before + // appending a new one. E.g. call `write_extra(out, "a", it, end)` to write out any keys from + // `it` that precede "a". `it` is mutated, and left at the first value > "a", ready for the + // next call. The iterator range must be sorted (such as a bt_dict, or a std::map, but not an unordered_map). + template End> + void write_extras(bt_dict_producer& out, std::string_view until, It& it, End end) { + for (; it != end; ++it) { + auto& [k, v] = *it; + if (auto comp = k <=> until; comp >= 0) { + if (comp == 0) + // We found an exact match, which probably means we upgraded and learned what + // the key meant. We probably shouldn't get here at all, but just in case skip + // it so we don't break the bt_dict. + ++it; + return; + } + out.append_bt(k, v); + } + } + + // Combines a call to write_extras + out.append for appending simple bt dict keys with scalar + // values. + template End> + void write_next( + oxenc::bt_dict_producer& out, std::string_view key, const T& value, It& it, End end) { + write_extras(out, key, it, end); + out.append(key, value); + } + + // Encodes the fields of a device::Info into an already-opened bt_dict_producer (passed as + // rvalue to allow callers to pass sub-producers directly from append_dict()). + void encode_device_info(oxenc::bt_dict_producer&& devout, const device::Info& info) { + auto xit = info.extra.cbegin(); + auto xend = info.extra.cend(); + write_next(devout, "#", info.seqno, xit, xend); + write_next(devout, "@", info.timestamp.time_since_epoch().count(), xit, xend); + write_next(devout, "M", info.pk_mlkem768, xit, xend); + write_next(devout, "X", info.pk_x25519, xit, xend); + write_next(devout, "d", info.description, xit, xend); + write_extras(devout, "t", xit, xend); + if (auto t = info.encoded_type(); !t.empty()) + devout.append("t", t); + auto ver = info.version[0] * 1000000 + std::clamp(info.version[1], 0, 999) * 1000 + + std::clamp(info.version[2], 0, 999); + write_extras(devout, "v", xit, xend); + if (ver != 0) + devout.append("v", ver); + for (; xit != xend; ++xit) + devout.append_bt(xit->first, xit->second); + } + + std::string encode_group_payload( + const device::map& devices, std::span acc_keys) { + oxenc::bt_dict_producer out; + + { + auto devs = out.append_dict("D"); + for (const auto& [id, info] : devices) { + + std::string_view id_sv{reinterpret_cast(id.data()), id.size()}; + + if (info.state == device::State::Pending) { + log::debug( + cat, + "Skipping pending device {} in device group data", + oxenc::to_hex(id)); + continue; + } else if (info.state == device::State::Kicked) { + // A kicked device goes in as a bare timestamp: that is how every other device + // learns of the removal, since a record merely absent from a message means + // "unchanged" rather than "removed". + // + // TODO: we should stop writing devices kicked a long time ago. Pruning means + // dropping them from *this payload* and never from the table -- the budget is + // on what the message carries, a local row costs nothing, and forgetting one + // would lower its rank and let a stale group resurrect the device. + assert(info.kicked); + devs.append(id_sv, info.kicked->time_since_epoch().count()); + continue; + } else if (info.state == device::State::Unregistered) { + // Never in the group rather than removed from it, so there is nothing to say + // about it: our own row before the group exists, and nothing else. + log::debug(cat, "Skipping unregistered device {}", oxenc::to_hex(id)); + continue; + } + + encode_device_info(devs.append_dict(id_sv), info); + } + } // "D" dict closed here + + if (!acc_keys.empty()) { + auto kl = out.append_list("K"); + for (const auto& k : acc_keys) { + auto e = kl.append_dict(); + e.append("c", k.created); + if (k.rotated) + e.append("r", *k.rotated); + e.append("s", k.seed); + } + } + + return std::move(out).str(); + } + + std::string encode_link_request_plaintext( + std::span device_id, const device::Info& info) { + oxenc::bt_dict_producer out; + // "I" (device id) sorts before "i" (info dict) + out.append("I", device_id); + encode_device_info(out.append_dict("i"), info); + return std::move(out).str(); + } + + // Stores the current btdc key/value in `extra`; the value is consumed (i.e. the consumer + // advances to the next key). + void consume_extra(oxenc::bt_dict_consumer& btdc, oxenc::bt_dict& extra) { + auto& x = extra[std::string{btdc.key()}]; + if (btdc.is_string()) + x = btdc.consume_string(); + else if (btdc.is_unsigned_integer()) + x = btdc.consume_integer(); + else if (btdc.is_integer()) + x = btdc.consume_integer(); + else if (btdc.is_dict()) + x = btdc.consume_dict(); + else + x = btdc.consume_list(); + } + + // Consumes and stores any unknown extra fields from `btdc` up to (but not including) `key` into + // `extras` + void read_extras(oxenc::bt_dict_consumer& btdc, std::string_view key, oxenc::bt_dict& extra) { + while (!btdc.is_finished() && btdc.key() < key) + consume_extra(btdc, extra); + } + + void decode_one(device::Info& info, oxenc::bt_dict_consumer dev, device::State state) { + info.state = state; + read_extras(dev, "#", info.extra); + info.seqno = dev.require("#"); + + read_extras(dev, "@", info.extra); + info.timestamp = std::chrono::sys_seconds{std::chrono::seconds{dev.require("@")}}; + + read_extras(dev, "M", info.extra); + auto M = dev.require_span("M"); + std::memcpy(info.pk_mlkem768.data(), M.data(), M.size()); + + read_extras(dev, "X", info.extra); + auto X = dev.require_span("X"); + std::memcpy(info.pk_x25519.data(), X.data(), X.size()); + + read_extras(dev, "d", info.extra); + info.description = dev.maybe("d").value_or(""sv); + + read_extras(dev, "t", info.extra); + auto type = dev.maybe("t").value_or(""sv); + info.type = device::type_from_encoded(type); + info.other_device = info.type == device::Type::Unknown ? std::string{type} : std::string{}; + + read_extras(dev, "v", info.extra); + auto ver = dev.maybe("v").value_or(0); + info.version[0] = ver / 1000000; + info.version[1] = ver / 1000 % 1000; + info.version[2] = ver % 1000; + + while (!dev.is_finished()) + consume_extra(dev, info.extra); + } + + // Decodes the plaintext bt-encoded device group payload. The returned device map will include + // both full device records and tombstoned devices: the latter have a mostly default-constructed + // Info where only id, state (=State::Kicked), and kicked (=removal timestamp) are set. + GroupPayload decode_group_payload(std::span data) { + GroupPayload result; + + oxenc::bt_dict_consumer in{data}; + auto devs = in.require("D"); + + while (!devs.is_finished()) { + auto in_id = devs.key(); + if (in_id.size() != 32) + throw std::runtime_error{ + "Invalid encoded device data: unexpected {}-byte key in device dict (expected 32)"_format( + in_id.size())}; + + std::array id; + std::memcpy(id.data(), in_id.data(), 32); + auto [it, ins] = result.devices.try_emplace(id); + if (!ins) + throw std::runtime_error{"Invalid encoded device data: duplicate device ids"}; + + auto& info = it->second; + info.id = id; + + if (devs.is_integer()) { + // An integer indicates a "device removed" timestamp, used to distinguish between + // "device removed" and "I don't know about the device yet". It gets pruned when + // updating once it hits a certain age threshold. + // + // If the device wants to get re-added to the group then it must generate a new + // device id. + info.state = device::State::Kicked; + info.kicked.emplace(std::chrono::seconds{devs.consume_integer()}); + } else { + decode_one(info, devs.consume_dict_consumer(), device::State::Registered); + } + } + + auto kl = in.require("K"); + while (!kl.is_finished()) { + auto& k = result.account_keys.emplace_back(); + auto e = kl.consume_dict_consumer(); + k.created = e.require("c"); + k.rotated = e.maybe("r"); + auto s = e.require_span("s"); + std::memcpy(k.seed.data(), s.data(), 32); + } + + return result; + } + + // Values for the devices.processing column, set during batch message processing and cleared + // after callbacks are fired at is_final. + enum class Processing { + LinkRequest = 1, // new/updated link request received + Registered = 2, // device newly transitioned to Registered + Removed = 3, // device newly transitioned to Unregistered + }; + + constexpr std::string_view format_as(Processing p) { + switch (p) { + case Processing::LinkRequest: return "link-request"; + case Processing::Registered: return "registered"; + case Processing::Removed: return "removed"; + } + return "unknown"; + } + + constexpr auto PERS_DEV_NONCE = "SessionDevDNonce"_b2b_pers; + constexpr auto PERS_KEY_NONCE = "SessionDevKNonce"_b2b_pers; + constexpr auto PERS_KEY_KEY = "SessionDevKeyKey"_b2b_pers; + constexpr auto PERS_KEY_KEY_IDX = "SessionDevKeyIdx"_b2b_pers; + constexpr auto PERS_ACC_KEY_ROT = "SessionAccKeyRot"_b2b_pers; + + // Device group payloads are null-padded to a multiple of this before encryption so that the + // encrypted size reveals only which bucket the payload falls in, not what it contains. A + // bucket is four devices at a budget of 1600 bytes each, a deliberate overestimate of a + // ~1341-byte record, with the remainder of a bucket left for removal tombstones. + constexpr size_t DEVICE_PAYLOAD_PADDING = 4 * 1600; + + // Added before the buckets: the account key list is carried by every payload whatever the + // device count, and is bounded by the rotation period and retention window -- at most 33 + // entries of ~70 bytes. Without a fixed allowance for it, it would consume most of the first + // bucket and the bucketing would stop meaning what it is supposed to mean. + constexpr size_t ACCOUNT_KEYS_ALLOWANCE = 2300; + + constexpr int bt_bytes_encoded(int x) { + int sz = 1 + x; + + do { + ++sz; + } while (x /= 10); + + return sz; + } + + static_assert(bt_bytes_encoded(0) == 2); // "0:" + static_assert(bt_bytes_encoded(9) == 11); // "9:…" + static_assert(bt_bytes_encoded(10) == 13); // "10:…" + static_assert(bt_bytes_encoded(99) == 102); // "99:…" + static_assert(bt_bytes_encoded(100) == 104); // "100:…" + +} // namespace + +std::vector Devices::encrypt_device_data(const device::map& devices) { + cleared_b32 a; + random::fill(a); + + auto A = x25519::scalarmult_base(a); + + // Who can read this, which is not the same as who appears in it. A kicked device is written + // into the payload -- a tombstone carrying when it was kicked is how every other device learns + // it is gone -- but must not be given a key, which is the entire point of removing it. A + // pending device is in neither: it is not in the group until someone accepts it. + std::vector recipients; + for (const auto& [id, info] : devices) + if (info.state == device::State::Registered) + recipients.push_back(&info); + + int padded_count = recipients.size(); + padded_count = (padded_count + 3) / 4 * 4; + + auto indices = std::views::iota(0, padded_count); + + // We randomize the positions of devices (and padding) in the list of keys, so build a random + // mapping first so that we place everything directly into its final position through it: + std::vector pos_map{indices.begin(), indices.end()}; + std::ranges::shuffle(pos_map, csrng); + + // Holds MLKEM ciphertexts: + std::vector ciphertext_raw; + ciphertext_raw.resize(mlkem768::CIPHERTEXTBYTES * padded_count); + // Holds per-device-encrypted copies of the base key, each prefixed with a 2-byte key indicator + // hash: + std::vector enc_key_raw; + enc_key_raw.resize((2 + 32) * padded_count); + + // Accessor for the relevant, position-mapped subspan of ciphertext_raw/enc_key_raw containing + // the location of index i as a subspan of the raw vector: + auto ciphertext = indices | std::views::transform([&](int i) { + return std::span{ + ciphertext_raw.data() + pos_map[i] * mlkem768::CIPHERTEXTBYTES, + mlkem768::CIPHERTEXTBYTES}; + }); + + auto enc_indicator = + indices | std::views::transform([&](int i) { + return std::span{enc_key_raw.data() + pos_map[i] * (2 + 32), 2}; + }); + + auto enc_key = + indices | std::views::transform([&](int i) { + return std::span{enc_key_raw.data() + pos_map[i] * (2 + 32) + 2, 32}; + }); + + cleared_vector ml_ss_raw(mlkem768::SHAREDSECRETBYTES * recipients.size()); + + // Dynamic ss subspan accessor of ml_ss_raw, but *doesn't* go through the pos_map (unlike the + // above constructs), and only goes up to the actual number of devices, not the padded number + // (because this is never transmitted, and so not shuffled or padded). + auto ml_ss = + std::views::iota(size_t{0}, recipients.size()) | std::views::transform([&](size_t i) { + return std::span{ + ml_ss_raw.data() + i * mlkem768::SHAREDSECRETBYTES, + mlkem768::SHAREDSECRETBYTES}; + }); + + cleared_b32 rnd; + int i = -1; + for (const auto* info : recipients) { + ++i; + random::fill(rnd); + mlkem768::encapsulate(ciphertext[i], ml_ss[i], info->pk_mlkem768, rnd); + } + // Fill padding entries with randomness. `++i` first: the loop above leaves `i` on the last + // real entry, and starting here would overwrite it with noise -- which nothing detects when + // there is more than one recipient, because some other slot still decrypts. + for (++i; i < padded_count; i++) + random::fill(ciphertext[i]); + + std::array nonce; + hash::blake2b_key_pers(nonce, A, PERS_DEV_NONCE, ciphertext_raw); + + cleared_b32 key_base; + random::fill(key_base); + + // Fetch account key seeds for inclusion in the payload. + std::vector acc_keys; + for (auto [seed, created, rotated] : + conn().prepared_results, int64_t, std::optional>( + "SELECT seed, created, rotated FROM device_account_keys" + " ORDER BY rotated DESC NULLS FIRST, created DESC")) { + auto& k = acc_keys.emplace_back(); + std::memcpy(k.seed.data(), seed.data(), 32); + k.created = created; + k.rotated = rotated; + } + + auto plaintext_devices = encode_group_payload(devices, acc_keys); + // 2300 + 6400N: at least one bucket, so a payload smaller than the account key allowance still + // pads up rather than down to nothing. + auto buckets = std::max( + 1, + (plaintext_devices.size() - std::min(plaintext_devices.size(), ACCOUNT_KEYS_ALLOWANCE) + + DEVICE_PAYLOAD_PADDING - 1) / + DEVICE_PAYLOAD_PADDING); + plaintext_devices.resize(ACCOUNT_KEYS_ALLOWANCE + buckets * DEVICE_PAYLOAD_PADDING); + + std::vector enc_devices; + enc_devices.resize(plaintext_devices.size() + encryption::XCHACHA20_ABYTES); + encryption::xchacha20poly1305_encrypt(enc_devices, to_span(plaintext_devices), nonce, key_base); + + cleared_b32 ki; + cleared_b32 aB; + i = -1; + for (const auto* info : recipients) { + ++i; + auto eind = enc_indicator[i]; + auto ekey = enc_key[i]; + auto ct = ciphertext[i]; + + auto& B = info->pk_x25519; + if (!x25519::scalarmult(aB, a, B)) { + // This really shouldn't happen: we shouldn't have accepted an invalid pubkey in the + // first place. + log::error( + cat, + "X25519 scalarmult failed: device '{}' ({}) published an invalid X25519 " + "pubkey!", + oxenc::to_hex(info->id), + info->description); + // Without a proper key, we can't properly encrypt for the device so we'll just have to + // fill the entry with random and move on. + random::fill(eind); + random::fill(ekey); + continue; + } + + hash::blake2b_key_pers(nonce, A, PERS_KEY_NONCE, ct, enc_devices); + hash::blake2b_pers(ki, PERS_KEY_KEY, aB, A, B, ml_ss[i], info->pk_mlkem768); + + static_assert(decltype(ekey)::extent == key_base.size()); + encryption::xchacha20_xor(ekey, key_base, nonce, ki); + + // Hash a bunch of stuff together as a checksum to let decryption skip most not-for-me + // values. + hash::blake2b_pers(eind, PERS_KEY_KEY_IDX, A, B, info->pk_mlkem768, ct, ekey); + } + // Fill padding entries with randomness; `++i` for the same reason as above. + for (++i; i < padded_count; i++) { + random::fill(enc_indicator[i]); + random::fill(enc_key[i]); + } + + // We're done: now we just need to encode everything together: + std::vector out; + out.resize( + 2 // Outer "d" ... "e" delimiters + + 5 // "0:" + "1:G" (message type indicator) + + 3 + bt_bytes_encoded(A.size()) // "1:A" + "32:...(A eph pk)..." + + 3 + bt_bytes_encoded(ciphertext_raw.size()) // "1:C" + "NNNN:...(mlkem cts)..." + + 3 + bt_bytes_encoded(enc_key_raw.size()) // "1:K" + "NNN:...(encrypted keys)..." + + 3 + bt_bytes_encoded(enc_devices.size()) // "1:d" + "MMMM:...(enc device info)..." + + 3 + bt_bytes_encoded(64) // "1:~" + "64:...(Ed25519 signature)..." + ); + + oxenc::bt_dict_producer o{reinterpret_cast(out.data()), out.size()}; + + o.append("", "G"); + o.append("A", A); + o.append("C", ciphertext_raw); + o.append("K", enc_key_raw); + o.append("d", enc_devices); + o.append_signature("~", [seed = core.globals.account_seed()](std::span body) { + return ed25519::sign(seed.ed25519_secret(), body); + }); + + assert(o.view().size() == out.size()); // Ensure we calculated exactly the right size above + + return out; +} + +// Prebuilt SQL with Processing/State enum values embedded as literals rather than parameters. +// Records a device as kicked, inserting a bare tombstone row if we hold no record of it. +// +// The insert half is what makes a removal durable for a device that joined after it: an update +// alone matches nothing, stores nothing, and leaves that device free to accept the removed one back +// into the group. A tombstone needs no details to do its job -- rank alone settles the merge -- so +// the columns the schema requires are filled with zeroes and the seqno left at 0, which no record +// off the wire can be. +// +// `processing` is set only where the device was Registered: a removal is news to the application +// only if we thought the device was a member, and a tombstone for one we never knew is not. +static const std::string KICK_DEVICE_SQL = + "INSERT INTO devices" + " (unique_id, state, seqno, timestamp, device_type, description, version," + " pubkey_mlkem768, pubkey_x25519, kicked_timestamp)" + " VALUES (?2, {0}, 0, ?1, '', '', 0, zeroblob(1184), zeroblob(32), ?1)" + " ON CONFLICT(unique_id) DO UPDATE SET" + " state = {0}, kicked_timestamp = excluded.kicked_timestamp," + " processing = CASE WHEN state = {1} THEN {2} ELSE processing END," + " broadcast_needed = CASE WHEN state = {1} THEN 1 ELSE broadcast_needed END"_format( + static_cast(device::State::Kicked), + static_cast(device::State::Registered), + static_cast(Processing::Removed)); + +static const std::string REGISTER_DEVICE_SQL = + "UPDATE devices SET processing = {}, broadcast_needed = 1 WHERE id = ?"_format( + static_cast(Processing::Registered)); + +// Restates a removal that an incoming message tried to undo, moving the tombstone to the front of +// the removed list and marking it for broadcast. +// +// Refusing the record locally is not enough: a device that never saw the removal -- offline at the +// time, or having since pruned the tombstone -- has no reason to refuse it, and would go on +// treating the device as a member and encrypting to it. Nothing reconciles that afterwards, since +// a record absent from a message means "unchanged" rather than "removed", so the two devices would +// hold permanently different groups. A tombstone propagates where a refusal does not. +// +// The timestamp is moved to now rather than left at the original removal, because retention keeps +// the most recently removed: a tombstone left to age could be evicted while the device it names is +// still trying to return, either by waiting out the window or by provoking enough other removals to +// displace it. +static const std::string REASSERT_KICK_SQL = + "UPDATE devices SET kicked_timestamp = ?, broadcast_needed = 1 WHERE unique_id = ?"; + +void Devices::receive_device_group_message(std::span data) { + GroupPayload payload; + try { + auto raw = decrypt_device_data(std::as_bytes(data)); + payload = decode_group_payload(raw); + } catch (const device::decryption_failed& e) { + log::warning(cat, "Ignoring incoming device group message: {}", e.what()); + return; + } + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + // Merge incoming account keys. New seeds are inserted and the rotation trigger applies + // tie-breaking: latest created wins (smallest seed as tiebreaker), so concurrent rotations + // from multiple devices converge deterministically. For seeds we already have, we reconcile + // the `rotated` column: if both sides have rotated at different times, take the minimum; if + // only one side has rotated, adopt that rotation. + for (const auto& k : payload.account_keys) { + auto keys = keys_from_seed(k.seed); + c.prepared_exec( + "INSERT INTO device_account_keys" + " (created, rotated, seed, pubkey_mlkem768, pubkey_x25519)" + " VALUES (?, ?, ?, ?, ?)" + " ON CONFLICT (seed) DO UPDATE SET" + " rotated = COALESCE(MIN(excluded.rotated, rotated), excluded.rotated, rotated)", + k.created, + k.rotated, + k.seed, + keys.mlkem768_pub, + keys.x25519_pub); + } + + for (const auto& [id, info] : payload.devices) { + if (info.state == device::State::Kicked) { + // Whatever details we already hold are kept; only the state and the timestamp move. A + // device we have never heard of gets a bare tombstone -- see KICK_DEVICE_SQL. + assert(info.kicked); + c.prepared_exec(KICK_DEVICE_SQL, info.kicked->time_since_epoch().count(), id); + continue; + } + + // A removal is one-way: a device we hold a tombstone for cannot be returned to the group by + // a record in a message, only by a fresh link request under a new device id. Anything + // claiming otherwise is either a device that was removed and is re-adding itself -- it + // still holds the account seed, so it can sign and push whatever it likes -- or a device + // relaying such a record. Either way the answer is to restate the removal rather than to + // adopt it. + // + // Asks the state, which now says only this: `Kicked` is removal and nothing else, where + // `Unregistered` covers a device that was never in the group -- our own row before the + // group is established, and an ignored link request -- both of which must still be able to + // register. + auto kicked = c.prepared_maybe_get("SELECT state FROM devices WHERE unique_id = ?", id) + .value_or(-1) == static_cast(device::State::Kicked); + if (kicked) { + log::warning( + cat, + "Device group message tried to restore removed device {}; restating removal", + oxenc::to_hex(id)); + c.prepared_exec(REASSERT_KICK_SQL, epoch_seconds(clock_now_s()), id); + continue; + } + + // Check state before upsert to detect a registration transition. + bool was_registered = + c.prepared_maybe_get("SELECT state FROM devices WHERE unique_id = ?", id) + .value_or(-1) == static_cast(device::State::Registered); + + auto dev_id = upsert_device_info(c, info); + if (!dev_id) + continue; + + // Mark as newly registered only on a state transition (not for info-only updates). + if (!was_registered) + c.prepared_exec(REGISTER_DEVICE_SQL, *dev_id); + } + + tx.commit(); +} + +Devices::LinkRequestResult Devices::build_link_request() { + auto [info, is_registered] = device_info(); + + if (is_registered) + throw std::logic_error{ + "build_link_request() called on a device that is already registered in the device " + "group"}; + + info.id = self_id; + info.seqno++; + info.timestamp = clock_now_s(); + + // Always use the current active device keys for the pubkeys in the link request, regardless + // of what is stored in the DB, as the DB may lag a key rotation. + auto keys = active_device_keys(); + std::memcpy( + info.pk_x25519.data(), + reinterpret_cast(keys.front().x25519_pub.data()), + info.pk_x25519.size()); + std::memcpy( + info.pk_mlkem768.data(), + reinterpret_cast(keys.front().mlkem768_pub.data()), + info.pk_mlkem768.size()); + + // Upsert our own device row with the updated seqno, timestamp, and pubkeys. The pending link + // request is detectable via state=Pending on our own row; needs_push() detects dirty state via + // the seqno increment above exceeding pushed_seqno. + auto c = conn(); + auto ver = info.version[0] * 1000000 + info.version[1] * 1000 + info.version[2]; + c.prepared_exec( + R"(INSERT INTO devices + (unique_id, state, seqno, timestamp, device_type, description, version, + pubkey_mlkem768, pubkey_x25519) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(unique_id) DO UPDATE SET + state = excluded.state, + seqno = excluded.seqno, + timestamp = excluded.timestamp, + device_type = excluded.device_type, + description = excluded.description, + version = excluded.version, + pubkey_mlkem768 = excluded.pubkey_mlkem768, + pubkey_x25519 = excluded.pubkey_x25519)", + self_id, + static_cast(device::State::Pending), + info.seqno, + info.timestamp.time_since_epoch().count(), + info.encoded_type(), + info.description, + ver, + info.pk_mlkem768, + info.pk_x25519); + + auto plaintext = encode_link_request_plaintext(self_id, info); + auto sas = link_request_sas(to_span(plaintext)); + + // Encrypt the plaintext + std::vector encrypted(plaintext.size() + config::ENCRYPT_DATA_OVERHEAD); + std::memcpy(encrypted.data(), plaintext.data(), plaintext.size()); + auto seed = core.globals.account_seed(); + config::encrypt_prealloced(encrypted, seed.seed(), "link-request"); + + // Wrap in outer bt-dict: {"": "L", "L": } + std::vector out( + 2 // Outer "d" ... "e" delimiters + + 5 // "0:" + "1:L" (message type indicator) + + 3 + bt_bytes_encoded(encrypted.size()) // "1:L" + "NNN:...(encrypted blob)..." + ); + oxenc::bt_dict_producer o{reinterpret_cast(out.data()), out.size()}; + o.append("", "L"); + o.append("L", std::span{encrypted}); + assert(o.view().size() == out.size()); + + return {std::move(out), sas}; +} + +std::vector Devices::decrypt_device_data(std::span enc_data) { + + oxenc::bt_dict_consumer in{enc_data}; + in.require(""); // skip the "" type key added by the outer wrapper + auto A = in.require_span("A"); + auto ciphertext_raw = in.require_span("C"); + auto enc_key_raw = in.require_span("K"); + auto enc_devices = in.require_span("d"); + + in.require_signature( + "~", [this](std::span body, std::span sig) { + if (sig.size() != 64 || + !ed25519::verify(sig.first<64>(), core.globals.pubkey_ed25519(), body)) + throw std::runtime_error{ + "Invalid encrypted device message: signature verification failed"}; + }); + + in.finish(); + + if (ciphertext_raw.size() % mlkem768::CIPHERTEXTBYTES != 0) + throw std::runtime_error{ + "Invalid encrypted device group data: invalid ciphertext size ({} is not N*{})"_format( + ciphertext_raw.size(), mlkem768::CIPHERTEXTBYTES)}; + const int count = ciphertext_raw.size() / mlkem768::CIPHERTEXTBYTES; + if (enc_key_raw.size() % (32 + 2) != 0) + throw std::runtime_error{ + "Invalid encrypted device group data: invalid encrypted keys size ({} is not N*34)"_format( + enc_key_raw.size())}; + if (const int k_count = enc_key_raw.size() / (32 + 2); count != k_count) + throw std::runtime_error{ + "Invalid encrypted device data: ciphertext ({}) vs enc key ({}) size mismatch"_format( + count, k_count)}; + if (enc_devices.size() <= encryption::XCHACHA20_ABYTES) + throw std::runtime_error{ + "Invalid encrypted device data: encrypted data is too short ({}B)"_format( + enc_devices.size())}; + + auto indices = std::views::iota(0, count); + + // Accessors for chunk-by-chunk access to the ciphertext_raw/enc_key_raw spans: + auto ciphertext = indices | std::views::transform([&](int i) { + return std::span{ + ciphertext_raw.data() + i * mlkem768::CIPHERTEXTBYTES, + mlkem768::CIPHERTEXTBYTES}; + }); + auto enc_indicator = + indices | std::views::transform([&](int i) { + return std::span{enc_key_raw.data() + i * (2 + 32), 2}; + }); + auto enc_key = + indices | std::views::transform([&](int i) { + return std::span{enc_key_raw.data() + i * (2 + 32) + 2, 32}; + }); + + auto active_keys = active_device_keys(); + + auto devices_nonce = hash::blake2b_key_pers<24>(A, PERS_DEV_NONCE, ciphertext_raw); + + cleared_b32 ml_ss, aB, ki, key_base; + + std::vector plaintext_devices; + plaintext_devices.resize(enc_devices.size() - encryption::XCHACHA20_ABYTES); + + // Trial decrypt until we find one that works, except that we can skip most of the heavy + // operations for most keys not intended for us. Note that we have to attempt each received key + // by all of our recent device keys because it might be a pre-rotation message encrypted using + // an older key, so even if we have only 4 incoming values, we might have 20 recent device keys + // meaning 80 potential decryptions. + bool found = false; + for (int i = 0; !found && i < count; i++) { + auto ct = ciphertext[i]; + auto ekey = enc_key[i]; + auto eind = enc_indicator[i]; + + auto knonce = hash::blake2b_key_pers<24>(A, PERS_KEY_NONCE, ct, enc_devices); + + for (int active_i = 0; active_i < active_keys.size(); active_i++) { + const auto& k = active_keys[active_i]; + const auto& b = k.x25519_sec; + const auto& B = k.x25519_pub; + const auto& M = k.mlkem768_pub; + + // First work out the checksum hash; the vast majority of the time this won't match for + // a key other than our own (only 1/65535 chance of collision), and so we can short + // circuit and save a bunch of calculations. + if (!std::ranges::equal( + hash::blake2b_pers<2>(PERS_KEY_KEY_IDX, A, B, M, ct, ekey), eind)) + continue; + + if (!x25519::scalarmult(aB, b, A)) { + log::warning(cat, "X25519 multiplication failed; ignoring encrypted entry"); + continue; + } + + if (!mlkem768::decapsulate(ml_ss, ct, k.mlkem768_sec)) { + log::warning(cat, "MLKEM768 decapsulation failed; skipping device entry"); + continue; + } + + // Now we have various shared secret data: hash it into the k[i] value that should have + // been used to encrypt the key_base value for us: + hash::blake2b_pers(ki, PERS_KEY_KEY, aB, A, B, ml_ss, M); + + // and then use it to recover the key_base: + static_assert(decltype(ekey)::extent == key_base.size()); + encryption::xchacha20_xor(key_base, ekey, knonce, ki); + + // Now we can decrypt the encrypted payload: + if (encryption::xchacha20poly1305_decrypt( + plaintext_devices, enc_devices, devices_nonce, key_base)) { + found = true; + break; + } + + log::debug( + cat, + "Decryption of record {} against recent key {} failed; probably a checksum " + "false positive", + i, + active_i); + } + } + + if (!found) { + // There are a bunch of reasons for this: maybe we aren't in the device group, maybe it was + // corrupted, or many it is an old message and we don't have the keys for it anymore. + log::warning(cat, "Failed to decrypt incoming device data"); + throw device::decryption_failed{"Failed to decrypt incoming device data"}; + } + + // Strip the padding appended before encryption. The payload is a bt-encoded dict, which always + // ends in 'e', so trailing null bytes are unambiguously padding rather than content. + trim_trailing(plaintext_devices); + + return plaintext_devices; +} + +void Devices::receive_link_request(std::span data) { + // Parse outer bt-dict: {"": "L", "L": } + oxenc::bt_dict_consumer outer{data}; + outer.require(""); // skip type indicator + auto encrypted = outer.require_span("L"); + + // Decrypt using the account seed + std::vector plaintext; + try { + auto seed = core.globals.account_seed(); + plaintext = config::decrypt(encrypted, seed.seed(), "link-request"); + } catch (const config::decrypt_error& e) { + log::warning(cat, "Ignoring incoming link request: decryption failed: {}", e.what()); + return; + } + + // Parse plaintext: {"I": <32-byte device id>, "i": {device info dict}} + device::Info info; + try { + oxenc::bt_dict_consumer pt{std::span{plaintext}}; + auto in_id = pt.require_span("I"); + std::memcpy(info.id.data(), in_id.data(), info.id.size()); + + // Skip any unknown keys between "I" and "i" + oxenc::bt_dict extra_outer; + while (!pt.is_finished() && pt.key() < "i") + consume_extra(pt, extra_outer); + if (pt.is_finished() || pt.key() != "i") + throw std::runtime_error{"missing 'i' device info dict"}; + decode_one(info, pt.consume_dict_consumer(), device::State::Pending); + } catch (const std::exception& e) { + log::warning(cat, "Ignoring incoming link request: failed to parse: {}", e.what()); + return; + } + + auto c = conn(); + + // Reject if already registered or unregistered; only Pending (or absent) is valid + auto existing_state = + c.prepared_maybe_get("SELECT state FROM devices WHERE unique_id = ?", info.id) + .value_or(-1); + if (existing_state != -1 && existing_state != static_cast(device::State::Pending)) { + log::debug( + cat, + "Ignoring link request from {}: device already in state {}", + oxenc::to_hex(info.id), + existing_state); + return; + } + + SQLite::Transaction tx{c.sql}; + + auto dev_id = upsert_device_info(c, info); + if (!dev_id) { + log::debug( + cat, + "Ignoring link request from {}: rejected by seqno guard", + oxenc::to_hex(info.id)); + return; + } + + auto sas_seed = derive_sas_seed(as_span(std::span{plaintext})); + + c.prepared_exec( + R"(INSERT INTO device_link_requests (device, received_at, sas_seed) + VALUES (?, ?, ?) + ON CONFLICT(device) DO UPDATE SET + received_at = excluded.received_at, + sas_seed = excluded.sas_seed)", + *dev_id, + epoch_seconds(clock_now_s()), + sas_seed); + + // Set processing=LinkRequest only if not already set to a higher-priority value by a + // concurrent device group message in the same batch + c.prepared_exec( + "UPDATE devices SET processing = ? WHERE id = ? AND processing IS NULL", + static_cast(Processing::LinkRequest), + *dev_id); + + tx.commit(); +} + +void Devices::parse_device_messages(std::span messages, bool is_final) { + for (const auto& msg : messages) { + try { + oxenc::bt_dict_consumer in{msg.data}; + auto type = in.require(""); + if (type == "G") + receive_device_group_message(msg.data); + else if (type == "L") + receive_link_request(msg.data); + else + log::warning(cat, "Ignoring device message with unknown type '{}'", type); + } catch (const std::exception& e) { + log::warning(cat, "Ignoring malformed device message: {}", e.what()); + } + } + + if (!is_final) + return; + + // Fire deferred callbacks for all devices with a pending processing state. We collect first + // to avoid nested statement conflicts during callback + processing-clear operations. + struct ProcessingItem { + int64_t row_id; + std::array id; + Processing processing; + device::Info info; + }; + + auto c = conn(); + std::vector items; + for (auto [row_id, + raw_id, + processing_int, + state_int, + seqno, + timestamp, + dtype, + desc, + ver, + pk_ml, + pk_x, + kicked_ts] : + c.prepared_results< + int64_t, + sqlite::blob_guts>, + int, + int, + int, + int64_t, + std::string, + std::string, + int64_t, + sqlite::blobn, + sqlite::blobn<32>, + std::optional>( + "SELECT id, unique_id, processing, state, seqno, timestamp, device_type," + " description, version, pubkey_mlkem768, pubkey_x25519, kicked_timestamp" + " FROM devices WHERE processing IS NOT NULL ORDER BY unique_id")) { + auto& item = items.emplace_back(); + item.row_id = row_id; + item.id = raw_id; + item.processing = static_cast(processing_int); + item.info = fill_device_info( + raw_id, + state_int, + seqno, + timestamp, + std::move(dtype), + std::move(desc), + ver, + pk_ml, + pk_x); + if (kicked_ts) + item.info.kicked.emplace(std::chrono::seconds{*kicked_ts}); + load_device_extras(c, row_id, item.info); + } + + // Non-const so a handler can take the info: each item reaches exactly one branch below, and + // nothing after the switch reads `info` again. + for (auto& item : items) { + bool is_self = (item.id == self_id); + try { + switch (item.processing) { + case Processing::LinkRequest: + if (auto& f = cb().device_link_request) { + auto [lr_id, sas_seed] = c.prepared_get< + int64_t, + sqlite::blob_guts>>( + "SELECT id, sas_seed FROM device_link_requests WHERE device = ?", + item.row_id); + f(static_cast(lr_id), std::move(item.info), sas_from_seed(sas_seed)); + } + break; + case Processing::Registered: + if (is_self) { + if (auto& f = cb().device_self_added) + f(); + } else { + if (auto& f = cb().device_added) { + auto reqid = + c.prepared_maybe_get( + "SELECT id FROM device_link_requests WHERE device = ?", + item.row_id) + .value_or(0LL); + f(static_cast(reqid), std::move(item.info)); + } + // Clean up any link request row (whether callback was set or not) + c.prepared_exec( + "DELETE FROM device_link_requests WHERE device = ?", item.row_id); + } + break; + case Processing::Removed: + if (is_self) { + if (auto& f = cb().device_self_removed) + f(); + } else { + if (auto& f = cb().device_removed) + f(std::move(item.info)); + } + break; + } + c.prepared_exec("UPDATE devices SET processing = NULL WHERE id = ?", item.row_id); + } catch (const std::exception& e) { + log::error( + cat, + "Exception in {} device callback for device {}: {}", + item.processing, + oxenc::to_hex(item.id), + e.what()); + // Don't clear processing so the callback will be retried + } + } + + // Prune stale link requests (older than 10 minutes) + c.prepared_exec( + "DELETE FROM device_link_requests WHERE received_at < ?", + epoch_seconds(clock_now_s() - LINK_REQUEST_MAX_AGE)); +} + +void Devices::parse_account_pubkeys(std::span messages, bool /*is_final*/) { + if (messages.empty()) + return; + + // The x25519 pubkey for signature verification: session_id() is 0x05 || x25519_pub + auto x25519_pub = core.globals.session_id().subspan<1>(); + + auto c = conn(); + for (const auto& msg : messages) { + try { + oxenc::bt_dict_consumer in{msg.data}; + auto M = in.require_span("M"); + auto X = in.require_span("X"); + in.require_signature( + "~", + [&x25519_pub](std::span body, std::span sig) { + if (sig.size() != 64 || + !xed25519::verify(sig.first<64>(), x25519_pub, body)) + throw std::runtime_error{ + "Invalid account pubkey message: signature verification " + "failed"}; + }); + + // Look up the key by indicator (indexed) then verify full pubkeys, and mark published. + c.prepared_exec( + "UPDATE device_account_keys SET published = 1" + " WHERE key_indicator = ? AND pubkey_mlkem768 = ? AND pubkey_x25519 = ?", + M.first<2>(), + M, + X); + } catch (const std::exception& e) { + log::warning(cat, "Ignoring malformed account pubkey message: {}", e.what()); + } + } +} + +static const std::string NEEDS_PUSH_SQL = + "SELECT" + // device_group: we are registered AND (own seqno dirty OR broadcast needed OR + // undistributed account key) + " CASE WHEN EXISTS(" + " SELECT 1 FROM devices WHERE unique_id = ? AND state = {0}" + " ) THEN (" + " (SELECT pushed_seqno IS NULL OR seqno > pushed_seqno" + " FROM devices WHERE unique_id = ?)" + " OR EXISTS(SELECT 1 FROM devices WHERE broadcast_needed)" + " OR EXISTS(SELECT 1 FROM device_account_keys WHERE NOT distributed)" + " ) ELSE 0 END," + // account_pubkey: the current active account key has not yet been confirmed on the swarm + " EXISTS(SELECT 1 FROM device_account_keys WHERE rotated IS NULL AND NOT published)"_format( + static_cast(device::State::Registered)); + +Devices::NeedsPush Devices::needs_push() { + auto c = conn(); + auto [dg, ap] = c.prepared_get(NEEDS_PUSH_SQL, self_id, self_id); + return {.device_group = bool(dg), .account_pubkey = bool(ap)}; +} + +void Devices::mark_device_group_pushed(int64_t seqno) { + auto c = conn(); + SQLite::Transaction tx{c.sql}; + c.prepared_exec("UPDATE devices SET pushed_seqno = ? WHERE unique_id = ?", seqno, self_id); + c.prepared_exec("UPDATE devices SET broadcast_needed = 0"); + c.prepared_exec("UPDATE device_account_keys SET distributed = 1"); + tx.commit(); +} + +std::optional Devices::next_account_rotation() { + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + if (!c.prepared_maybe_get( + "SELECT 1 FROM devices WHERE unique_id = ? AND state = ?", + self_id, + static_cast(device::State::Registered))) + return std::nullopt; + + int64_t t_created = 0; + std::optional active_seed; + for (auto [created, seed] : c.prepared_results>( + "SELECT created, seed FROM device_account_keys" + " WHERE rotated IS NULL ORDER BY created DESC LIMIT 1")) { + t_created = created; + std::memcpy(active_seed.emplace().data(), seed.data(), seed.size()); + } + if (!active_seed) + return std::nullopt; + + auto N = c.prepared_get( + "SELECT count(*) FROM devices WHERE state = ?", + static_cast(device::State::Registered)); + + tx.commit(); + + // u is a per-device uniform random value in [0,1], derived deterministically from the device + // ID and current account key seed so that each device independently computes a consistent + // rotation schedule. + std::array hash_out; + hash::blake2b_key_pers(hash_out, *active_seed, PERS_ACC_KEY_ROT, self_id); + double u = oxenc::load_little_to_host(hash_out.data()) / 0x1p64; + + // With N registered devices, the minimum of their N individual offsets is uniformly + // distributed in [PERIOD - WINDOW/2, PERIOD + WINDOW/2]. + auto offset = std::chrono::duration_cast( + (ACCOUNT_KEY_ROTATION_PERIOD - ACCOUNT_KEY_ROTATION_WINDOW / 2) + + ACCOUNT_KEY_ROTATION_WINDOW * (1.0 - std::pow(u, static_cast(N)))); + + return std::chrono::sys_seconds{std::chrono::seconds{t_created}} + offset; +} + +std::optional Devices::next_device_rotation() { + // TODO: implement device key rotation scheduling + return std::nullopt; +} + +Devices::DeviceGroupPush Devices::build_device_group_message() { + // One query for the whole group, so the seqno we report is the one the payload was built from. + // Reading our own row separately would let an update_info() land between the two and have the + // push confirm a seqno that is not what went out. + auto devs = devices(true, true, true); + + auto self = devs.find(self_id); + if (self == devs.end() || self->second.state != device::State::Registered) + throw std::logic_error{"Cannot build device group message: this device is not registered"}; + + return {encrypt_device_data(devs), self->second.seqno}; +} + +std::vector Devices::build_account_pubkey_message() { + auto keys = active_account_keys(); + if (keys.empty()) + throw std::runtime_error{"build_account_pubkey_message: no active account keys"}; + const auto& k = keys.front(); + + std::vector out( + 2 // outer dict d...e + + 3 + bt_bytes_encoded(1184) // "1:M" + mlkem768_pub + + 3 + bt_bytes_encoded(32) // "1:X" + x25519_pub + + 3 + bt_bytes_encoded(64) // "1:~" + XEd25519 signature + ); + + oxenc::bt_dict_producer o{reinterpret_cast(out.data()), out.size()}; + o.append("M", k.mlkem768_pub); + o.append("X", k.x25519_pub); + o.append_signature("~", [seed = core.globals.account_seed()](std::span body) { + return xed25519::sign(seed.x25519_key(), body); + }); + + assert(o.view().size() == out.size()); // Ensure we calculated exactly the right size above + return out; +} + +} // namespace session::core diff --git a/src/core/globals.cpp b/src/core/globals.cpp new file mode 100644 index 000000000..f410431fb --- /dev/null +++ b/src/core/globals.cpp @@ -0,0 +1,230 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::core { + +namespace log = oxen::log; +auto cat = log::Cat("core.gbl"); + +using namespace std::literals; + +static const std::string GET_ONE = "SELECT value FROM globals WHERE key = ? AND typeof(value) = ?"s; + +std::optional Globals::get_integer(std::string_view key) { + return conn().prepared_maybe_get(GET_ONE, key, "integer"); +} +std::optional Globals::get_real(std::string_view key) { + return conn().prepared_maybe_get(GET_ONE, key, "real"); +} +std::optional Globals::get_text(std::string_view key) { + return conn().prepared_maybe_get(GET_ONE, key, "text"); +} +std::optional> Globals::get_blob(std::string_view key) { + std::optional> result; + auto c = conn(); + auto st = c.prepared_bind(GET_ONE, key, "blob"); + if (st->executeStep()) { + auto data = sqlite::get(st); + result.emplace().reserve(data.size()); + result->assign(data.begin(), data.end()); + } + return result; +} +std::optional Globals::get_blob_secure(std::string_view key) { + std::optional result; + auto c = conn(); + auto st = c.prepared_bind(GET_ONE, key, "blob"); + if (st->executeStep()) + result.emplace(sqlite::get(st)); + return result; +} +bool Globals::get_blob_to(std::string_view key, std::span to) { + auto c = conn(); + auto st = c.prepared_bind(GET_ONE, key, "blob"); + if (st->executeStep()) { + if (auto data = sqlite::get(st); data.size() == to.size()) { + std::memcpy(to.data(), data.data(), to.size()); + return true; + } + } + return false; +} + +static const std::string GET_ANY = "SELECT value, typeof(value) FROM globals WHERE key = ?"s; + +template BlobLoader> +static auto get_variant_impl(sqlite::Connection&& c, std::string_view key, BlobLoader&& b) { + + using blob_t = decltype(b(std::declval())); + + std::variant result; + + auto st = c.prepared_bind(GET_ANY, key); + if (st->executeStep()) { + auto val = st->getColumn(0); + auto type = static_cast(st->getColumn(1)); + if (type == "int") + result.template emplace(std::move(val)); + else if (type == "text") + result.template emplace(std::move(val)); + else if (type == "blob") + result = b(sqlite::blob{std::move(val)}); + else if (type == "real") + result.template emplace(std::move(val)); + } + + return result; +} + +std::variant> Globals::get( + std::string_view key) { + return get_variant_impl(conn(), key, [](sqlite::blob data) { + std::vector v; + v.reserve(data.size()); + v.assign(data.begin(), data.end()); + return v; + }); +} + +std::variant +Globals::get_secure(std::string_view key) { + return get_variant_impl(conn(), key, [](sqlite::blob data) { return secure_buffer{data}; }); +} + +static const std::string SET_VAL = + "INSERT INTO globals (key, value) VALUES (?, ?)" + " ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value"s; + +void Globals::set(std::string_view key, int64_t integer) { + conn().prepared_exec(SET_VAL, key, integer); +} +void Globals::set(std::string_view key, double real) { + conn().prepared_exec(SET_VAL, key, real); +} +void Globals::set(std::string_view key, std::string_view text) { + conn().prepared_exec(SET_VAL, key, text); +} +void Globals::set(std::string_view key, std::span blob) { + conn().prepared_exec(SET_VAL, key, blob); +} + +bool Globals::erase(std::string_view key) { + return conn().prepared_exec("DELETE FROM globals WHERE key = ?"s, key) > 0; +} + +void Globals::_adopt_seed(const cleared_b32& seed, bool persist) { + // Layout: [ed25519_sk(64) | x25519_sk(32)] = 96 bytes + auto rw = _account_seed.resize(96); + + ed25519::seed_keypair(_pubkey_ed25519, rw.buf.first<64>(), seed); + ed25519::sk_to_x25519(rw.buf.last<32>(), seed); + + ed25519::pk_to_x25519(_pubkey_x25519, _pubkey_ed25519); + + _session_id[0] = std::byte{0x05}; + std::copy(_pubkey_x25519.begin(), _pubkey_x25519.end(), _session_id.data() + 1); + _session_id_hex = oxenc::to_hex(_session_id); + + if (persist) + set("_seed", rw.buf.first(32)); + + _have_account = true; + + log::info(cat, "Initialized with Session ID: {}", _session_id_hex); +} + +// Generates the 16-byte/128-bit seed that Session accounts use: 16 random bytes followed by 16 +// zeros. +// FIXME: we should allow full 32-byte seeds here. +static cleared_b32 generate_seed() { + cleared_b32 seed; + random::fill(std::span{seed}.first<16>()); + std::memset(seed.data() + 16, 0, 16); + return seed; +} + +void Globals::init() { + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + cleared_b32 seed; + if (get_blob_to("_seed", seed)) { + _adopt_seed(seed, false); + } else if (_predefined_seed) { + _adopt_seed(*_predefined_seed, true); + _predefined_seed.reset(); // Clear now that it has been consumed + } else if (!_defer_account) { + log::info(cat, "Generated new Session account seed"); + _adopt_seed(generate_seed(), true); + core.configs.initialise_new_account(); + _mark_new_account(); + } else { + // defer_account, and nothing stored: the application chooses an identity before this + // account can do anything. Nothing else in Core needs the seed at init time -- Devices + // only stores its own device id, and polling does not start until a network is attached. + log::info(cat, "Opened with no account; awaiting create_account() or restore_account()"); + } + + tx.commit(); +} + +void Globals::create_account() { + if (_have_account) + throw std::logic_error{"This account already has an identity"}; + auto c = conn(); + SQLite::Transaction tx{c.sql}; + log::info(cat, "Generated new Session account seed"); + _adopt_seed(generate_seed(), true); + core.configs.initialise_new_account(); + _mark_new_account(); + tx.commit(); + + // After the commit, not inside it: establishing the group opens transactions of its own, and + // this connection is thread-unique, so nesting would fail. Safe to defer -- the flag written + // above is what survives a crash here, and Devices::init() acts on it next time. + core.devices.establish_group(); +} + +// A generated account owes a device group; a restored one does not, since it may already have one +// belonging to devices that are simply offline. Recorded rather than acted on here because Devices +// initialises after this component and has no device id yet during init. +void Globals::_mark_new_account() { + core.devices._mark_group_owed(); +} + +void Globals::restore_account(const predefined_seed& seed) { + if (_have_account) + throw std::logic_error{"This account already has an identity"}; + auto c = conn(); + SQLite::Transaction tx{c.sql}; + _adopt_seed(seed.bytes, true); + tx.commit(); +} + +mnemonics::secure_mnemonic Globals::seed_mnemonic(const mnemonics::Mnemonics& lang, bool force_24) { + _require_account(); + auto seed = _account_seed.access(); + // _account_seed stores the 96-byte key material; the first 32 bytes are the account seed. + // A Session account uses 128-bit entropy when the last 16 bytes of that seed are all zero; + // in that case we encode only the first 16 bytes. + auto seed32 = seed.buf.first(32); + bool is_128bit = !force_24 && + sodium_memcmp(seed32.data() + 16, std::array{}.data(), 16) == 0; + return mnemonics::bytes_to_words(is_128bit ? seed32.first(16) : seed32, lang); +} + +mnemonics::secure_mnemonic Globals::seed_mnemonic(std::string_view lang_name, bool force_24) { + return seed_mnemonic(mnemonics::get_language(lang_name), force_24); +} + +} // namespace session::core diff --git a/src/core/link_sas.cpp b/src/core/link_sas.cpp new file mode 100644 index 000000000..85e1703bf --- /dev/null +++ b/src/core/link_sas.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include +#include +#include + +using namespace session::literals; + +namespace session::core { + +std::array derive_sas_seed(std::span plaintext) { + auto salt = hash::blake2b_pers<16>("SessionLinkEmoji"_b2b_pers, plaintext); + + std::array seed; + if (0 != crypto_pwhash( + reinterpret_cast(seed.data()), + seed.size(), + reinterpret_cast(plaintext.data()), + plaintext.size(), + reinterpret_cast(salt.data()), + /*opslimit=*/2, + /*memlimit=*/16ULL * 1024 * 1024, + crypto_pwhash_ALG_ARGON2ID13)) + throw std::runtime_error{"derive_sas_seed: Argon2id key derivation failed"}; + + return seed; +} + +std::array sas_from_seed(std::span seed) { + // Interpret the 16-byte seed as a 128-bit little-endian integer split into two 64-bit words. + uint64_t lo = oxenc::load_little_to_host(seed.data()); + uint64_t hi = oxenc::load_little_to_host(seed.data() + 8); + + std::array result; + for (int k = 0; k < 21; k++) { + int bit = k * 6; + uint8_t index; + if (bit + 6 <= 64) + index = (lo >> bit) & 0x3F; + else if (bit >= 64) + index = (hi >> (bit - 64)) & 0x3F; + else + // Single crossing point: k=10, bit=60. + // Low 4 bits come from lo (bits 60-63), high 2 bits come from hi (bits 64-65). + index = ((lo >> 60) | (hi << 4)) & 0x3F; + result[k] = SAS_EMOJI[index]; + } + return result; +} + +std::array link_request_sas(std::span plaintext) { + return sas_from_seed(derive_sas_seed(plaintext)); +} + +} // namespace session::core diff --git a/src/core/pro.cpp b/src/core/pro.cpp new file mode 100644 index 000000000..5e23e3791 --- /dev/null +++ b/src/core/pro.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include + +#include "SQLiteCpp/Transaction.h" +#include "session/sqlite.hpp" + +namespace session::core { + +bool Pro::proof_is_revoked( + std::span revocation_tag, std::chrono::sys_seconds unix_ts) { + return conn().prepared_get( + "SELECT EXISTS (SELECT 1 FROM pro_revocations" + " WHERE revocation_tag = ? AND effective_ts <= ?)", + revocation_tag, + unix_ts.time_since_epoch().count()); +} + +/// API: core/Pro::pro_update_revocations +/// +/// Update the list of pro revocations. If the `revocations_ticket` matches the current ticket, +/// this is a no-op. +/// +/// Inputs: +/// - `ticket` -- Ticket that describes the version of the revocations. This value comes +/// alongside the revocation list when queried. This ticket changes whenever the revocation +/// list is updated and is used to identify when an actual update is needed. +/// - `revocations` -- New list of Session Pro revocations. +void Pro::update_revocations( + uint32_t ticket, + std::span revocations, + std::chrono::seconds retain_for) { + + if (revocations_ticket_ && ticket == *revocations_ticket_) + return; + + auto now = session::clock_now_s().time_since_epoch().count(); + + auto c = conn(); + SQLite::Transaction tx{c.sql}; + + // Upsert each listed revocation, (re)setting its last-seen time to now. + for (auto st = + c.prepared_st("INSERT INTO pro_revocations (revocation_tag, effective_ts, seen_at)" + " VALUES (?, ?, ?)" + " ON CONFLICT (revocation_tag) DO UPDATE SET" + " effective_ts = excluded.effective_ts, seen_at = excluded.seen_at"); + const auto& revoke : revocations) { + exec_query(st, revoke.revocation_tag, revoke.effective_at.time_since_epoch().count(), now); + st->reset(); + } + + // Memory-only aging: drop entries not seen within the retain window. Unlike the wire list, + // absent entries are not deleted immediately -- holding a stale entry is harmless (its random + // tag never matches a live proof), and retain_for >= the proof-validity window guarantees we + // never drop an entry while a valid proof could still carry it. + auto del = c.prepared_st("DELETE FROM pro_revocations WHERE seen_at + ? < ?"); + exec_query(del, retain_for.count(), now); + + revocations_ticket_ = ticket; + tx.commit(); +} + +} // namespace session::core diff --git a/src/core/schema/000_config_dumps.sql b/src/core/schema/000_config_dumps.sql new file mode 100644 index 000000000..bd9649810 --- /dev/null +++ b/src/core/schema/000_config_dumps.sql @@ -0,0 +1,21 @@ +-- The serialised state of each config object, so a config survives a restart without being rebuilt +-- from its swarm. A dump holds the merged config together with the bookkeeping a merge needs -- +-- which message hashes currently represent it, and which ones it obsoletes -- so restoring one is +-- what lets this device rejoin an ongoing exchange with its other devices rather than starting from +-- whatever the swarm happens to still hold. +-- +-- Keyed by the config's encryption domain rather than its storage namespace. The two coincide for +-- most configs but answer different questions: the domain says *which config this is*, while the +-- namespace says *where it is pushed*, which the Local config has no answer for -- it is never +-- pushed anywhere, and reports UserProfile's namespace as a stand-in. The domain is also already +-- required to be unique per config type, and can never be changed, since changing it would break +-- decryption of everything previously written under it. +-- +-- pubkey is whose config it is: this account's, or a particular group's. Configs for different +-- pubkeys are stored, pushed and fetched separately even when their swarms coincide. +CREATE TABLE config_dumps ( + pubkey BLOB NOT NULL CHECK(length(pubkey) = 33), + type TEXT NOT NULL, -- ConfigBase::encryption_domain(): "UserProfile", "Contacts", ... + data BLOB NOT NULL, + PRIMARY KEY (pubkey, type) +) STRICT; diff --git a/src/core/schema/001_swarm_hash_history.sql b/src/core/schema/001_swarm_hash_history.sql new file mode 100644 index 000000000..26bfd73f9 --- /dev/null +++ b/src/core/schema/001_swarm_hash_history.sql @@ -0,0 +1,21 @@ +CREATE TABLE swarm_nodes ( + id INTEGER PRIMARY KEY, + pubkey BLOB NOT NULL UNIQUE CHECK(length(pubkey) = 32) +) STRICT; + +CREATE TABLE swarm_hashes ( + id INTEGER PRIMARY KEY, + namespace INTEGER NOT NULL, + node INTEGER NOT NULL REFERENCES swarm_nodes(id) ON DELETE CASCADE, + hash TEXT NOT NULL, + expiry INTEGER, + UNIQUE(namespace, node, hash) +) STRICT; + +CREATE INDEX swarm_hashes_cursor ON swarm_hashes(namespace, node, id DESC); + +-- The cursors this replaces cannot be carried across: an entry is only usable if we know when the +-- server will drop it, and the old table never recorded that. Inventing an expiry to preserve them +-- would put a guess into the one column whose whole meaning is a fact the server told us. Losing +-- them costs one full retrieve per namespace per node, once, and everything in it dedupes. +DROP TABLE namespace_sync; diff --git a/src/core/schema/CMakeLists.txt b/src/core/schema/CMakeLists.txt new file mode 100644 index 000000000..66a708041 --- /dev/null +++ b/src/core/schema/CMakeLists.txt @@ -0,0 +1,5 @@ +session_schema_dir( + TARGET core + NAMESPACE session::core::schema + DECLARE_HEADER session/core/schema/schema_registry.hpp +) diff --git a/src/core/schema/README b/src/core/schema/README new file mode 100644 index 000000000..def387257 --- /dev/null +++ b/src/core/schema/README @@ -0,0 +1,58 @@ +Any files in this directory beginning with a number and ending with .cpp or .sql are one-time +migrations: they are processed in ascii-sorted order against the `migrations_applied` table: if the +migration doesn't exist there then the migration is executed and the entry is inserted into +`migrations_applied`. + +The sort is over the migration *name* -- the filename without its extension, which is what gets +recorded in `migrations_applied` -- not over the filename. This matters when one name is a prefix +of another: sorting filenames would order `001_foo+002.sql` before `001_foo.sql`, because '+' sorts +below '.'. Sorting names makes it a prefix comparison, so the shorter name always runs first +regardless of what follows it. + +It is recommended that migrations defined here use the format `NNN_description_of_migration.sql`, +such as `000_device_tables.sql`. Updates that depend on earlier migrations having happend must then +ensure that they use a larger `NNN` value than the update that they depend on. Migrations that +depend on no other database components at all should start at 000. +There are two sorts of migrations permitted here: SQL and C++. + +SQL migrations are simply a file of queries to execute. If the queries execute successfully, the +migration is considered applied. Each migration runs inside a transaction (and so does not need to +worry about starting a transaction itself); if migration fails, the transaction is rolled back, +otherwise it (and the insertion into `migrations_applied`) is committed. + +C++ migrations are for cases where more complex logic is needed to perform a migration: any such +migration should define a function `apply_FILENAME` in the session::core::schema namespace that +takes arguments `(session::sqlite::Connection&, session::core::Core&)` and performs any needed +migrations. FILENAME here will be the filename on disk without the .cpp extension, with any +non-alphanumeric characters replaced with `_`. As with the SQL migration, there will be an active +transaction around the call; the C++ code should throw if the migration cannot be performed (which +will roll back the transaction). Migrations are performed as the final step of Core instance +construction (and so Core initialization is finished, but the Core object has not yet been returned +to the creator). + +It is not permitted to use the same base filename for both SQL and C++ migrations (e.g. having both +007_bond_james_bond.sql and 007_bond_james_bond.cpp): if a migration wants to use both, use a +different numeric prefix or some sort of suffix (either of which will also defines the order the two +migrations would be applied). + + +Iterating on a branch +--------------------- + +While developing a schema change you generally want to keep updating a database you have already +been using, rather than recreating it after every edit. To do that, name the first migration what +the finished one will be called, and add addenda alongside it as you go: + + 001_new_feature.sql + 001_new_feature+001.sql + 001_new_feature+002.sql + +Before merging, fold the addenda back into `001_new_feature.sql` and delete them. Because your +database recorded `001_new_feature` when you first created that file, the squashed migration is +already applied and is not re-run: your development database survives the squash. The orphaned +`001_new_feature+NNN` rows left in `migrations_applied` are ignored, since only names present in the +registry are ever consulted. + +If instead you name the first file something you later rename, the squash leaves your development +database out of step and you have to recreate it -- a nuisance rather than a problem, but easily +avoided. diff --git a/src/core/schema/full_schema.sql b/src/core/schema/full_schema.sql new file mode 100644 index 000000000..3353924da --- /dev/null +++ b/src/core/schema/full_schema.sql @@ -0,0 +1,211 @@ +-- Core's schema with every migration in src/core/schema/ applied. A database with none of them +-- applied is built from this in one step and they are all recorded without running, so this file +-- -- not the migration chain -- is where the current schema is read. +-- +-- Keep in step with the migrations: test_core_schema.cpp builds a database both ways and +-- compares them. + +-- Table storing all the device group info +CREATE TABLE devices ( + id INTEGER PRIMARY KEY NOT NULL, + unique_id BLOB UNIQUE NOT NULL CHECK(length(unique_id) == 32), + + -- Membership rank: 0 unregistered, 1 pending, 2 registered, 3 kicked. Ordered least to most + -- authoritative because merging compares (state, seqno) as a row value -- see device::State. + state INTEGER NOT NULL CHECK(state >= 0 AND state <= 3), + processing INTEGER, -- non-null during batch processing: 1=new link request, 2=newly registered, 3=newly removed + seqno INTEGER NOT NULL DEFAULT 1, + pushed_seqno INTEGER, -- seqno of the last confirmed device group push; NULL = never pushed + broadcast_needed INTEGER NOT NULL DEFAULT 0, -- 1 when a state transition (registered/removed) needs broadcasting + timestamp INTEGER NOT NULL, + kicked_timestamp INTEGER, -- set when the device was kicked from the device group + device_type TEXT NOT NULL, -- typically a/i/d (Android/iOS/Desktop), but can be anything + description TEXT NOT NULL, -- freeform device description + version INTEGER NOT NULL, -- = 1000000*V + 1000*v + p for version "V.v.p" + pubkey_mlkem768 BLOB NOT NULL CHECK(length(pubkey_mlkem768) == 1184), + pubkey_x25519 BLOB NOT NULL CHECK(length(pubkey_x25519) == 32), + + -- A kick is the one state that carries a timestamp, and is meaningless without one, so the two + -- are tied together here rather than left to each call site to remember. + CHECK((state == 3) == (kicked_timestamp IS NOT NULL)) +) STRICT; + +-- This table holds any extra info not captured by the above. The data is stored as key/value pairs +-- where the value is the bt-encoded data received in the last device info message. The purpose of +-- this is so that future versions that add new fields can have those unknown fields propagated by +-- older clients that do not yet understand them without the older clients silently dropping unknown +-- fields. +CREATE TABLE device_unknown ( + device INTEGER NOT NULL REFERENCES devices(id) ON DELETE CASCADE, + key TEXT NOT NULL, + bt_value BLOB NOT NULL, + PRIMARY KEY(device, key) +) STRICT; + +-- This table tracks pending incoming device link requests from other devices that have been +-- received but not yet accepted, ignored, or denied. Device info for the requesting device is +-- stored in the devices table (with state=Pending); this table holds the link-request-specific +-- fields: when the request was received locally, and the precomputed Argon2id seed from which +-- the short authentication string emoji are derived (stored to avoid re-running the expensive +-- hash on every display). +CREATE TABLE device_link_requests ( + id INTEGER PRIMARY KEY NOT NULL, + device INTEGER UNIQUE NOT NULL REFERENCES devices(id) ON DELETE CASCADE, + received_at INTEGER NOT NULL, -- unix timestamp of when this request was stored locally + sas_seed BLOB NOT NULL CHECK(length(sas_seed) == 16) -- 16-byte Argon2id output for SAS display +) STRICT; + +-- This table holds current and recent device private keys for *this* device, including the +-- timestamp then the device keypairs were created, and when they were rotated away from. +CREATE TABLE device_privkeys ( + id INTEGER PRIMARY KEY NOT NULL, + created INTEGER NOT NULL, -- unix timestamp + rotated INTEGER, -- timestamp when a newer key was added, superceding this key + seed BLOB NOT NULL CHECK(length(seed) == 32) +) STRICT; + +-- This trigger handles key rotation: whenever we insert a new key, any existing keys are +-- automatically rotated with the `creation` timestamp of the new row as the rotation timestamp. +CREATE TRIGGER device_privkey_rotation AFTER INSERT ON device_privkeys +FOR EACH ROW WHEN NEW.rotated IS NULL +BEGIN + UPDATE device_privkeys SET rotated = NEW.created WHERE rotated IS NULL AND id != NEW.id; +END; + +-- This table holds current and recent *account* keys, which are shared within the device +-- group and have their public keys published for remote users to use to encrypt messages. +-- Unlike device_privkeys, these keys are shared among all devices in the device group. +CREATE TABLE device_account_keys ( + id INTEGER PRIMARY KEY NOT NULL, + created INTEGER NOT NULL, + rotated INTEGER, -- timestamp when a new key superceded this key + distributed INTEGER NOT NULL DEFAULT 0, -- 1 once this key's seed has been included in a confirmed device group push + published INTEGER NOT NULL DEFAULT 0, -- 1 once this key's pubkeys have been confirmed pushed as the account pubkey message + seed BLOB UNIQUE NOT NULL CHECK(length(seed) == 32), + pubkey_mlkem768 BLOB NOT NULL CHECK(length(pubkey_mlkem768) == 1184), + pubkey_x25519 BLOB NOT NULL CHECK(length(pubkey_x25519) == 32), + -- Virtual column containing the first two mlkem pubkey values to assist with lookups based on + -- incoming message key indicator: + key_indicator BLOB GENERATED ALWAYS AS (substr(pubkey_mlkem768, 1, 2)) VIRTUAL +) STRICT; +CREATE INDEX device_account_keys_ki_index ON device_account_keys(key_indicator); + +-- When a new account key is inserted as active (rotated IS NULL), apply deterministic +-- tie-breaking: the key with the latest created timestamp wins (ties broken by smallest seed), +-- and all unrotated losers are immediately marked as rotated at the winner's creation time. +-- This handles concurrent rotations from multiple devices: once all devices sync, the trigger +-- guarantees they all converge on the same active key regardless of insertion order. +CREATE TRIGGER device_account_key_rotation AFTER INSERT ON device_account_keys +FOR EACH ROW WHEN NEW.rotated IS NULL +BEGIN + UPDATE device_account_keys SET rotated = winner.created + FROM (SELECT id, created FROM device_account_keys + WHERE rotated IS NULL + ORDER BY created DESC, seed ASC + LIMIT 1) AS winner + WHERE device_account_keys.rotated IS NULL AND device_account_keys.id != winner.id; +END; + +-- from 000_config_dumps.sql +-- The serialised state of each config object, so a config survives a restart without being rebuilt +-- from its swarm. A dump holds the merged config together with the bookkeeping a merge needs -- +-- which message hashes currently represent it, and which ones it obsoletes -- so restoring one is +-- what lets this device rejoin an ongoing exchange with its other devices rather than starting from +-- whatever the swarm happens to still hold. +-- +-- Keyed by the config's encryption domain rather than its storage namespace. The two coincide for +-- most configs but answer different questions: the domain says *which config this is*, while the +-- namespace says *where it is pushed*, which the Local config has no answer for -- it is never +-- pushed anywhere, and reports UserProfile's namespace as a stand-in. The domain is also already +-- required to be unique per config type, and can never be changed, since changing it would break +-- decryption of everything previously written under it. +-- +-- pubkey is whose config it is: this account's, or a particular group's. Configs for different +-- pubkeys are stored, pushed and fetched separately even when their swarms coincide. +CREATE TABLE config_dumps ( + pubkey BLOB NOT NULL CHECK(length(pubkey) = 33), + type TEXT NOT NULL, -- ConfigBase::encryption_domain(): "UserProfile", "Contacts", ... + data BLOB NOT NULL, + PRIMARY KEY (pubkey, type) +) STRICT; + +CREATE TABLE globals ( + key TEXT PRIMARY KEY NOT NULL, + value ANY NOT NULL +) STRICT; + + +-- from 001_swarm_hash_history.sql +-- +-- Every message hash a storage node has handed us, in the order that node handed it over, with the +-- moment the node will drop it. +-- +-- This is where a retrieve's `last_hash` comes from -- the newest unexpired row for that node and +-- namespace -- rather than a stored cursor, so that deleting a hash from the swarm moves the cursor +-- by itself. A stored cursor has to be corrected by whoever deletes, and the correction is +-- invisible from the delete: config pushes have destroyed their own cursor on every push since they +-- were written, which costs a full retrieve each time and was never noticed because it is only +-- expensive and never wrong. +-- +-- Per node, because the cursor is. Nodes do not agree on storage order, so a hash learned from one +-- node is not a safe cursor for another: at best it is unrecognised and the node replays its whole +-- retention window, at worst the node holds it *after* something we never received, and advancing +-- there skips that message permanently. +-- +-- Expiry rather than a row count is what bounds this. A hash is a usable cursor for exactly as long +-- as the node still holds the message, so the server's own expiry is the honest limit; a count would +-- keep dead hashes and drop live ones at the same time. +-- +-- Every hash from a retrieve goes in, including messages we store nothing for -- typing indicators, +-- payloads that do not parse, anything a namespace carries that this build ignores. Recording only +-- what we kept would leave the cursor behind those, and we would fetch them again on every poll. +-- +-- Deliberately no foreign key to the client's `messages`: this is Core's, and a bare Core has no +-- such table. It would also be the wrong fact -- a row here says a *node* still holds the message, +-- which has nothing to do with whether we kept our copy of it. +-- +-- The nodes are their own table because the alternative repeats a 32-byte key in every row and +-- again in every index entry, to say something there are only a handful of distinct answers to. +-- +-- Nothing prunes nodes by swarm membership, deliberately. A node that has left is dead weight, but +-- deciding that requires trusting a swarm list to be complete, and a momentarily narrow one would +-- delete cursors that are still good -- costing a full retrieve from every node it omitted, which is +-- the exact thing this table exists to avoid. Expiry clears their rows soon enough anyway. +CREATE TABLE swarm_nodes ( + id INTEGER PRIMARY KEY, + pubkey BLOB NOT NULL UNIQUE CHECK(length(pubkey) = 32) +) STRICT; + +CREATE TABLE swarm_hashes ( + id INTEGER PRIMARY KEY, + namespace INTEGER NOT NULL, + node INTEGER NOT NULL REFERENCES swarm_nodes(id) ON DELETE CASCADE, + hash TEXT NOT NULL, + expiry INTEGER, + UNIQUE(namespace, node, hash) +) STRICT; + +CREATE INDEX swarm_hashes_cursor ON swarm_hashes(namespace, node, id DESC); + +-- Cache of remote account public keys (X25519 + ML-KEM-768) used for PFS+PQ message encryption. +-- Keys are considered fresh for PFS_KEY_FRESH_DURATION (24h) and expire after +-- PFS_KEY_EXPIRY_DURATION (48h); stale entries (24-48h old) are still usable as a fallback. +-- +-- nak_at is set whenever a successful fetch returns no valid keys, and is never cleared. It +-- suppresses re-fetching for PFS_KEY_NAK_DURATION (1h) when no valid keys exist. When valid +-- keys are present nak_at may coexist with them (the keys are still usable as a fallback). +-- In SQLite, CHECK constraints with a NULL argument evaluate to NULL (not FALSE), so the length +-- checks do not reject NULL pubkeys. +CREATE TABLE pfs_key_cache ( + session_id BLOB NOT NULL PRIMARY KEY CHECK(length(session_id) = 33), + fetched_at INTEGER, -- unix timestamp (seconds) of last fetch with valid keys; NULL if none + nak_at INTEGER, -- unix timestamp of last fetch returning no keys; NULL if none + pubkey_x25519 BLOB CHECK(length(pubkey_x25519) = 32), + pubkey_mlkem768 BLOB CHECK(length(pubkey_mlkem768) = 1184) +) STRICT; + +CREATE TABLE pro_revocations ( + revocation_tag BLOB PRIMARY KEY NOT NULL, + effective_ts INTEGER NOT NULL, -- unix seconds; a matching proof is revoked once the clock reaches this + seen_at INTEGER NOT NULL -- unix seconds when last seen in a fetched list (for retain_for aging) +) STRICT diff --git a/src/core/swarm_request.hpp b/src/core/swarm_request.hpp new file mode 100644 index 000000000..f08f1440d --- /dev/null +++ b/src/core/swarm_request.hpp @@ -0,0 +1,127 @@ +#pragma once + +/// Shared plumbing for requests a Core makes to a storage server swarm: which of them have to be +/// signed, what the signed value looks like, and how long one is allowed to take. +/// +/// Internal to the core library. It exists because more than one part of Core talks to a swarm -- +/// polling, sending a message, pushing a config -- and each of them needs the same three answers. +/// Transcribing them separately is how one copy ends up disagreeing with the storage server. + +#include +#include +#include +#include +#include +#include +#include + +namespace session::core { + +using namespace std::literals; +using namespace session::literals; + +// Namespace access rules, mirrored from the storage server's oxenss/common/namespace.h. The names +// match it deliberately: these decide whether a request we build has to be signed, and getting them +// out of step with the server is a 401 in production and nothing at all in testing. + +// Namespaces anyone may store to, every one divisible by 10. Namespace 0 is one, which is what +// makes it possible to be messaged by someone who does not have our keys. +constexpr bool is_public_inbox_namespace(int16_t ns) { + return ns % 10 == 0; +} + +// Namespaces anyone may retrieve from but only the owner may store to: the negative namespaces of +// the form -(20n+1), i.e. -1, -21, -41. Storing implicitly replaces what is there. +constexpr bool is_public_outbox_namespace(int16_t ns) { + return ns < 0 && -ns % 20 == 1; +} + +// Deprecated legacy closed groups, which allow both unauthenticated store and retrieval. +constexpr int16_t LEGACY_CLOSED_NAMESPACE = -10; + +constexpr bool retrieve_requires_auth(int16_t ns) { + return !(ns == LEGACY_CLOSED_NAMESPACE || is_public_outbox_namespace(ns)); +} + +constexpr bool store_requires_auth(int16_t ns) { + return !is_public_inbox_namespace(ns); +} + +// The examples the storage server's own comments give, so a rule transcribed wrongly cannot build. +static_assert( + is_public_inbox_namespace(0) && is_public_inbox_namespace(10) && + is_public_inbox_namespace(400) && is_public_inbox_namespace(-1230)); +static_assert( + !is_public_inbox_namespace(11) && !is_public_inbox_namespace(21) && + !is_public_inbox_namespace(-21)); +static_assert( + is_public_outbox_namespace(-1) && is_public_outbox_namespace(-21) && + is_public_outbox_namespace(-981)); +static_assert( + !is_public_outbox_namespace(1) && !is_public_outbox_namespace(21) && + !is_public_outbox_namespace(-20)); + +// The namespaces we actually use, spelled out: a DM is deposited unsigned in a stranger's inbox, +// while our own device, config and account-key namespaces are ours alone to write. +static_assert(!store_requires_auth(0) && retrieve_requires_auth(0)); +static_assert(store_requires_auth(21) && retrieve_requires_auth(21)); +static_assert(store_requires_auth(-21) && !retrieve_requires_auth(-21)); +static_assert(!retrieve_requires_auth(LEGACY_CLOSED_NAMESPACE)); +static_assert(store_requires_auth(2) && retrieve_requires_auth(2)); +static_assert(store_requires_auth(5) && retrieve_requires_auth(5)); + +// How long one attempt at a swarm request gets, and how long the whole operation gets across every +// swarm member it tries. +// +// A node that has not answered in ten seconds is better abandoned than waited on: the swarm has +// several other members, and moving to one of them costs less than the rest of a long timeout, so +// the budget is better spent on several short attempts than on one patient one. +// +// The overall figure is what actually bounds the operation. Without it, a swarm whose members are +// all unreachable would cost the per-request timeout once per member, which is the sort of +// arithmetic that only shows up in front of a user on a bad network. +constexpr auto SWARM_REQUEST_TIMEOUT = 10s; +constexpr auto SWARM_OVERALL_TIMEOUT = 60s; + +// Builds a request sent to `node` *about* the account identified by `swarm_pubkey`. Recording the +// swarm pubkey is what allows a 421 (the node is no longer in that account's swarm) to be recovered +// by re-resolving the swarm, and it is also what lets an unreachable node be replaced by the next +// member of the same swarm -- so it is bundled in here rather than left to each call site to +// remember. +inline network::Request swarm_request( + const network::service_node& node, + const network::x25519_pubkey& swarm_pubkey, + std::string endpoint, + std::vector body) { + network::Request req{ + node, + std::move(endpoint), + std::move(body), + network::RequestCategory::standard_small, + SWARM_REQUEST_TIMEOUT}; + req.swarm_pubkey = swarm_pubkey; + req.overall_timeout = SWARM_OVERALL_TIMEOUT; + return req; +} + +// Builds the value a storage server signs for `endpoint` ("store", "retrieve", ...) against the +// given namespace and request timestamp. The default namespace contributes nothing at all rather +// than a literal "0"; signing the 0 gets the request rejected with a 401. +inline std::string ns_signature_value( + std::string_view endpoint, int16_t ns_val, int64_t timestamp_ms) { + if (ns_val == 0) + return "{}{}"_format(endpoint, timestamp_ms); + return "{}{}{}"_format(endpoint, ns_val, timestamp_ms); +} + +// The value signed for a `delete`, which is the odd one out: it covers the hashes being deleted and +// carries neither a namespace nor a timestamp, since the hashes it names are specific enough to +// stand alone. Mirrors the storage server's delete_msgs documentation. +inline std::string delete_signature_value(std::span hashes) { + std::string to_sign = "delete"; + for (const auto& h : hashes) + to_sign += h; + return to_sign; +} + +} // namespace session::core diff --git a/src/crypto/ed25519.cpp b/src/crypto/ed25519.cpp new file mode 100644 index 000000000..104ee7413 --- /dev/null +++ b/src/crypto/ed25519.cpp @@ -0,0 +1,328 @@ +#include "session/crypto/ed25519.hpp" + +#include +#include +#include +#include + +#include +#include + +#include "session/ed25519.h" +#include "session/export.h" +#include "session/hash.hpp" +#include "session/pro_backend.hpp" +#include "session/sodium_array.hpp" +#include "session/util.hpp" + +namespace session::ed25519 { + +PrivKeySpan::PrivKeySpan(const std::byte* data, size_t size) { + if (size == 64) + data_ = data; + else if (size == 32) { + expand_seed(std::span{data, 32}); + data_ = storage_->data(); + } else + throw std::invalid_argument{ + "Ed25519 private key must be 32 or 64 bytes (got {})"_format(size)}; +} + +void PrivKeySpan::expand_seed(std::span seed) { + auto& buf = storage_.emplace(); + b32 ignore_pk; + crypto_sign_ed25519_seed_keypair( + to_unsigned(ignore_pk.data()), to_unsigned(buf.data()), to_unsigned(seed.data())); +} + +void PrivKeySpan::expand_seed(std::span seed) { + auto& buf = storage_.emplace(); + b32 ignore_pk; + crypto_sign_ed25519_seed_keypair( + to_unsigned(ignore_pk.data()), to_unsigned(buf.data()), seed.data()); +} + +void keypair(std::span pk, std::span sk) { + crypto_sign_ed25519_keypair(to_unsigned(pk.data()), to_unsigned(sk.data())); +} + +std::pair keypair() { + std::pair kp; + keypair(kp.first, kp.second); + return kp; +} + +void seed_keypair( + std::span pk, + std::span sk, + std::span seed) { + crypto_sign_ed25519_seed_keypair( + to_unsigned(pk.data()), to_unsigned(sk.data()), to_unsigned(seed.data())); +} + +std::pair keypair(std::span ed25519_seed) { + std::pair kp; + seed_keypair(kp.first, kp.second, ed25519_seed); + return kp; +} + +void sk_to_pk(std::span pk, const PrivKeySpan& sk) { + crypto_sign_ed25519_sk_to_pk(to_unsigned(pk.data()), to_unsigned(sk.data())); +} + +b32 sk_to_pk(const PrivKeySpan& sk) { + b32 pk; + sk_to_pk(pk, sk); + return pk; +} + +bool is_valid_pubkey(std::span pk) { + return crypto_core_ed25519_is_valid_point(to_unsigned(pk.data())) == 1; +} + +void pk_to_x25519(std::span out, std::span pk) { + if (0 != crypto_sign_ed25519_pk_to_curve25519(to_unsigned(out.data()), to_unsigned(pk.data()))) + throw std::runtime_error{"Failed to convert Ed25519 pubkey to X25519: invalid key"}; +} + +b32 pk_to_x25519(std::span pk) { + b32 xpk; + pk_to_x25519(xpk, pk); + return xpk; +} + +void pk_to_session_id(std::span out, std::span pk) { + out[0] = std::byte{0x05}; + pk_to_x25519(out.last<32>(), pk); +} + +b33 pk_to_session_id(std::span pk) { + b33 sid; + pk_to_session_id(sid, pk); + return sid; +} + +void sk_to_x25519(std::span out, std::span seed) { + crypto_sign_ed25519_sk_to_curve25519(to_unsigned(out.data()), to_unsigned(seed.data())); +} + +std::pair x25519_keypair(const PrivKeySpan& sk) { + return {sk_to_x25519(sk), pk_to_x25519(sk.pubkey())}; +} + +void scalarmult_base(std::span out, std::span scalar) { + if (0 != crypto_scalarmult_ed25519_base(to_unsigned(out.data()), to_unsigned(scalar.data()))) + throw std::runtime_error{"crypto_scalarmult_ed25519_base failed"}; +} + +b32 scalarmult_base(std::span scalar) { + b32 out; + scalarmult_base(out, scalar); + return out; +} + +void scalarmult_base_noclamp(std::span out, std::span scalar) { + if (0 != + crypto_scalarmult_ed25519_base_noclamp(to_unsigned(out.data()), to_unsigned(scalar.data()))) + throw std::runtime_error{"crypto_scalarmult_ed25519_base_noclamp failed"}; +} + +b32 scalarmult_base_noclamp(std::span scalar) { + b32 out; + scalarmult_base_noclamp(out, scalar); + return out; +} + +void scalarmult_noclamp( + std::span out, + std::span scalar, + std::span point) { + if (0 != + crypto_scalarmult_ed25519_noclamp( + to_unsigned(out.data()), to_unsigned(scalar.data()), to_unsigned(point.data()))) + throw std::runtime_error{"crypto_scalarmult_ed25519_noclamp failed"}; +} + +b32 scalarmult_noclamp( + std::span scalar, std::span point) { + b32 out; + scalarmult_noclamp(out, scalar, point); + return out; +} + +void scalar_reduce(std::span out, std::span in) { + crypto_core_ed25519_scalar_reduce(to_unsigned(out.data()), to_unsigned(in.data())); +} + +b32 scalar_reduce(std::span in) { + b32 out; + scalar_reduce(out, in); + return out; +} + +void scalar_negate(std::span out, std::span in) { + crypto_core_ed25519_scalar_negate(to_unsigned(out.data()), to_unsigned(in.data())); +} + +b32 scalar_negate(std::span in) { + b32 out; + scalar_negate(out, in); + return out; +} + +void scalar_mul( + std::span out, + std::span x, + std::span y) { + crypto_core_ed25519_scalar_mul( + to_unsigned(out.data()), to_unsigned(x.data()), to_unsigned(y.data())); +} + +b32 scalar_mul(std::span x, std::span y) { + b32 out; + scalar_mul(out, x, y); + return out; +} + +void scalar_add( + std::span out, + std::span x, + std::span y) { + crypto_core_ed25519_scalar_add( + to_unsigned(out.data()), to_unsigned(x.data()), to_unsigned(y.data())); +} + +b32 scalar_add(std::span x, std::span y) { + b32 out; + scalar_add(out, x, y); + return out; +} + +void sign( + std::span sig, + const PrivKeySpan& ed25519_privkey, + std::span msg) { + if (0 != crypto_sign_ed25519_detached( + to_unsigned(sig.data()), + nullptr, + to_unsigned(msg.data()), + msg.size(), + to_unsigned(ed25519_privkey.data()))) + throw std::runtime_error{"Failed to sign; perhaps the secret key is invalid?"}; +} + +b64 sign(const PrivKeySpan& ed25519_privkey, std::span msg) { + b64 sig; + sign(sig, ed25519_privkey, msg); + return sig; +} + +b64 decoy_signature() { + b64 sig; + // R = r·B for a random scalar r: a canonical point in the prime-order (main) subgroup, exactly + // like a real signature's R. (A directly-sampled random group element lands off the main + // subgroup ~7/8 of the time -- detectable via an L-torsion check.) + std::array r; + crypto_core_ed25519_scalar_random(r.data()); + crypto_scalarmult_ed25519_base_noclamp(to_unsigned(sig.data()), r.data()); + // s = a second, independent random scalar in ]0, L[, used as-is. + crypto_core_ed25519_scalar_random(to_unsigned(sig.data() + crypto_core_ed25519_BYTES)); + return sig; +} + +bool verify( + std::span sig, + std::span pubkey, + std::span msg) { + return (0 == crypto_sign_ed25519_verify_detached( + to_unsigned(sig.data()), + to_unsigned(msg.data()), + msg.size(), + to_unsigned(pubkey.data()))); +} + +std::pair derive_subkey( + std::span ed25519_seed, std::span domain) { + // Construct seed for derived key: + // new_seed = Blake2b32(ed25519_seed, key=domain) + cleared_b32 derived_seed; + hash::blake2b_key(derived_seed, domain, ed25519_seed); + return keypair(derived_seed); +} + +} // namespace session::ed25519 + +using namespace session; + +LIBSESSION_C_API bool session_ed25519_key_pair( + unsigned char* ed25519_pk_out, unsigned char* ed25519_sk_out) { + try { + auto [ed_pk, ed_sk] = session::ed25519::keypair(); + std::memcpy(ed25519_pk_out, ed_pk.data(), ed_pk.size()); + std::memcpy(ed25519_sk_out, ed_sk.data(), ed_sk.size()); + return true; + } catch (...) { + return false; + } +} + +LIBSESSION_C_API bool session_ed25519_key_pair_seed( + const unsigned char* ed25519_seed, + unsigned char* ed25519_pk_out, + unsigned char* ed25519_sk_out) { + try { + auto [ed_pk, ed_sk] = session::ed25519::keypair(to_byte_span<32>(ed25519_seed)); + std::memcpy(ed25519_pk_out, ed_pk.data(), ed_pk.size()); + std::memcpy(ed25519_sk_out, ed_sk.data(), ed_sk.size()); + return true; + } catch (...) { + return false; + } +} + +LIBSESSION_C_API bool session_seed_for_ed_privkey( + const unsigned char* ed25519_privkey, unsigned char* ed25519_seed_out) { + try { + auto result = session::ed25519::extract_seed(to_byte_span<64>(ed25519_privkey)); + std::memcpy(ed25519_seed_out, result.data(), result.size()); + return true; + } catch (...) { + return false; + } +} + +LIBSESSION_C_API bool session_ed25519_sign( + const unsigned char* ed25519_privkey, + const unsigned char* msg, + size_t msg_len, + unsigned char* ed25519_sig_out) { + try { + auto result = session::ed25519::sign( + to_byte_span<64>(ed25519_privkey), to_byte_span(msg, msg_len)); + std::memcpy(ed25519_sig_out, result.data(), result.size()); + return true; + } catch (...) { + return false; + } +} + +LIBSESSION_C_API bool session_ed25519_verify( + const unsigned char* sig, + const unsigned char* pubkey, + const unsigned char* msg, + size_t msg_len) { + return session::ed25519::verify( + to_byte_span<64>(sig), to_byte_span<32>(pubkey), to_byte_span(msg, msg_len)); +} + +LIBSESSION_C_API bool session_ed25519_pro_privkey_for_ed25519_seed( + const unsigned char* ed25519_seed, unsigned char* ed25519_sk_out) { + try { + auto [pub, sk] = session::ed25519::derive_subkey( + to_byte_span<32>(ed25519_seed), session::pro_backend::pro_subkey_domain); + std::memcpy(ed25519_sk_out, sk.data(), sk.size()); + return true; + } catch (...) { + return false; + } +} diff --git a/src/crypto/mlkem768.cpp b/src/crypto/mlkem768.cpp new file mode 100644 index 000000000..f49d4fba2 --- /dev/null +++ b/src/crypto/mlkem768.cpp @@ -0,0 +1,47 @@ +#include "session/crypto/mlkem768.hpp" + +#include + +#include + +namespace session::mlkem768 { + +static_assert(PUBLICKEYBYTES == MLKEM768_PUBLICKEYBYTES); +static_assert(SECRETKEYBYTES == MLKEM768_SECRETKEYBYTES); +static_assert(CIPHERTEXTBYTES == MLKEM768_CIPHERTEXTBYTES); +static_assert(SHAREDSECRETBYTES == MLKEM_SYMBYTES); +static_assert(SEEDBYTES == 2 * MLKEM_SYMBYTES); + +void keygen( + std::span pk, + std::span sk, + std::span seed) { + if (0 != sr_mlkem768_keypair_derand( + to_unsigned(pk.data()), to_unsigned(sk.data()), to_unsigned(seed.data()))) + throw std::runtime_error{"ML-KEM-768 keygen failed"}; +} + +void encapsulate( + std::span ciphertext, + std::span shared_secret, + std::span pk, + std::span seed) { + if (0 != sr_mlkem768_enc_derand( + to_unsigned(ciphertext.data()), + to_unsigned(shared_secret.data()), + to_unsigned(pk.data()), + to_unsigned(seed.data()))) + throw std::runtime_error{"ML-KEM-768 encapsulation failed"}; +} + +bool decapsulate( + std::span shared_secret, + std::span ciphertext, + std::span sk) { + return 0 == sr_mlkem768_dec( + to_unsigned(shared_secret.data()), + to_unsigned(ciphertext.data()), + to_unsigned(sk.data())); +} + +} // namespace session::mlkem768 diff --git a/src/crypto/x25519.cpp b/src/crypto/x25519.cpp new file mode 100644 index 000000000..872399e3c --- /dev/null +++ b/src/crypto/x25519.cpp @@ -0,0 +1,58 @@ +#include "session/crypto/x25519.hpp" + +#include +#include + +namespace session::x25519 { + +void keypair(std::span pk, std::span sk) { + crypto_box_keypair(to_unsigned(pk.data()), to_unsigned(sk.data())); +} + +std::pair keypair() { + std::pair kp; + keypair(kp.first, kp.second); + return kp; +} + +void seed_keypair( + std::span pk, + std::span sk, + std::span seed) { + crypto_box_seed_keypair( + to_unsigned(pk.data()), to_unsigned(sk.data()), to_unsigned(seed.data())); +} + +std::pair seed_keypair(std::span seed) { + std::pair kp; + seed_keypair(kp.first, kp.second, seed); + return kp; +} + +void scalarmult_base(std::span out, std::span scalar) { + crypto_scalarmult_curve25519_base(to_unsigned(out.data()), to_unsigned(scalar.data())); +} + +b32 scalarmult_base(std::span scalar) { + b32 out; + scalarmult_base(out, scalar); + return out; +} + +bool scalarmult( + std::span out, + std::span scalar, + std::span point) { + return 0 == + crypto_scalarmult_curve25519( + to_unsigned(out.data()), to_unsigned(scalar.data()), to_unsigned(point.data())); +} + +b32 scalarmult(std::span scalar, std::span point) { + b32 out; + if (!scalarmult(out, scalar, point)) + throw std::runtime_error{"x25519 scalarmult failed (degenerate point)"}; + return out; +} + +} // namespace session::x25519 diff --git a/src/curve25519.cpp b/src/curve25519.cpp index a9daea6cf..5bc45bee2 100644 --- a/src/curve25519.cpp +++ b/src/curve25519.cpp @@ -1,63 +1,24 @@ -#include "session/curve25519.hpp" +#include "session/curve25519.h" -#include -#include - -#include +#include +#include "session/crypto/ed25519.hpp" +#include "session/crypto/x25519.hpp" #include "session/export.h" #include "session/util.hpp" -namespace session::curve25519 { - -std::pair, std::array> curve25519_key_pair() { - std::array curve_pk; - std::array curve_sk; - crypto_box_keypair(curve_pk.data(), curve_sk.data()); - - return {curve_pk, curve_sk}; -} - -std::array to_curve25519_pubkey(std::span ed25519_pubkey) { - if (ed25519_pubkey.size() != 32) { - throw std::invalid_argument{"Invalid ed25519_pubkey: expected 32 bytes"}; - } - - std::array curve_pk; - - if (0 != crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed25519_pubkey.data())) - throw std::runtime_error{ - "An error occured while attempting to convert Ed25519 pubkey to curve25519; " - "is the pubkey valid?"}; - - return curve_pk; -} - -std::array to_curve25519_seckey(std::span ed25519_seckey) { - if (ed25519_seckey.size() != 64) { - throw std::invalid_argument{"Invalid ed25519_seckey: expected 64 bytes"}; - } - - std::array curve_sk; - if (0 != crypto_sign_ed25519_sk_to_curve25519(curve_sk.data(), ed25519_seckey.data())) - throw std::runtime_error{ - "An error occured while attempting to convert Ed25519 pubkey to curve25519; " - "is the seckey valid?"}; - - return curve_sk; -} - -} // namespace session::curve25519 +// This file provides the C API wrappers for curve25519/x25519 operations. The C++ functions +// these previously wrapped (session::curve25519::*) have been replaced by session::x25519::* and +// session::ed25519::*. using namespace session; LIBSESSION_C_API bool session_curve25519_key_pair( unsigned char* curve25519_pk_out, unsigned char* curve25519_sk_out) { try { - auto result = session::curve25519::curve25519_key_pair(); - auto [curve_pk, curve_sk] = result; - std::memcpy(curve25519_pk_out, curve_pk.data(), curve_pk.size()); - std::memcpy(curve25519_sk_out, curve_sk.data(), curve_sk.size()); + auto [pk, sk] = x25519::keypair(); + std::memcpy(curve25519_pk_out, pk.data(), pk.size()); + std::memcpy(curve25519_sk_out, sk.data(), sk.size()); return true; } catch (...) { return false; @@ -67,9 +28,8 @@ LIBSESSION_C_API bool session_curve25519_key_pair( LIBSESSION_C_API bool session_to_curve25519_pubkey( const unsigned char* ed25519_pubkey, unsigned char* curve25519_pk_out) { try { - auto curve_pk = session::curve25519::to_curve25519_pubkey( - std::span{ed25519_pubkey, 32}); - std::memcpy(curve25519_pk_out, curve_pk.data(), curve_pk.size()); + auto xpk = ed25519::pk_to_x25519(to_byte_span<32>(ed25519_pubkey)); + std::memcpy(curve25519_pk_out, xpk.data(), xpk.size()); return true; } catch (...) { return false; @@ -79,9 +39,8 @@ LIBSESSION_C_API bool session_to_curve25519_pubkey( LIBSESSION_C_API bool session_to_curve25519_seckey( const unsigned char* ed25519_seckey, unsigned char* curve25519_sk_out) { try { - auto curve_sk = session::curve25519::to_curve25519_seckey( - std::span{ed25519_seckey, 64}); - std::memcpy(curve25519_sk_out, curve_sk.data(), curve_sk.size()); + auto xsk = ed25519::sk_to_x25519(to_byte_span<64>(ed25519_seckey)); + std::memcpy(curve25519_sk_out, xsk.data(), xsk.size()); return true; } catch (...) { return false; diff --git a/src/ed25519.cpp b/src/ed25519.cpp deleted file mode 100644 index bce3d269a..000000000 --- a/src/ed25519.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "session/ed25519.hpp" - -#include -#include -#include - -#include - -#include "session/export.h" -#include "session/sodium_array.hpp" - -template -using uc32 = std::array; -using uc64 = std::array; - -namespace { -uc64 derived_ed25519_privkey(std::span ed25519_seed, std::string_view key) { - if (ed25519_seed.size() != 32 && ed25519_seed.size() != 64) - throw std::invalid_argument{ - "Invalid ed25519_seed: expected 32 bytes or libsodium style 64 bytes seed"}; - - // Construct seed for derived key - // new_seed = Blake2b32(ed25519_seed, key=) - // b/B = Ed25519FromSeed(new_seed) - session::cleared_uc32 s2 = {}; - int hash_result = crypto_generichash_blake2b( - s2.data(), - s2.size(), - ed25519_seed.data(), - ed25519_seed.size(), - reinterpret_cast(key.data()), - key.size()); - assert(hash_result == 0); // This function can't return 0 unless misused - - auto [pubkey, privkey] = session::ed25519::ed25519_key_pair(s2); - return privkey; -} -} // namespace - -namespace session::ed25519 { - -std::pair, std::array> ed25519_key_pair() { - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_keypair(ed_pk.data(), ed_sk.data()); - - return {ed_pk, ed_sk}; -} - -std::pair, std::array> ed25519_key_pair( - std::span ed25519_seed) { - if (ed25519_seed.size() != 32) { - throw std::invalid_argument{"Invalid ed25519_seed: expected 32 bytes"}; - } - - std::array ed_pk; - std::array ed_sk; - - crypto_sign_ed25519_seed_keypair(ed_pk.data(), ed_sk.data(), ed25519_seed.data()); - - return {ed_pk, ed_sk}; -} - -std::array seed_for_ed_privkey(std::span ed25519_privkey) { - std::array seed; - - if (ed25519_privkey.size() == 32 || ed25519_privkey.size() == 64) - // The first 32 bytes of a 64 byte ed25519 private key are the seed, otherwise - // if the provided value is 32 bytes we just assume we were given a seed - std::memcpy(seed.data(), ed25519_privkey.data(), 32); - else - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - - return seed; -} - -std::vector sign( - std::span ed25519_privkey, std::span msg) { - cleared_uc64 ed_sk_from_seed; - if (ed25519_privkey.size() == 32) { - uc32 ignore_pk; - crypto_sign_ed25519_seed_keypair( - ignore_pk.data(), ed_sk_from_seed.data(), ed25519_privkey.data()); - ed25519_privkey = {ed_sk_from_seed.data(), ed_sk_from_seed.size()}; - } else if (ed25519_privkey.size() != 64) { - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - } - - std::vector sig; - sig.resize(64); - - if (0 != crypto_sign_ed25519_detached( - sig.data(), nullptr, msg.data(), msg.size(), ed25519_privkey.data())) - throw std::runtime_error{"Failed to sign; perhaps the secret key is invalid?"}; - - return sig; -} - -bool verify( - std::span sig, - std::span pubkey, - std::span msg) { - if (sig.size() != 64) - throw std::invalid_argument{"Invalid sig: expected 64 bytes"}; - if (pubkey.size() != 32) - throw std::invalid_argument{"Invalid pubkey: expected 32 bytes"}; - - return (0 == - crypto_sign_ed25519_verify_detached(sig.data(), msg.data(), msg.size(), pubkey.data())); -} - -std::array ed25519_pro_privkey_for_ed25519_seed( - std::span ed25519_seed) { - auto result = derived_ed25519_privkey(ed25519_seed, "SessionProRandom"); - return result; -} -} // namespace session::ed25519 - -using namespace session; - -LIBSESSION_C_API bool session_ed25519_key_pair( - unsigned char* ed25519_pk_out, unsigned char* ed25519_sk_out) { - try { - auto result = session::ed25519::ed25519_key_pair(); - auto [ed_pk, ed_sk] = result; - std::memcpy(ed25519_pk_out, ed_pk.data(), ed_pk.size()); - std::memcpy(ed25519_sk_out, ed_sk.data(), ed_sk.size()); - return true; - } catch (...) { - return false; - } -} - -LIBSESSION_C_API bool session_ed25519_key_pair_seed( - const unsigned char* ed25519_seed, - unsigned char* ed25519_pk_out, - unsigned char* ed25519_sk_out) { - try { - auto result = session::ed25519::ed25519_key_pair( - std::span{ed25519_seed, 32}); - auto [ed_pk, ed_sk] = result; - std::memcpy(ed25519_pk_out, ed_pk.data(), ed_pk.size()); - std::memcpy(ed25519_sk_out, ed_sk.data(), ed_sk.size()); - return true; - } catch (...) { - return false; - } -} - -LIBSESSION_C_API bool session_seed_for_ed_privkey( - const unsigned char* ed25519_privkey, unsigned char* ed25519_seed_out) { - try { - auto result = session::ed25519::seed_for_ed_privkey( - std::span{ed25519_privkey, 64}); - std::memcpy(ed25519_seed_out, result.data(), result.size()); - return true; - } catch (...) { - return false; - } -} - -LIBSESSION_C_API bool session_ed25519_sign( - const unsigned char* ed25519_privkey, - const unsigned char* msg, - size_t msg_len, - unsigned char* ed25519_sig_out) { - try { - auto result = session::ed25519::sign( - std::span{ed25519_privkey, 64}, - std::span{msg, msg_len}); - std::memcpy(ed25519_sig_out, result.data(), result.size()); - return true; - } catch (...) { - return false; - } -} - -LIBSESSION_C_API bool session_ed25519_verify( - const unsigned char* sig, - const unsigned char* pubkey, - const unsigned char* msg, - size_t msg_len) { - return session::ed25519::verify( - std::span{sig, 64}, - std::span{pubkey, 32}, - std::span{msg, msg_len}); -} - -LIBSESSION_C_API bool session_ed25519_pro_privkey_for_ed25519_seed( - const unsigned char* ed25519_seed, unsigned char* ed25519_sk_out) { - try { - auto seed = std::span(ed25519_seed, 32); - uc64 sk = session::ed25519::ed25519_pro_privkey_for_ed25519_seed(seed); - std::memcpy(ed25519_sk_out, sk.data(), sk.size()); - return true; - } catch (...) { - return false; - } -} diff --git a/src/fields.cpp b/src/fields.cpp deleted file mode 100644 index b9b2515f0..000000000 --- a/src/fields.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "session/fields.hpp" - -#include - -#include - -namespace session { - -std::string SessionID::hex() const { - std::string id; - id.reserve(33); - id.push_back(static_cast(netid)); - oxenc::to_hex(pubkey.begin(), pubkey.end(), std::back_inserter(id)); - return id; -} - -} // namespace session diff --git a/src/hash.cpp b/src/hash.cpp index b698f6b31..a5ba9aa79 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -3,14 +3,17 @@ #include #include "session/export.h" +#include "session/hash.h" #include "session/util.hpp" -namespace session::hash { +namespace { -void hash( - std::span hash, - std::span msg, - std::optional> key) { +using namespace session; + +void hash_impl( + std::span hash, + std::span msg, + std::optional> key) { const auto size = hash.size(); if (size < crypto_generichash_blake2b_BYTES_MIN || size > crypto_generichash_blake2b_BYTES_MAX) throw std::invalid_argument{"Invalid size: expected between 16 and 64 bytes (inclusive)"}; @@ -19,22 +22,31 @@ void hash( throw std::invalid_argument{"Invalid key: expected less than 65 bytes"}; crypto_generichash_blake2b( - hash.data(), + to_unsigned(hash.data()), size, - msg.data(), + to_unsigned(msg.data()), msg.size(), - key ? key->data() : nullptr, + key ? to_unsigned(key->data()) : nullptr, key ? key->size() : 0); } -std::vector hash( - const size_t size, - std::span msg, - std::optional> key) { - std::vector result; - result.resize(size); - hash(result, msg, key); +} // namespace + +namespace session::hash { +void hash( + std::span hash, + std::span msg, + std::optional> key) { + hash_impl(hash, msg, key); +} + +std::vector hash( + const size_t size, + std::span msg, + std::optional> key) { + std::vector result(size); + hash_impl(result, msg, key); return result; } @@ -50,13 +62,15 @@ LIBSESSION_C_API bool session_hash( size_t key_len, unsigned char* hash_out) { try { - std::optional> key; + std::optional> key; if (key_in && key_len) - key = {key_in, key_len}; + key = std::span{reinterpret_cast(key_in), key_len}; - std::vector result = session::hash::hash(size, {msg_in, msg_len}, key); - std::memcpy(hash_out, result.data(), size); + hash_impl( + std::span{reinterpret_cast(hash_out), size}, + std::span{reinterpret_cast(msg_in), msg_len}, + key); return true; } catch (...) { return false; diff --git a/src/internal-util.hpp b/src/internal-util.hpp index 2525666a3..67bc5fd68 100644 --- a/src/internal-util.hpp +++ b/src/internal-util.hpp @@ -1,22 +1,67 @@ #pragma once +#include + +#include #include +#include +#include #include +using namespace session::literals; + namespace session { -// Used by various C APIs with false returns to write a caught exception message into an error -// buffer (if provided) on the way out. The error buffer is expected to have at least 256 bytes -// available (the exception message will be truncated if longer than 255). -inline bool set_error(char* error, const std::exception& e) { - if (error) { - std::string_view err{e.what()}; - if (err.size() > 255) - err.remove_suffix(err.size() - 255); - std::memcpy(error, err.data(), err.size()); - error[err.size()] = 0; - } - - return false; +// Counts the run of trailing `value` elements at the end of a range. For a non-resizable range +// such as a span, pair this with `.first(size() - count_trailing(...))`. +template + requires std::equality_comparable> +auto count_trailing(const R& r, const std::ranges::range_value_t& value = {}) { + return std::ranges::distance( + r | std::views::reverse | + std::views::take_while([&value](const auto& v) { return v == value; })); +} + +// Trims any run of trailing `trim` values off the end of a resizable container, in place. A +// container consisting entirely of `trim` values is left empty. +// +// Typically used to strip the null padding off a decrypted payload: +// +// trim_trailing(plaintext); +template + requires std::ranges::sized_range && + std::equality_comparable> +void trim_trailing(Container& c, const std::ranges::range_value_t& trim = {}) { + c.resize(c.size() - count_trailing(c, trim)); +} + +// Copies `msg` into `buf`, truncating if necessary, always null-terminating. Returns the number +// of bytes written INCLUDING the null terminator (i.e. the number of bytes of `buf` that were +// touched), or 0 if buf is null/empty. +inline size_t copy_c_str(char* buf, size_t buf_len, std::string_view msg) { + if (!buf || !buf_len) + return 0; + auto n = std::min(msg.size(), buf_len - 1); + std::memcpy(buf, msg.data(), n); + buf[n++] = 0; + return n; +} + +// Overload for fixed-size char arrays; deduces the buffer size automatically. +template +size_t copy_c_str(char (&buf)[N], std::string_view msg) { + return copy_c_str(buf, N, msg); +} + +// Formats a message directly into a buffer with compile-time format checking. Truncates if +// necessary, always null-terminates. Returns the number of bytes written INCLUDING the null +// terminator, or 0 if buf is null/empty. +template +size_t format_c_str(char* buf, size_t buf_len, fmt::format_string format, Args&&... args) { + if (!buf || !buf_len) + return 0; + auto result = fmt::format_to_n(buf, buf_len - 1, format, std::forward(args)...); + *result.out = '\0'; + return static_cast(result.out - buf) + 1; } } // namespace session diff --git a/src/json_parser.hpp b/src/json_parser.hpp index 9ec65b62a..4bf646122 100644 --- a/src/json_parser.hpp +++ b/src/json_parser.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -9,23 +10,24 @@ #include #include #include +#include #include #include -namespace session::detail { +namespace session::json { // A T that is carried on the wire as an integer count of seconds: either a duration (seconds) or a -// system-clock time point at second granularity (sys_seconds). json_require/json_maybe read +// system-clock time point at second granularity (sys_seconds). require/maybe read // the integer and wrap it, so a call site never repeats `sys_seconds{seconds{get()}}`. template concept wire_seconds = std::same_as || std::same_as; // Whether the JSON value `v` holds a T, paired with a human name for the type (used in the error -// message on a mismatch). Single source of truth so json_require and json_maybe cannot drift on +// message on a mismatch). Single source of truth so require and maybe cannot drift on // what counts as a valid T. template -std::pair json_is(const nlohmann::json& v) { +std::pair is(const nlohmann::json& v) { if constexpr (wire_seconds) // Integer seconds on the wire; reject a fractional value rather than truncating it. return {v.is_number_integer(), "an integer"}; @@ -39,7 +41,7 @@ std::pair json_is(const nlohmann::json& v) { return {v.is_number_integer(), "an integer"}; else if constexpr (std::is_enum_v) // A (scoped) enum reads as its underlying integer -- nlohmann's default serializer converts - // through the underlying type (json_extract's get_to) -- so callers can request the enum + // through the underlying type (extract's get_to) -- so callers can request the enum // directly rather than reading an integer and casting. No caller does today: this arrived // for the proof `version` read, which no longer exists. return {v.is_number_integer(), "an integer"}; @@ -55,7 +57,7 @@ std::pair json_is(const nlohmann::json& v) { // Extracts an already-type-validated value as T, applying the seconds-wrapping for wire_seconds. template -T json_extract(const nlohmann::json& v) { +T extract(const nlohmann::json& v) { if constexpr (wire_seconds) return T{std::chrono::seconds{v.template get()}}; else { @@ -66,7 +68,7 @@ T json_extract(const nlohmann::json& v) { } // Parse `input` as JSON, throwing parse_error (not a nlohmann exception) if it is not valid JSON. -inline nlohmann::json json_parse(std::string_view input) { +inline nlohmann::json parse(std::string_view input) { try { return nlohmann::json::parse(input); } catch (const std::exception& e) { @@ -76,46 +78,81 @@ inline nlohmann::json json_parse(std::string_view input) { // Reads a required field: throws parse_error_missing if absent, parse_error_type if the wrong type. template -T json_require(const nlohmann::json& j, std::string_view key) { +T require(const nlohmann::json& j, std::string_view key) { auto it = j.find(key); if (it == j.end()) throw parse_error_missing{key}; - if (auto [ok, type] = json_is(*it); !ok) + if (auto [ok, type] = is(*it); !ok) throw parse_error_type{key, type, it->dump(1)}; - return json_extract(*it); + return extract(*it); } // Reads an optional field: a missing key or a wrong-typed value both yield nullopt (rather than // throwing) -- for advisory fields a caller should read leniently and skip when absent. template -std::optional json_maybe(const nlohmann::json& j, std::string_view key) { +std::optional maybe(const nlohmann::json& j, std::string_view key) { auto it = j.find(key); - if (it == j.end() || !json_is(*it).first) + if (it == j.end() || !is(*it).first) return std::nullopt; - return json_extract(*it); + return extract(*it); } -inline void json_require_hex( - const nlohmann::json& j, std::string_view key, std::span dest) { - auto hex = json_require(j, key); - if (hex.starts_with("0X") || hex.starts_with("0x")) - hex = hex.substr(2); +// Reads a fixed-length binary value (a pubkey, a signature, a tag) that the wire carries either +// hex- or base64-encoded, filling `dest` exactly. `dest.size()` is the expected byte length, and +// the encoding is identified from the encoded length alone -- no sniffing of the alphabet, which +// cannot distinguish the two in general (any hex string is also valid base64). +// +// The destination must be a fixed-size byte buffer (a std::array, a C array, or an already +// fixed-extent span) of at least 5 bytes: below that the encodings collide in length and cannot be +// told apart, so the requirement is enforced at compile time. For 1 byte hex and unpadded base64 +// are both 2 chars, for 2 bytes hex and padded base64 are both 4, and for 4 bytes both are 8. From +// 5 bytes up hex (2N) is strictly longer than padded base64 (4*ceil(N/3) <= (4N+8)/3) and than +// unpadded (ceil(4N/3) <= (4N+2)/3), so the three lengths are always distinct. +template + requires requires(Dest& d) { std::span{d}; } +inline void require_binary(const nlohmann::json& j, std::string_view key, Dest& dest_) { + std::span dest{dest_}; + using D = decltype(dest); + static_assert( + std::same_as, + "require_binary writes into a std::byte buffer"); + static_assert( + D::extent != std::dynamic_extent, + "require_binary needs a fixed-size destination: the byte length is what selects the " + "encoding"); + static_assert( + D::extent >= 5, + "require_binary cannot disambiguate hex from base64 below 5 bytes (at 1, 2 and 4 bytes " + "the encoded lengths coincide)"); - size_t hex_avail = dest.size() * 2; - if (hex.size() != hex_avail) + auto enc = require(j, key); + + const auto hex_size = oxenc::to_hex_size(dest.size()); + const auto b64_padded = oxenc::to_base64_size(dest.size(), true); + const auto b64_unpadded = oxenc::to_base64_size(dest.size(), false); + + if (enc.size() == hex_size) { + if (!oxenc::is_hex(enc)) + throw session::parse_error_key{ + key, fmt::format("Key value ({}) was not valid hex: '{}'", key, enc)}; + oxenc::from_hex(enc.begin(), enc.end(), dest.begin()); + } else if (enc.size() == b64_padded || enc.size() == b64_unpadded) { + if (!oxenc::is_base64(enc)) + throw session::parse_error_key{ + key, fmt::format("Key value ({}) was not valid base64: '{}'", key, enc)}; + oxenc::from_base64(enc.begin(), enc.end(), dest.begin()); + } else throw session::parse_error_key{ key, fmt::format( - "Hex -> bytes failed ({}, {}). {} hex chars capacity (requires {})", + "Key value ({}) was not a {}-byte value: expected {} hex chars or {}/{} " + "base64 chars, got {}", key, - hex, - hex_avail, - hex.size())}; - - if (!oxenc::is_hex(hex)) - throw session::parse_error_key{ - key, fmt::format("Key value string was not hex: '{}': '{}'", key, hex)}; - oxenc::from_hex(hex.begin(), hex.end(), dest.begin()); + dest.size(), + hex_size, + b64_unpadded, + b64_padded, + enc.size())}; } -} // namespace session::detail +} // namespace session::json diff --git a/src/logging.cpp b/src/logging.cpp index 779b1f669..1be935f29 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -8,6 +8,7 @@ #include "oxen/log/level.hpp" #include "session/export.h" +#include "session/logging.h" namespace session { @@ -23,12 +24,21 @@ std::string_view LogLevel::to_string() const { return log::to_string(spdlog_level()); } -void add_logger(std::function cb) { - log::add_sink(std::make_shared(std::move(cb))); +LoggerHandle add_logger(std::function cb) { + auto sink = std::make_shared(std::move(cb)); + log::add_sink(sink); + return sink; } -void add_logger( +LoggerHandle add_logger( std::function cb) { - log::add_sink(std::make_shared(std::move(cb))); + auto sink = std::make_shared(std::move(cb)); + log::add_sink(sink); + return sink; +} + +void remove_logger(const LoggerHandle& logger) { + if (logger) + log::remove_sink(logger); } void manual_log(std::string_view msg) { diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt new file mode 100644 index 000000000..681b1b594 --- /dev/null +++ b/src/mnemonics/CMakeLists.txt @@ -0,0 +1,114 @@ +file(GLOB LANG_FILES "languages/*.txt") +list(SORT LANG_FILES) + +# Watch the languages directory (for added/removed files) and each individual file (for edits): +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "languages" ${LANG_FILES}) + +# Reorder to put english first +set(LANG_FILES_ORDERED "") +set(ENGLISH_FILE "") +foreach(f IN LISTS LANG_FILES) + get_filename_component(filename "${f}" NAME_WE) + if(filename STREQUAL "english") + set(ENGLISH_FILE "${f}") + else() + list(APPEND LANG_FILES_ORDERED "${f}") + endif() +endforeach() + +if(NOT ENGLISH_FILE STREQUAL "") + set(LANG_FILES "${ENGLISH_FILE}" ${LANG_FILES_ORDERED}) +else() + set(LANG_FILES ${LANG_FILES_ORDERED}) +endif() + +set(MNEMONIC_SOURCES "") +set(LANG_EXTERNS "") +set(LANG_LIST "") +set(LANG_COUNT 0) + +foreach(f IN LISTS LANG_FILES) + get_filename_component(lang_var "${f}" NAME_WE) + + # Read file and split by newline manually to avoid semicolon/parentheses issues + file(READ "${f}" RAW_CONTENT) + string(REPLACE "\n" ";" LINES "${RAW_CONTENT}") + + list(LENGTH LINES LINE_COUNT) + # Check if we have exactly 1629 lines. + # If the file ends with a newline, file(READ) + string(REPLACE) might produce 1630 elements. + # We'll normalize this by checking for an empty last element if LINE_COUNT is 1630. + if(LINE_COUNT EQUAL 1630) + list(GET LINES 1629 LAST_LINE) + if(LAST_LINE STREQUAL "") + list(REMOVE_AT LINES 1629) + set(LINE_COUNT 1629) + endif() + endif() + + if(NOT LINE_COUNT EQUAL 1629) + message(FATAL_ERROR "Language file ${f} has ${LINE_COUNT} lines, expected 1629.") + endif() + + list(GET LINES 0 ENGLISH_NAME) + list(GET LINES 1 NATIVE_NAME) + list(GET LINES 2 PREFIX_LEN) + + set(WORDS_CODE "") + # Exactly 1626 words starting at index 3. + foreach(I RANGE 3 1628) + list(GET LINES ${I} word) + if(word STREQUAL "") + message(FATAL_ERROR "Empty word found in ${f} at index ${I}") + endif() + if(word MATCHES "[ \t]") + message(FATAL_ERROR "Word '${word}' in ${f} contains whitespace") + endif() + string(APPEND WORDS_CODE " \"${word}\",\n") + endforeach() + + set(GENERATED_CPP "${CMAKE_CURRENT_BINARY_DIR}/lang_${lang_var}.cpp") + file(WRITE "${GENERATED_CPP}" +"#include + +namespace session::mnemonics { + +extern const Mnemonics ${lang_var}; +const Mnemonics ${lang_var} = { + \"${ENGLISH_NAME}\", + \"${NATIVE_NAME}\", + ${PREFIX_LEN}, + {{ +${WORDS_CODE} }} +}; + +} // namespace session::mnemonics +") + list(APPEND MNEMONIC_SOURCES "${GENERATED_CPP}") + string(APPEND LANG_EXTERNS "extern const Mnemonics ${lang_var};\n") + string(APPEND LANG_LIST " &${lang_var},\n") + math(EXPR LANG_COUNT "${LANG_COUNT} + 1") +endforeach() + +set(MASTER_CPP "${CMAKE_CURRENT_BINARY_DIR}/mnemonics_registry.cpp") +file(WRITE "${MASTER_CPP}" +"#include +#include + +namespace session::mnemonics { + +${LANG_EXTERNS} +static constexpr std::array all_languages = {{ +${LANG_LIST}}}; + +std::span get_languages() { + return all_languages; +} + +} // namespace session::mnemonics +") + +list(APPEND MNEMONIC_SOURCES "${MASTER_CPP}") + +# Add the generated sources to the crypto library +target_sources(crypto PRIVATE ${MNEMONIC_SOURCES} mnemonics.cpp) diff --git a/src/mnemonics/languages/README.md b/src/mnemonics/languages/README.md new file mode 100644 index 000000000..d8b08b4ee --- /dev/null +++ b/src/mnemonics/languages/README.md @@ -0,0 +1,46 @@ +# Mnemonic Language Word Lists + +This directory contains word lists for different languages used in mnemonic seed generation. + +A mnemonic seed phrase consists of a multiple of 3 words (typically 12 or 24 words), optionally +followed by a checksum, where each group of 3 words encodes a 4 byte (32 bit) value. Thus 12 words +is used for a 128-bit value and 24 words for a 256-bit value. + +Each language has a unique "prefix length" which indicates the word prefix required: i.e. if +set to 3 then any 3-character sequence should match at most one word in the list. This also allows +faster seed word input by allowing a user to simply provide the first three letters (e.g. "ver" +instead of "verification"). + +For unjustifiable by fixed historical reasons, the encoding also uses a pointless complication in +the actual calculation: rather than each 32-bit chunk being computed as `A + B·1626 + C·1626²` +(where A, B, C are the 0-1625 indices of the words) it is instead computed as: + + V = A + + ((1626 - A + B) % 1626) × 1626 + + ((1626 - B + C) % 1626) × 1626² + +The little-endian encoding of this 4-byte value becomes the 32-bit value. + +(Note that that are a relatively small number of "impossible" seed values here that would overflow +this calculation: these are explicitly not allowed as valid seeds by failing if the above +calculation overflows a 32-bit integer). + +This entirely pointless complication has some misguided historical reasoning about trying to make +poor entropy values not look so poor (e.g. by repeating words), but that is just so incredibly +misguided that it should be given no weight. Unfortunately, however, this is already in use and we +are stuck with it. + +Computing A, B, and C *from* a 32-bit value X is performed by interpreting X as a little-endian, +unsigned 32-bit value V and then: + + A = V % 1626 + B = ((V / 1626) + A) % 1626 + C = ((V / 1626²) + B) % 1626 + +## File Format + +Each `.txt` file consists of 1629 lines, following this structure: +1. English name of the language (e.g., `German`) +2. Native name of the language (e.g., `Deutsch`) +3. Unique prefix length (e.g., `4`). (The script utils/verify_mnemonics.py can verify this.) +4. 1626 lines, each containing a single word from the word list, in order. diff --git a/src/mnemonics/languages/chinese_simplified.txt b/src/mnemonics/languages/chinese_simplified.txt new file mode 100644 index 000000000..59a5adfe2 --- /dev/null +++ b/src/mnemonics/languages/chinese_simplified.txt @@ -0,0 +1,1629 @@ +Chinese (simplified) +简体中文 (中国) +1 +的 +一 +是 +在 +不 +了 +有 +和 +人 +这 +中 +大 +为 +上 +个 +国 +我 +以 +要 +他 +时 +来 +用 +们 +生 +到 +作 +地 +于 +出 +就 +分 +对 +成 +会 +可 +主 +发 +年 +动 +同 +工 +也 +能 +下 +过 +子 +说 +产 +种 +面 +而 +方 +后 +多 +定 +行 +学 +法 +所 +民 +得 +经 +十 +三 +之 +进 +着 +等 +部 +度 +家 +电 +力 +里 +如 +水 +化 +高 +自 +二 +理 +起 +小 +物 +现 +实 +加 +量 +都 +两 +体 +制 +机 +当 +使 +点 +从 +业 +本 +去 +把 +性 +好 +应 +开 +它 +合 +还 +因 +由 +其 +些 +然 +前 +外 +天 +政 +四 +日 +那 +社 +义 +事 +平 +形 +相 +全 +表 +间 +样 +与 +关 +各 +重 +新 +线 +内 +数 +正 +心 +反 +你 +明 +看 +原 +又 +么 +利 +比 +或 +但 +质 +气 +第 +向 +道 +命 +此 +变 +条 +只 +没 +结 +解 +问 +意 +建 +月 +公 +无 +系 +军 +很 +情 +者 +最 +立 +代 +想 +已 +通 +并 +提 +直 +题 +党 +程 +展 +五 +果 +料 +象 +员 +革 +位 +入 +常 +文 +总 +次 +品 +式 +活 +设 +及 +管 +特 +件 +长 +求 +老 +头 +基 +资 +边 +流 +路 +级 +少 +图 +山 +统 +接 +知 +较 +将 +组 +见 +计 +别 +她 +手 +角 +期 +根 +论 +运 +农 +指 +几 +九 +区 +强 +放 +决 +西 +被 +干 +做 +必 +战 +先 +回 +则 +任 +取 +据 +处 +队 +南 +给 +色 +光 +门 +即 +保 +治 +北 +造 +百 +规 +热 +领 +七 +海 +口 +东 +导 +器 +压 +志 +世 +金 +增 +争 +济 +阶 +油 +思 +术 +极 +交 +受 +联 +什 +认 +六 +共 +权 +收 +证 +改 +清 +美 +再 +采 +转 +更 +单 +风 +切 +打 +白 +教 +速 +花 +带 +安 +场 +身 +车 +例 +真 +务 +具 +万 +每 +目 +至 +达 +走 +积 +示 +议 +声 +报 +斗 +完 +类 +八 +离 +华 +名 +确 +才 +科 +张 +信 +马 +节 +话 +米 +整 +空 +元 +况 +今 +集 +温 +传 +土 +许 +步 +群 +广 +石 +记 +需 +段 +研 +界 +拉 +林 +律 +叫 +且 +究 +观 +越 +织 +装 +影 +算 +低 +持 +音 +众 +书 +布 +复 +容 +儿 +须 +际 +商 +非 +验 +连 +断 +深 +难 +近 +矿 +千 +周 +委 +素 +技 +备 +半 +办 +青 +省 +列 +习 +响 +约 +支 +般 +史 +感 +劳 +便 +团 +往 +酸 +历 +市 +克 +何 +除 +消 +构 +府 +称 +太 +准 +精 +值 +号 +率 +族 +维 +划 +选 +标 +写 +存 +候 +毛 +亲 +快 +效 +斯 +院 +查 +江 +型 +眼 +王 +按 +格 +养 +易 +置 +派 +层 +片 +始 +却 +专 +状 +育 +厂 +京 +识 +适 +属 +圆 +包 +火 +住 +调 +满 +县 +局 +照 +参 +红 +细 +引 +听 +该 +铁 +价 +严 +首 +底 +液 +官 +德 +随 +病 +苏 +失 +尔 +死 +讲 +配 +女 +黄 +推 +显 +谈 +罪 +神 +艺 +呢 +席 +含 +企 +望 +密 +批 +营 +项 +防 +举 +球 +英 +氧 +势 +告 +李 +台 +落 +木 +帮 +轮 +破 +亚 +师 +围 +注 +远 +字 +材 +排 +供 +河 +态 +封 +另 +施 +减 +树 +溶 +怎 +止 +案 +言 +士 +均 +武 +固 +叶 +鱼 +波 +视 +仅 +费 +紧 +爱 +左 +章 +早 +朝 +害 +续 +轻 +服 +试 +食 +充 +兵 +源 +判 +护 +司 +足 +某 +练 +差 +致 +板 +田 +降 +黑 +犯 +负 +击 +范 +继 +兴 +似 +余 +坚 +曲 +输 +修 +故 +城 +夫 +够 +送 +笔 +船 +占 +右 +财 +吃 +富 +春 +职 +觉 +汉 +画 +功 +巴 +跟 +虽 +杂 +飞 +检 +吸 +助 +升 +阳 +互 +初 +创 +抗 +考 +投 +坏 +策 +古 +径 +换 +未 +跑 +留 +钢 +曾 +端 +责 +站 +简 +述 +钱 +副 +尽 +帝 +射 +草 +冲 +承 +独 +令 +限 +阿 +宣 +环 +双 +请 +超 +微 +让 +控 +州 +良 +轴 +找 +否 +纪 +益 +依 +优 +顶 +础 +载 +倒 +房 +突 +坐 +粉 +敌 +略 +客 +袁 +冷 +胜 +绝 +析 +块 +剂 +测 +丝 +协 +诉 +念 +陈 +仍 +罗 +盐 +友 +洋 +错 +苦 +夜 +刑 +移 +频 +逐 +靠 +混 +母 +短 +皮 +终 +聚 +汽 +村 +云 +哪 +既 +距 +卫 +停 +烈 +央 +察 +烧 +迅 +境 +若 +印 +洲 +刻 +括 +激 +孔 +搞 +甚 +室 +待 +核 +校 +散 +侵 +吧 +甲 +游 +久 +菜 +味 +旧 +模 +湖 +货 +损 +预 +阻 +毫 +普 +稳 +乙 +妈 +植 +息 +扩 +银 +语 +挥 +酒 +守 +拿 +序 +纸 +医 +缺 +雨 +吗 +针 +刘 +啊 +急 +唱 +误 +训 +愿 +审 +附 +获 +茶 +鲜 +粮 +斤 +孩 +脱 +硫 +肥 +善 +龙 +演 +父 +渐 +血 +欢 +械 +掌 +歌 +沙 +刚 +攻 +谓 +盾 +讨 +晚 +粒 +乱 +燃 +矛 +乎 +杀 +药 +宁 +鲁 +贵 +钟 +煤 +读 +班 +伯 +香 +介 +迫 +句 +丰 +培 +握 +兰 +担 +弦 +蛋 +沉 +假 +穿 +执 +答 +乐 +谁 +顺 +烟 +缩 +征 +脸 +喜 +松 +脚 +困 +异 +免 +背 +星 +福 +买 +染 +井 +概 +慢 +怕 +磁 +倍 +祖 +皇 +促 +静 +补 +评 +翻 +肉 +践 +尼 +衣 +宽 +扬 +棉 +希 +伤 +操 +垂 +秋 +宜 +氢 +套 +督 +振 +架 +亮 +末 +宪 +庆 +编 +牛 +触 +映 +雷 +销 +诗 +座 +居 +抓 +裂 +胞 +呼 +娘 +景 +威 +绿 +晶 +厚 +盟 +衡 +鸡 +孙 +延 +危 +胶 +屋 +乡 +临 +陆 +顾 +掉 +呀 +灯 +岁 +措 +束 +耐 +剧 +玉 +赵 +跳 +哥 +季 +课 +凯 +胡 +额 +款 +绍 +卷 +齐 +伟 +蒸 +殖 +永 +宗 +苗 +川 +炉 +岩 +弱 +零 +杨 +奏 +沿 +露 +杆 +探 +滑 +镇 +饭 +浓 +航 +怀 +赶 +库 +夺 +伊 +灵 +税 +途 +灭 +赛 +归 +召 +鼓 +播 +盘 +裁 +险 +康 +唯 +录 +菌 +纯 +借 +糖 +盖 +横 +符 +私 +努 +堂 +域 +枪 +润 +幅 +哈 +竟 +熟 +虫 +泽 +脑 +壤 +碳 +欧 +遍 +侧 +寨 +敢 +彻 +虑 +斜 +薄 +庭 +纳 +弹 +饲 +伸 +折 +麦 +湿 +暗 +荷 +瓦 +塞 +床 +筑 +恶 +户 +访 +塔 +奇 +透 +梁 +刀 +旋 +迹 +卡 +氯 +遇 +份 +毒 +泥 +退 +洗 +摆 +灰 +彩 +卖 +耗 +夏 +择 +忙 +铜 +献 +硬 +予 +繁 +圈 +雪 +函 +亦 +抽 +篇 +阵 +阴 +丁 +尺 +追 +堆 +雄 +迎 +泛 +爸 +楼 +避 +谋 +吨 +野 +猪 +旗 +累 +偏 +典 +馆 +索 +秦 +脂 +潮 +爷 +豆 +忽 +托 +惊 +塑 +遗 +愈 +朱 +替 +纤 +粗 +倾 +尚 +痛 +楚 +谢 +奋 +购 +磨 +君 +池 +旁 +碎 +骨 +监 +捕 +弟 +暴 +割 +贯 +殊 +释 +词 +亡 +壁 +顿 +宝 +午 +尘 +闻 +揭 +炮 +残 +冬 +桥 +妇 +警 +综 +招 +吴 +付 +浮 +遭 +徐 +您 +摇 +谷 +赞 +箱 +隔 +订 +男 +吹 +园 +纷 +唐 +败 +宋 +玻 +巨 +耕 +坦 +荣 +闭 +湾 +键 +凡 +驻 +锅 +救 +恩 +剥 +凝 +碱 +齿 +截 +炼 +麻 +纺 +禁 +废 +盛 +版 +缓 +净 +睛 +昌 +婚 +涉 +筒 +嘴 +插 +岸 +朗 +庄 +街 +藏 +姑 +贸 +腐 +奴 +啦 +惯 +乘 +伙 +恢 +匀 +纱 +扎 +辩 +耳 +彪 +臣 +亿 +璃 +抵 +脉 +秀 +萨 +俄 +网 +舞 +店 +喷 +纵 +寸 +汗 +挂 +洪 +贺 +闪 +柬 +爆 +烯 +津 +稻 +墙 +软 +勇 +像 +滚 +厘 +蒙 +芳 +肯 +坡 +柱 +荡 +腿 +仪 +旅 +尾 +轧 +冰 +贡 +登 +黎 +削 +钻 +勒 +逃 +障 +氨 +郭 +峰 +币 +港 +伏 +轨 +亩 +毕 +擦 +莫 +刺 +浪 +秘 +援 +株 +健 +售 +股 +岛 +甘 +泡 +睡 +童 +铸 +汤 +阀 +休 +汇 +舍 +牧 +绕 +炸 +哲 +磷 +绩 +朋 +淡 +尖 +启 +陷 +柴 +呈 +徒 +颜 +泪 +稍 +忘 +泵 +蓝 +拖 +洞 +授 +镜 +辛 +壮 +锋 +贫 +虚 +弯 +摩 +泰 +幼 +廷 +尊 +窗 +纲 +弄 +隶 +疑 +氏 +宫 +姐 +震 +瑞 +怪 +尤 +琴 +循 +描 +膜 +违 +夹 +腰 +缘 +珠 +穷 +森 +枝 +竹 +沟 +催 +绳 +忆 +邦 +剩 +幸 +浆 +栏 +拥 +牙 +贮 +礼 +滤 +钠 +纹 +罢 +拍 +咱 +喊 +袖 +埃 +勤 +罚 +焦 +潜 +伍 +墨 +欲 +缝 +姓 +刊 +饱 +仿 +奖 +铝 +鬼 +丽 +跨 +默 +挖 +链 +扫 +喝 +袋 +炭 +污 +幕 +诸 +弧 +励 +梅 +奶 +洁 +灾 +舟 +鉴 +苯 +讼 +抱 +毁 +懂 +寒 +智 +埔 +寄 +届 +跃 +渡 +挑 +丹 +艰 +贝 +碰 +拔 +爹 +戴 +码 +梦 +芽 +熔 +赤 +渔 +哭 +敬 +颗 +奔 +铅 +仲 +虎 +稀 +妹 +乏 +珍 +申 +桌 +遵 +允 +隆 +螺 +仓 +魏 +锐 +晓 +氮 +兼 +隐 +碍 +赫 +拨 +忠 +肃 +缸 +牵 +抢 +博 +巧 +壳 +兄 +杜 +讯 +诚 +碧 +祥 +柯 +页 +巡 +矩 +悲 +灌 +龄 +伦 +票 +寻 +桂 +铺 +圣 +恐 +恰 +郑 +趣 +抬 +荒 +腾 +贴 +柔 +滴 +猛 +阔 +辆 +妻 +填 +撤 +储 +签 +闹 +扰 +紫 +砂 +递 +戏 +吊 +陶 +伐 +喂 +疗 +瓶 +婆 +抚 +臂 +摸 +忍 +虾 +蜡 +邻 +胸 +巩 +挤 +偶 +弃 +槽 +劲 +乳 +邓 +吉 +仁 +烂 +砖 +租 +乌 +舰 +伴 +瓜 +浅 +丙 +暂 +燥 +橡 +柳 +迷 +暖 +牌 +秧 +胆 +详 +簧 +踏 +瓷 +谱 +呆 +宾 +糊 +洛 +辉 +愤 +竞 +隙 +怒 +粘 +乃 +绪 +肩 +籍 +敏 +涂 +熙 +皆 +侦 +悬 +掘 +享 +纠 +醒 +狂 +锁 +淀 +恨 +牲 +霸 +爬 +赏 +逆 +玩 +陵 +祝 +秒 +浙 +貌 diff --git a/src/mnemonics/languages/dutch.txt b/src/mnemonics/languages/dutch.txt new file mode 100644 index 000000000..e162ee07e --- /dev/null +++ b/src/mnemonics/languages/dutch.txt @@ -0,0 +1,1629 @@ +Dutch +Nederlands +4 +aalglad +aalscholver +aambeeld +aangeef +aanlandig +aanvaard +aanwakker +aapmens +aarten +abdicatie +abnormaal +abrikoos +accu +acuut +adjudant +admiraal +advies +afbidding +afdracht +affaire +affiche +afgang +afkick +afknap +aflees +afmijner +afname +afpreekt +afrader +afspeel +aftocht +aftrek +afzijdig +ahornboom +aktetas +akzo +alchemist +alcohol +aldaar +alexander +alfabet +alfredo +alice +alikruik +allrisk +altsax +alufolie +alziend +amai +ambacht +ambieer +amina +amnestie +amok +ampul +amuzikaal +angela +aniek +antje +antwerpen +anya +aorta +apache +apekool +appelaar +arganolie +argeloos +armoede +arrenslee +artritis +arubaan +asbak +ascii +asgrauw +asjes +asml +aspunt +asurn +asveld +aterling +atomair +atrium +atsma +atypisch +auping +aura +avifauna +axiaal +azoriaan +azteek +azuur +bachelor +badderen +badhotel +badmantel +badsteden +balie +ballans +balvers +bamibal +banneling +barracuda +basaal +batelaan +batje +beambte +bedlamp +bedwelmd +befaamd +begierd +begraaf +behield +beijaard +bejaagd +bekaaid +beks +bektas +belaad +belboei +belderbos +beloerd +beluchten +bemiddeld +benadeeld +benijd +berechten +beroemd +besef +besseling +best +betichten +bevind +bevochten +bevraagd +bewust +bidplaats +biefstuk +biemans +biezen +bijbaan +bijeenkom +bijfiguur +bijkaart +bijlage +bijpaard +bijtgaar +bijweg +bimmel +binck +bint +biobak +biotisch +biseks +bistro +bitter +bitumen +bizar +blad +bleken +blender +bleu +blief +blijven +blozen +bock +boef +boei +boks +bolder +bolus +bolvormig +bomaanval +bombarde +bomma +bomtapijt +bookmaker +boos +borg +bosbes +boshuizen +bosloop +botanicus +bougie +bovag +boxspring +braad +brasem +brevet +brigade +brinckman +bruid +budget +buffel +buks +bulgaar +buma +butaan +butler +buuf +cactus +cafeetje +camcorder +cannabis +canyon +capoeira +capsule +carkit +casanova +catalaan +ceintuur +celdeling +celplasma +cement +censeren +ceramisch +cerberus +cerebraal +cesium +cirkel +citeer +civiel +claxon +clenbuterol +clicheren +clijsen +coalitie +coassistentschap +coaxiaal +codetaal +cofinanciering +cognac +coltrui +comfort +commandant +condensaat +confectie +conifeer +convector +copier +corfu +correct +coup +couvert +creatie +credit +crematie +cricket +croupier +cruciaal +cruijff +cuisine +culemborg +culinair +curve +cyrano +dactylus +dading +dagblind +dagje +daglicht +dagprijs +dagranden +dakdekker +dakpark +dakterras +dalgrond +dambord +damkat +damlengte +damman +danenberg +debbie +decibel +defect +deformeer +degelijk +degradant +dejonghe +dekken +deppen +derek +derf +derhalve +detineren +devalueer +diaken +dicht +dictaat +dief +digitaal +dijbreuk +dijkmans +dimbaar +dinsdag +diode +dirigeer +disbalans +dobermann +doenbaar +doerak +dogma +dokhaven +dokwerker +doling +dolphijn +dolven +dombo +dooraderd +dopeling +doping +draderig +drama +drenkbak +dreumes +drol +drug +duaal +dublin +duplicaat +durven +dusdanig +dutchbat +dutje +dutten +duur +duwwerk +dwaal +dweil +dwing +dyslexie +ecostroom +ecotaks +educatie +eeckhout +eede +eemland +eencellig +eeneiig +eenruiter +eenwinter +eerenberg +eerrover +eersel +eetmaal +efteling +egaal +egtberts +eickhoff +eidooier +eiland +eind +eisden +ekster +elburg +elevatie +elfkoppig +elfrink +elftal +elimineer +elleboog +elma +elodie +elsa +embleem +embolie +emoe +emonds +emplooi +enduro +enfin +engageer +entourage +entstof +epileer +episch +eppo +erasmus +erboven +erebaan +erelijst +ereronden +ereteken +erfhuis +erfwet +erger +erica +ermitage +erna +ernie +erts +ertussen +eruitzien +ervaar +erven +erwt +esbeek +escort +esdoorn +essing +etage +eter +ethanol +ethicus +etholoog +eufonisch +eurocent +evacuatie +exact +examen +executant +exen +exit +exogeen +exotherm +expeditie +expletief +expres +extase +extinctie +faal +faam +fabel +facultair +fakir +fakkel +faliekant +fallisch +famke +fanclub +fase +fatsoen +fauna +federaal +feedback +feest +feilbaar +feitelijk +felblauw +figurante +fiod +fitheid +fixeer +flap +fleece +fleur +flexibel +flits +flos +flow +fluweel +foezelen +fokkelman +fokpaard +fokvee +folder +follikel +folmer +folteraar +fooi +foolen +forfait +forint +formule +fornuis +fosfaat +foxtrot +foyer +fragiel +frater +freak +freddie +fregat +freon +frijnen +fructose +frunniken +fuiven +funshop +furieus +fysica +gadget +galder +galei +galg +galvlieg +galzuur +ganesh +gaswet +gaza +gazelle +geaaid +gebiecht +gebufferd +gedijd +geef +geflanst +gefreesd +gegaan +gegijzeld +gegniffel +gegraaid +gehikt +gehobbeld +gehucht +geiser +geiten +gekaakt +gekheid +gekijf +gekmakend +gekocht +gekskap +gekte +gelubberd +gemiddeld +geordend +gepoederd +gepuft +gerda +gerijpt +geseald +geshockt +gesierd +geslaagd +gesnaaid +getracht +getwijfel +geuit +gevecht +gevlagd +gewicht +gezaagd +gezocht +ghanees +giebelen +giechel +giepmans +gips +giraal +gistachtig +gitaar +glaasje +gletsjer +gleuf +glibberen +glijbaan +gloren +gluipen +gluren +gluur +gnoe +goddelijk +godgans +godschalk +godzalig +goeierd +gogme +goklustig +gokwereld +gonggrijp +gonje +goor +grabbel +graf +graveer +grif +grolleman +grom +groosman +grubben +gruijs +grut +guacamole +guido +guppy +haazen +hachelijk +haex +haiku +hakhout +hakken +hanegem +hans +hanteer +harrie +hazebroek +hedonist +heil +heineken +hekhuis +hekman +helbig +helga +helwegen +hengelaar +herkansen +hermafrodiet +hertaald +hiaat +hikspoors +hitachi +hitparade +hobo +hoeve +holocaust +hond +honnepon +hoogacht +hotelbed +hufter +hugo +huilbier +hulk +humus +huwbaar +huwelijk +hype +iconisch +idema +ideogram +idolaat +ietje +ijker +ijkheid +ijklijn +ijkmaat +ijkwezen +ijmuiden +ijsbox +ijsdag +ijselijk +ijskoud +ilse +immuun +impliceer +impuls +inbijten +inbuigen +indijken +induceer +indy +infecteer +inhaak +inkijk +inluiden +inmijnen +inoefenen +inpolder +inrijden +inslaan +invitatie +inwaaien +ionisch +isaac +isolatie +isotherm +isra +italiaan +ivoor +jacobs +jakob +jammen +jampot +jarig +jehova +jenever +jezus +joana +jobdienst +josua +joule +juich +jurk +juut +kaas +kabelaar +kabinet +kagenaar +kajuit +kalebas +kalm +kanjer +kapucijn +karregat +kart +katvanger +katwijk +kegelaar +keiachtig +keizer +kenletter +kerdijk +keus +kevlar +kezen +kickback +kieviet +kijken +kikvors +kilheid +kilobit +kilsdonk +kipschnitzel +kissebis +klad +klagelijk +klak +klapbaar +klaver +klene +klets +klijnhout +klit +klok +klonen +klotefilm +kluif +klumper +klus +knabbel +knagen +knaven +kneedbaar +knmi +knul +knus +kokhals +komiek +komkommer +kompaan +komrij +komvormig +koning +kopbal +kopklep +kopnagel +koppejan +koptekst +kopwand +koraal +kosmisch +kostbaar +kram +kraneveld +kras +kreling +krengen +kribbe +krik +kruid +krulbol +kuijper +kuipbank +kuit +kuiven +kutsmoes +kuub +kwak +kwatong +kwetsbaar +kwezelaar +kwijnen +kwik +kwinkslag +kwitantie +lading +lakbeits +lakken +laklaag +lakmoes +lakwijk +lamheid +lamp +lamsbout +lapmiddel +larve +laser +latijn +latuw +lawaai +laxeerpil +lebberen +ledeboer +leefbaar +leeman +lefdoekje +lefhebber +legboor +legsel +leguaan +leiplaat +lekdicht +lekrijden +leksteen +lenen +leraar +lesbienne +leugenaar +leut +lexicaal +lezing +lieten +liggeld +lijdzaam +lijk +lijmstang +lijnschip +likdoorn +likken +liksteen +limburg +link +linoleum +lipbloem +lipman +lispelen +lissabon +litanie +liturgie +lochem +loempia +loesje +logheid +lonen +lonneke +loom +loos +losbaar +loslaten +losplaats +loting +lotnummer +lots +louie +lourdes +louter +lowbudget +luijten +luikenaar +luilak +luipaard +luizenbos +lulkoek +lumen +lunzen +lurven +lutjeboer +luttel +lutz +luuk +luwte +luyendijk +lyceum +lynx +maakbaar +magdalena +malheid +manchet +manfred +manhaftig +mank +mantel +marion +marxist +masmeijer +massaal +matsen +matverf +matze +maude +mayonaise +mechanica +meifeest +melodie +meppelink +midvoor +midweeks +midzomer +miezel +mijnraad +minus +mirck +mirte +mispakken +misraden +miswassen +mitella +moker +molecule +mombakkes +moonen +mopperaar +moraal +morgana +mormel +mosselaar +motregen +mouw +mufheid +mutueel +muzelman +naaidoos +naald +nadeel +nadruk +nagy +nahon +naima +nairobi +napalm +napels +napijn +napoleon +narigheid +narratief +naseizoen +nasibal +navigatie +nawijn +negatief +nekletsel +nekwervel +neolatijn +neonataal +neptunus +nerd +nest +neuzelaar +nihiliste +nijenhuis +nijging +nijhoff +nijl +nijptang +nippel +nokkenas +noordam +noren +normaal +nottelman +notulant +nout +nuance +nuchter +nudorp +nulde +nullijn +nulmeting +nunspeet +nylon +obelisk +object +oblie +obsceen +occlusie +oceaan +ochtend +ockhuizen +oerdom +oergezond +oerlaag +oester +okhuijsen +olifant +olijfboer +omaans +ombudsman +omdat +omdijken +omdoen +omgebouwd +omkeer +omkomen +ommegaand +ommuren +omroep +omruil +omslaan +omsmeden +omvaar +onaardig +onedel +onenig +onheilig +onrecht +onroerend +ontcijfer +onthaal +ontvallen +ontzadeld +onzacht +onzin +onzuiver +oogappel +ooibos +ooievaar +ooit +oorarts +oorhanger +oorijzer +oorklep +oorschelp +oorworm +oorzaak +opdagen +opdien +opdweilen +opel +opgebaard +opinie +opjutten +opkijken +opklaar +opkuisen +opkwam +opnaaien +opossum +opsieren +opsmeer +optreden +opvijzel +opvlammen +opwind +oraal +orchidee +orkest +ossuarium +ostendorf +oublie +oudachtig +oudbakken +oudnoors +oudshoorn +oudtante +oven +over +oxidant +pablo +pacht +paktafel +pakzadel +paljas +panharing +papfles +paprika +parochie +paus +pauze +paviljoen +peek +pegel +peigeren +pekela +pendant +penibel +pepmiddel +peptalk +periferie +perron +pessarium +peter +petfles +petgat +peuk +pfeifer +picknick +pief +pieneman +pijlkruid +pijnacker +pijpelink +pikdonker +pikeer +pilaar +pionier +pipet +piscine +pissebed +pitchen +pixel +plamuren +plan +plausibel +plegen +plempen +pleonasme +plezant +podoloog +pofmouw +pokdalig +ponywagen +popachtig +popidool +porren +positie +potten +pralen +prezen +prijzen +privaat +proef +prooi +prozawerk +pruik +prul +publiceer +puck +puilen +pukkelig +pulveren +pupil +puppy +purmerend +pustjens +putemmer +puzzelaar +queenie +quiche +raam +raar +raat +raes +ralf +rally +ramona +ramselaar +ranonkel +rapen +rapunzel +rarekiek +rarigheid +rattenhol +ravage +reactie +recreant +redacteur +redster +reewild +regie +reijnders +rein +replica +revanche +rigide +rijbaan +rijdansen +rijgen +rijkdom +rijles +rijnwijn +rijpma +rijstafel +rijtaak +rijzwepen +rioleer +ripdeal +riphagen +riskant +rits +rivaal +robbedoes +robot +rockact +rodijk +rogier +rohypnol +rollaag +rolpaal +roltafel +roof +roon +roppen +rosbief +rosharig +rosielle +rotan +rotleven +rotten +rotvaart +royaal +royeer +rubato +ruby +ruche +rudge +ruggetje +rugnummer +rugpijn +rugtitel +rugzak +ruilbaar +ruis +ruit +rukwind +rulijs +rumoeren +rumsdorp +rumtaart +runnen +russchen +ruwkruid +saboteer +saksisch +salade +salpeter +sambabal +samsam +satelliet +satineer +saus +scampi +scarabee +scenario +schobben +schubben +scout +secessie +secondair +seculair +sediment +seeland +settelen +setwinst +sheriff +shiatsu +siciliaan +sidderaal +sigma +sijben +silvana +simkaart +sinds +situatie +sjaak +sjardijn +sjezen +sjor +skinhead +skylab +slamixen +sleijpen +slijkerig +slordig +slowaak +sluieren +smadelijk +smiecht +smoel +smos +smukken +snackcar +snavel +sneaker +sneu +snijdbaar +snit +snorder +soapbox +soetekouw +soigneren +sojaboon +solo +solvabel +somber +sommatie +soort +soppen +sopraan +soundbar +spanen +spawater +spijgat +spinaal +spionage +spiraal +spleet +splijt +spoed +sporen +spul +spuug +spuw +stalen +standaard +star +stefan +stencil +stijf +stil +stip +stopdas +stoten +stoven +straat +strobbe +strubbel +stucadoor +stuif +stukadoor +subhoofd +subregent +sudoku +sukade +sulfaat +surinaams +suus +syfilis +symboliek +sympathie +synagoge +synchroon +synergie +systeem +taanderij +tabak +tachtig +tackelen +taiwanees +talman +tamheid +tangaslip +taps +tarkan +tarwe +tasman +tatjana +taxameter +teil +teisman +telbaar +telco +telganger +telstar +tenant +tepel +terzet +testament +ticket +tiesinga +tijdelijk +tika +tiksel +tilleman +timbaal +tinsteen +tiplijn +tippelaar +tjirpen +toezeggen +tolbaas +tolgeld +tolhek +tolo +tolpoort +toltarief +tolvrij +tomaat +tondeuse +toog +tooi +toonbaar +toos +topclub +toppen +toptalent +topvrouw +toque +torment +tornado +tosti +totdat +toucheer +toulouse +tournedos +tout +trabant +tragedie +trailer +traject +traktaat +trauma +tray +trechter +tred +tref +treur +troebel +tros +trucage +truffel +tsaar +tucht +tuenter +tuitelig +tukje +tuktuk +tulp +tuma +tureluurs +twijfel +twitteren +tyfoon +typograaf +ugandees +uiachtig +uier +uisnipper +ultiem +unitair +uranium +urbaan +urendag +ursula +uurcirkel +uurglas +uzelf +vaat +vakantie +vakleraar +valbijl +valpartij +valreep +valuatie +vanmiddag +vanonder +varaan +varken +vaten +veenbes +veeteler +velgrem +vellekoop +velvet +veneberg +venlo +vent +venusberg +venw +veredeld +verf +verhaaf +vermaak +vernaaid +verraad +vers +veruit +verzaagd +vetachtig +vetlok +vetmesten +veto +vetrek +vetstaart +vetten +veurink +viaduct +vibrafoon +vicariaat +vieux +vieveen +vijfvoud +villa +vilt +vimmetje +vindbaar +vips +virtueel +visdieven +visee +visie +vlaag +vleugel +vmbo +vocht +voesenek +voicemail +voip +volg +vork +vorselaar +voyeur +vracht +vrekkig +vreten +vrije +vrozen +vrucht +vucht +vugt +vulkaan +vulmiddel +vulva +vuren +waas +wacht +wadvogel +wafel +waffel +walhalla +walnoot +walraven +wals +walvis +wandaad +wanen +wanmolen +want +warklomp +warm +wasachtig +wasteil +watt +webhandel +weblog +webpagina +webzine +wedereis +wedstrijd +weeda +weert +wegmaaien +wegscheer +wekelijks +wekken +wekroep +wektoon +weldaad +welwater +wendbaar +wenkbrauw +wens +wentelaar +wervel +wesseling +wetboek +wetmatig +whirlpool +wijbrands +wijdbeens +wijk +wijnbes +wijting +wild +wimpelen +wingebied +winplaats +winter +winzucht +wipstaart +wisgerhof +withaar +witmaker +wokkel +wolf +wonenden +woning +worden +worp +wortel +wrat +wrijf +wringen +yoghurt +ypsilon +zaaijer +zaak +zacharias +zakelijk +zakkam +zakwater +zalf +zalig +zaniken +zebracode +zeeblauw +zeef +zeegaand +zeeuw +zege +zegje +zeil +zesbaans +zesenhalf +zeskantig +zesmaal +zetbaas +zetpil +zeulen +ziezo +zigzag +zijaltaar +zijbeuk +zijlijn +zijmuur +zijn +zijwaarts +zijzelf +zilt +zimmerman +zinledig +zinnelijk +zionist +zitdag +zitruimte +zitzak +zoal +zodoende +zoekbots +zoem +zoiets +zojuist +zondaar +zotskap +zottebol +zucht +zuivel +zulk +zult +zuster +zuur +zweedijk +zwendel +zwepen +zwiep +zwijmel +zworen diff --git a/src/mnemonics/languages/english.txt b/src/mnemonics/languages/english.txt new file mode 100644 index 000000000..755b15442 --- /dev/null +++ b/src/mnemonics/languages/english.txt @@ -0,0 +1,1629 @@ +English +English +3 +abbey +abducts +ability +ablaze +abnormal +abort +abrasive +absorb +abyss +academy +aces +aching +acidic +acoustic +acquire +across +actress +acumen +adapt +addicted +adept +adhesive +adjust +adopt +adrenalin +adult +adventure +aerial +afar +affair +afield +afloat +afoot +afraid +after +against +agenda +aggravate +agile +aglow +agnostic +agony +agreed +ahead +aided +ailments +aimless +airport +aisle +ajar +akin +alarms +album +alchemy +alerts +algebra +alkaline +alley +almost +aloof +alpine +already +also +altitude +alumni +always +amaze +ambush +amended +amidst +ammo +amnesty +among +amply +amused +anchor +android +anecdote +angled +ankle +annoyed +answers +antics +anvil +anxiety +anybody +apart +apex +aphid +aplomb +apology +apply +apricot +aptitude +aquarium +arbitrary +archer +ardent +arena +argue +arises +army +around +arrow +arsenic +artistic +ascend +ashtray +aside +asked +asleep +aspire +assorted +asylum +athlete +atlas +atom +atrium +attire +auburn +auctions +audio +august +aunt +austere +autumn +avatar +avidly +avoid +awakened +awesome +awful +awkward +awning +awoken +axes +axis +axle +aztec +azure +baby +bacon +badge +baffles +bagpipe +bailed +bakery +balding +bamboo +banjo +baptism +basin +batch +bawled +bays +because +beer +befit +begun +behind +being +below +bemused +benches +berries +bested +betting +bevel +beware +beyond +bias +bicycle +bids +bifocals +biggest +bikini +bimonthly +binocular +biology +biplane +birth +biscuit +bite +biweekly +blender +blip +bluntly +boat +bobsled +bodies +bogeys +boil +boldly +bomb +border +boss +both +bounced +bovine +bowling +boxes +boyfriend +broken +brunt +bubble +buckets +budget +buffet +bugs +building +bulb +bumper +bunch +business +butter +buying +buzzer +bygones +byline +bypass +cabin +cactus +cadets +cafe +cage +cajun +cake +calamity +camp +candy +casket +catch +cause +cavernous +cease +cedar +ceiling +cell +cement +cent +certain +chlorine +chrome +cider +cigar +cinema +circle +cistern +citadel +civilian +claim +click +clue +coal +cobra +cocoa +code +coexist +coffee +cogs +cohesive +coils +colony +comb +cool +copy +corrode +costume +cottage +cousin +cowl +criminal +cube +cucumber +cuddled +cuffs +cuisine +cunning +cupcake +custom +cycling +cylinder +cynical +dabbing +dads +daft +dagger +daily +damp +dangerous +dapper +darted +dash +dating +dauntless +dawn +daytime +dazed +debut +decay +dedicated +deepest +deftly +degrees +dehydrate +deity +dejected +delayed +demonstrate +dented +deodorant +depth +desk +devoid +dewdrop +dexterity +dialect +dice +diet +different +digit +dilute +dime +dinner +diode +diplomat +directed +distance +ditch +divers +dizzy +doctor +dodge +does +dogs +doing +dolphin +domestic +donuts +doorway +dormant +dosage +dotted +double +dove +down +dozen +dreams +drinks +drowning +drunk +drying +dual +dubbed +duckling +dude +duets +duke +dullness +dummy +dunes +duplex +duration +dusted +duties +dwarf +dwelt +dwindling +dying +dynamite +dyslexic +each +eagle +earth +easy +eating +eavesdrop +eccentric +echo +eclipse +economics +ecstatic +eden +edgy +edited +educated +eels +efficient +eggs +egotistic +eight +either +eject +elapse +elbow +eldest +eleven +elite +elope +else +eluded +emails +ember +emerge +emit +emotion +empty +emulate +energy +enforce +enhanced +enigma +enjoy +enlist +enmity +enough +enraged +ensign +entrance +envy +epoxy +equip +erase +erected +erosion +error +eskimos +espionage +essential +estate +etched +eternal +ethics +etiquette +evaluate +evenings +evicted +evolved +examine +excess +exhale +exit +exotic +exquisite +extra +exult +fabrics +factual +fading +fainted +faked +fall +family +fancy +farming +fatal +faulty +fawns +faxed +fazed +feast +february +federal +feel +feline +females +fences +ferry +festival +fetches +fever +fewest +fiat +fibula +fictional +fidget +fierce +fifteen +fight +films +firm +fishing +fitting +five +fixate +fizzle +fleet +flippant +flying +foamy +focus +foes +foggy +foiled +folding +fonts +foolish +fossil +fountain +fowls +foxes +foyer +framed +friendly +frown +fruit +frying +fudge +fuel +fugitive +fully +fuming +fungal +furnished +fuselage +future +fuzzy +gables +gadget +gags +gained +galaxy +gambit +gang +gasp +gather +gauze +gave +gawk +gaze +gearbox +gecko +geek +gels +gemstone +general +geometry +germs +gesture +getting +geyser +ghetto +ghost +giant +giddy +gifts +gigantic +gills +gimmick +ginger +girth +giving +glass +gleeful +glide +gnaw +gnome +goat +goblet +godfather +goes +goggles +going +goldfish +gone +goodbye +gopher +gorilla +gossip +gotten +gourmet +governing +gown +greater +grunt +guarded +guest +guide +gulp +gumball +guru +gusts +gutter +guys +gymnast +gypsy +gyrate +habitat +hacksaw +haggled +hairy +hamburger +happens +hashing +hatchet +haunted +having +hawk +haystack +hazard +hectare +hedgehog +heels +hefty +height +hemlock +hence +heron +hesitate +hexagon +hickory +hiding +highway +hijack +hiker +hills +himself +hinder +hippo +hire +history +hitched +hive +hoax +hobby +hockey +hoisting +hold +honked +hookup +hope +hornet +hospital +hotel +hounded +hover +howls +hubcaps +huddle +huge +hull +humid +hunter +hurried +husband +huts +hybrid +hydrogen +hyper +iceberg +icing +icon +identity +idiom +idled +idols +igloo +ignore +iguana +illness +imagine +imbalance +imitate +impel +inactive +inbound +incur +industrial +inexact +inflamed +ingested +initiate +injury +inkling +inline +inmate +innocent +inorganic +input +inquest +inroads +insult +intended +inundate +invoke +inwardly +ionic +irate +iris +irony +irritate +island +isolated +issued +italics +itches +items +itinerary +itself +ivory +jabbed +jackets +jaded +jagged +jailed +jamming +january +jargon +jaunt +javelin +jaws +jazz +jeans +jeers +jellyfish +jeopardy +jerseys +jester +jetting +jewels +jigsaw +jingle +jittery +jive +jobs +jockey +jogger +joining +joking +jolted +jostle +journal +joyous +jubilee +judge +juggled +juicy +jukebox +july +jump +junk +jury +justice +juvenile +kangaroo +karate +keep +kennel +kept +kernels +kettle +keyboard +kickoff +kidneys +king +kiosk +kisses +kitchens +kiwi +knapsack +knee +knife +knowledge +knuckle +koala +laboratory +ladder +lagoon +lair +lakes +lamb +language +laptop +large +last +later +launching +lava +lawsuit +layout +lazy +lectures +ledge +leech +left +legion +leisure +lemon +lending +leopard +lesson +lettuce +lexicon +liar +library +licks +lids +lied +lifestyle +light +likewise +lilac +limits +linen +lion +lipstick +liquid +listen +lively +loaded +lobster +locker +lodge +lofty +logic +loincloth +long +looking +lopped +lordship +losing +lottery +loudly +love +lower +loyal +lucky +luggage +lukewarm +lullaby +lumber +lunar +lurk +lush +luxury +lymph +lynx +lyrics +macro +madness +magically +mailed +major +makeup +malady +mammal +maps +masterful +match +maul +maverick +maximum +mayor +maze +meant +mechanic +medicate +meeting +megabyte +melting +memoir +menu +merger +mesh +metro +mews +mice +midst +mighty +mime +mirror +misery +mittens +mixture +moat +mobile +mocked +mohawk +moisture +molten +moment +money +moon +mops +morsel +mostly +motherly +mouth +movement +mowing +much +muddy +muffin +mugged +mullet +mumble +mundane +muppet +mural +musical +muzzle +myriad +mystery +myth +nabbing +nagged +nail +names +nanny +napkin +narrate +nasty +natural +nautical +navy +nearby +necklace +needed +negative +neither +neon +nephew +nerves +nestle +network +neutral +never +newt +nexus +nibs +niche +niece +nifty +nightly +nimbly +nineteen +nirvana +nitrogen +nobody +nocturnal +nodes +noises +nomad +noodles +northern +nostril +noted +nouns +novelty +nowhere +nozzle +nuance +nucleus +nudged +nugget +nuisance +null +number +nuns +nurse +nutshell +nylon +oaks +oars +oasis +oatmeal +obedient +object +obliged +obnoxious +observant +obtains +obvious +occur +ocean +october +odds +odometer +offend +often +oilfield +ointment +okay +older +olive +olympics +omega +omission +omnibus +onboard +oncoming +oneself +ongoing +onion +online +onslaught +onto +onward +oozed +opacity +opened +opposite +optical +opus +orange +orbit +orchid +orders +organs +origin +ornament +orphans +oscar +ostrich +otherwise +otter +ouch +ought +ounce +ourselves +oust +outbreak +oval +oven +owed +owls +owner +oxidant +oxygen +oyster +ozone +pact +paddles +pager +pairing +palace +pamphlet +pancakes +paper +paradise +pastry +patio +pause +pavements +pawnshop +payment +peaches +pebbles +peculiar +pedantic +peeled +pegs +pelican +pencil +people +pepper +perfect +pests +petals +phase +pheasants +phone +phrases +physics +piano +picked +pierce +pigment +piloted +pimple +pinched +pioneer +pipeline +pirate +pistons +pitched +pivot +pixels +pizza +playful +pledge +pliers +plotting +plus +plywood +poaching +pockets +podcast +poetry +point +poker +polar +ponies +pool +popular +portents +possible +potato +pouch +poverty +powder +pram +present +pride +problems +pruned +prying +psychic +public +puck +puddle +puffin +pulp +pumpkins +punch +puppy +purged +push +putty +puzzled +pylons +pyramid +python +queen +quick +quote +rabbits +racetrack +radar +rafts +rage +railway +raking +rally +ramped +randomly +rapid +rarest +rash +rated +ravine +rays +razor +react +rebel +recipe +reduce +reef +refer +regular +reheat +reinvest +rejoices +rekindle +relic +remedy +renting +reorder +repent +request +reruns +rest +return +reunion +revamp +rewind +rhino +rhythm +ribbon +richly +ridges +rift +rigid +rims +ringing +riots +ripped +rising +ritual +river +roared +robot +rockets +rodent +rogue +roles +romance +roomy +roped +roster +rotate +rounded +rover +rowboat +royal +ruby +rudely +ruffled +rugged +ruined +ruling +rumble +runway +rural +rustled +ruthless +sabotage +sack +sadness +safety +saga +sailor +sake +salads +sample +sanity +sapling +sarcasm +sash +satin +saucepan +saved +sawmill +saxophone +sayings +scamper +scenic +school +science +scoop +scrub +scuba +seasons +second +sedan +seeded +segments +seismic +selfish +semifinal +sensible +september +sequence +serving +session +setup +seventh +sewage +shackles +shelter +shipped +shocking +shrugged +shuffled +shyness +siblings +sickness +sidekick +sieve +sifting +sighting +silk +simplest +sincerely +sipped +siren +situated +sixteen +sizes +skater +skew +skirting +skulls +skydive +slackens +sleepless +slid +slower +slug +smash +smelting +smidgen +smog +smuggled +snake +sneeze +sniff +snout +snug +soapy +sober +soccer +soda +software +soggy +soil +solved +somewhere +sonic +soothe +soprano +sorry +southern +sovereign +sowed +soya +space +speedy +sphere +spiders +splendid +spout +sprig +spud +spying +square +stacking +stellar +stick +stockpile +strained +stunning +stylishly +subtly +succeed +suddenly +suede +suffice +sugar +suitcase +sulking +summon +sunken +superior +surfer +sushi +suture +swagger +swept +swiftly +sword +swung +syllabus +symptoms +syndrome +syringe +system +taboo +tacit +tadpoles +tagged +tail +taken +talent +tamper +tanks +tapestry +tarnished +tasked +tattoo +taunts +tavern +tawny +taxi +teardrop +technical +tedious +teeming +tell +template +tender +tepid +tequila +terminal +testing +tether +textbook +thaw +theatrics +thirsty +thorn +threaten +thumbs +thwart +ticket +tidy +tiers +tiger +tilt +timber +tinted +tipsy +tirade +tissue +titans +toaster +tobacco +today +toenail +toffee +together +toilet +token +tolerant +tomorrow +tonic +toolbox +topic +torch +tossed +total +touchy +towel +toxic +toyed +trash +trendy +tribal +trolling +truth +trying +tsunami +tubes +tucks +tudor +tuesday +tufts +tugs +tuition +tulips +tumbling +tunnel +turnip +tusks +tutor +tuxedo +twang +tweezers +twice +twofold +tycoon +typist +tyrant +ugly +ulcers +ultimate +umbrella +umpire +unafraid +unbending +uncle +under +uneven +unfit +ungainly +unhappy +union +unjustly +unknown +unlikely +unmask +unnoticed +unopened +unplugs +unquoted +unrest +unsafe +until +unusual +unveil +unwind +unzip +upbeat +upcoming +update +upgrade +uphill +upkeep +upload +upon +upper +upright +upstairs +uptight +upwards +urban +urchins +urgent +usage +useful +usher +using +usual +utensils +utility +utmost +utopia +uttered +vacation +vague +vain +value +vampire +vane +vapidly +vary +vastness +vats +vaults +vector +veered +vegan +vehicle +vein +velvet +venomous +verification +vessel +veteran +vexed +vials +vibrate +victim +video +viewpoint +vigilant +viking +village +vinegar +violin +vipers +virtual +visited +vitals +vivid +vixen +vocal +vogue +voice +volcano +vortex +voted +voucher +vowels +voyage +vulture +wade +waffle +wagtail +waist +waking +wallets +wanted +warped +washing +water +waveform +waxing +wayside +weavers +website +wedge +weekday +weird +welders +went +wept +were +western +wetsuit +whale +when +whipped +whole +wickets +width +wield +wife +wiggle +wildly +winter +wipeout +wiring +wise +withdrawn +wives +wizard +wobbly +woes +woken +wolf +womanly +wonders +woozy +worry +wounded +woven +wrap +wrist +wrong +yacht +yahoo +yanks +yard +yawning +yearbook +yellow +yesterday +yeti +yields +yodel +yoga +younger +yoyo +zapped +zeal +zebra +zero +zesty +zigzags +zinger +zippers +zodiac +zombie +zones +zoom diff --git a/src/mnemonics/languages/esperanto.txt b/src/mnemonics/languages/esperanto.txt new file mode 100644 index 000000000..3238b89c2 --- /dev/null +++ b/src/mnemonics/languages/esperanto.txt @@ -0,0 +1,1629 @@ +Esperanto +Esperanto +3 +abako +abdiki +abelo +abituriento +ablativo +abnorma +abonantoj +abrikoto +absoluta +abunda +acetono +acida +adapti +adekvata +adheri +adicii +adjektivo +administri +adolesko +adreso +adstringa +adulto +advokato +adzo +aeroplano +aferulo +afgana +afiksi +aflaba +aforismo +afranki +aftozo +afusto +agavo +agento +agiti +aglo +agmaniero +agnoski +agordo +agrabla +agtipo +agutio +aikido +ailanto +aina +ajatolo +ajgenvaloro +ajlobulbo +ajnlitera +ajuto +ajzi +akademio +akcepti +akeo +akiri +aklamado +akmeo +akno +akompani +akrobato +akselo +aktiva +akurata +akvofalo +alarmo +albumo +alcedo +aldoni +aleo +alfabeto +algo +alhasti +aligatoro +alkoholo +almozo +alnomo +alojo +alpinisto +alrigardi +alskribi +alta +alumeto +alveni +alzaca +amaso +ambasado +amdeklaro +amebo +amfibio +amhara +amiko +amkanto +amletero +amnestio +amoranto +amplekso +amrakonto +amsterdama +amuzi +ananaso +androido +anekdoto +anfrakto +angulo +anheli +animo +anjono +ankro +anonci +anpriskribo +ansero +antikva +anuitato +aorto +aparta +aperti +apika +aplikado +apneo +apogi +aprobi +apsido +apterigo +apudesto +araneo +arbo +ardeco +aresti +argilo +aristokrato +arko +arlekeno +armi +arniko +aromo +arpio +arsenalo +artisto +aruba +arvorto +asaio +asbesto +ascendi +asekuri +asfalto +asisti +askalono +asocio +aspekti +astro +asulo +atakonto +atendi +atingi +atleto +atmosfero +atomo +atropino +atuto +avataro +aventuro +aviadilo +avokado +azaleo +azbuko +azenino +azilpetanto +azoto +azteka +babili +bacilo +badmintono +bagatelo +bahama +bajoneto +baki +balai +bambuo +bani +baobabo +bapti +baro +bastono +batilo +bavara +bazalto +beata +bebofono +bedo +begonio +behaviorismo +bejlo +bekero +belarto +bemolo +benko +bereto +besto +betulo +bevelo +bezoni +biaso +biblioteko +biciklo +bidaro +bieno +bifsteko +bigamiulo +bijekcio +bikino +bildo +bimetalismo +bindi +biografio +birdo +biskvito +bitlibro +bivako +bizara +bjalistoka +blanka +bleki +blinda +blovi +blua +boato +bobsledo +bocvanano +bodisatvo +bofratino +bogefratoj +bohema +boji +bokalo +boli +bombono +bona +bopatrino +bordo +bosko +botelo +bovido +brakpleno +bretaro +brikmuro +broso +brulema +bubalo +buctrapi +budo +bufedo +bugio +bujabeso +buklo +buldozo +bumerango +bunta +burokrataro +busbileto +butero +buzuko +caro +cebo +ceceo +cedro +cefalo +cejana +cekumo +celebri +cemento +cent +cepo +certa +cetera +cezio +ciano +cibeto +cico +cidro +cifero +cigaredo +ciklo +cilindro +cimbalo +cinamo +cipreso +cirkonstanco +cisterno +citrono +ciumi +civilizado +colo +congo +cunamo +cvana +dabi +daco +dadaismo +dafodilo +dago +daimio +dajmono +daktilo +dalio +damo +danki +darmo +datumoj +dazipo +deadmoni +debeto +decidi +dedukti +deerigi +defendi +degeli +dehaki +deirpunkto +deklaracio +delikata +demandi +dento +dependi +derivi +desegni +detrui +devi +deziri +dialogo +dicentro +didaktika +dieto +diferenci +digesti +diino +dikfingro +diligenta +dimensio +dinamo +diodo +diplomo +direkte +diskuti +diurno +diversa +dizajno +dobrogitaro +docento +dogano +dojeno +doktoro +dolori +domego +donaci +dopado +dormi +dosierujo +dotita +dozeno +drato +dresi +drinki +droni +druido +duaranga +dubi +ducent +dudek +duelo +dufoje +dugongo +duhufa +duilo +dujare +dukato +duloka +dumtempe +dungi +duobla +dupiedulo +dura +dusenca +dutaga +duuma +duvalvuloj +duzo +ebena +eblecoj +ebono +ebria +eburo +ecaro +ecigi +ecoj +edelvejso +editoro +edro +eduki +edzino +efektiva +efiki +efloreski +egala +egeco +egiptologo +eglefino +egoista +egreto +ejakuli +ejlo +ekarto +ekbruligi +ekceli +ekde +ekesti +ekfirmao +ekgliti +ekhavi +ekipi +ekkapti +eklezio +ekmalsati +ekonomio +ekpluvi +ekrano +ekster +ektiri +ekumeno +ekvilibro +ekzemplo +elasta +elbalai +elcento +eldoni +elektro +elfari +elgliti +elhaki +elipso +elkovi +ellasi +elmeti +elnutri +elokventa +elparoli +elrevigi +elstari +elteni +eluzita +elvoki +elzasa +emajlo +embaraso +emerito +emfazo +eminenta +emocio +empiria +emulsio +enarkivigi +enboteligi +enciklopedio +endorfino +energio +enfermi +engluti +enhavo +enigmo +enjekcio +enketi +enlanda +enmeti +enorma +enplanti +enradiki +enspezo +entrepreni +enui +envolvi +enzimo +eono +eosto +epitafo +epoko +epriskribebla +epsilono +erari +erbio +erco +erekti +ergonomia +erikejo +ermito +erotika +erpilo +erupcio +esameno +escepti +esenco +eskapi +esotera +esperi +estonto +etapo +etendi +etfingro +etikedo +etlitero +etmakleristo +etnika +etoso +etradio +etskala +etullernejo +evakui +evento +eviti +evolui +ezoko +fabriko +facila +fadeno +fagoto +fajro +fakto +fali +familio +fanatiko +farbo +fasko +fatala +favora +fazeolo +febro +federacio +feino +fekunda +felo +femuro +fenestro +fermi +festi +fetora +fezo +fiasko +fibro +fidela +fiera +fifama +figuro +fiherbo +fiinsekto +fiksa +filmo +fimensa +finalo +fiolo +fiparoli +firmao +fisko +fitingo +fiuzanto +fivorto +fiziko +fjordo +flago +flegi +flirti +floro +flugi +fobio +foceno +foirejo +fojfoje +fokuso +folio +fomenti +fonto +formulo +fosforo +fotografi +fratino +fremda +friti +frosto +frua +ftizo +fuelo +fugo +fuksia +fulmilo +fumanto +fundamento +fuorto +furioza +fusilo +futbalo +fuzio +gabardino +gado +gaela +gafo +gagato +gaja +gaki +galanta +gamao +ganto +gapulo +gardi +gasto +gavio +gazeto +geamantoj +gebani +geedzeco +gefratoj +geheno +gejsero +geko +gelateno +gemisto +geniulo +geografio +gepardo +geranio +gestolingvo +geto +geumo +gibono +giganta +gildo +gimnastiko +ginekologo +gipsi +girlando +gistfungo +gitaro +glazuro +glebo +gliti +globo +gluti +gnafalio +gnejso +gnomo +gnuo +gobio +godetio +goeleto +gojo +golfludejo +gombo +gondolo +gorilo +gospelo +gotika +granda +greno +griza +groto +grupo +guano +gubernatoro +gudrotuko +gufo +gujavo +guldeno +gumi +gupio +guruo +gusto +guto +guvernistino +gvardio +gverilo +gvidanto +habitato +hadito +hafnio +hagiografio +haitiano +hajlo +hakbloko +halti +hamstro +hangaro +hapalo +haro +hasta +hati +havebla +hazardo +hebrea +hedero +hegemonio +hejmo +hektaro +helpi +hemisfero +heni +hepato +herbo +hesa +heterogena +heziti +hiacinto +hibrida +hidrogeno +hieroglifo +higieno +hihii +hilumo +himno +hindino +hiperteksto +hirundo +historio +hobio +hojli +hokeo +hologramo +homido +honesta +hopi +horizonto +hospitalo +hotelo +huadi +hubo +hufumo +hugenoto +hukero +huligano +humana +hundo +huoj +hupilo +hurai +husaro +hutuo +huzo +iafoje +iagrade +iamaniere +iarelate +iaspeca +ibekso +ibiso +idaro +ideala +idiomo +idolo +iele +igluo +ignori +iguamo +igvano +ikono +iksodo +ikto +iliaflanke +ilkomputilo +ilobreto +ilremedo +ilumini +imagi +imitado +imperio +imuna +incidento +industrio +inerta +infano +ingenra +inhali +iniciati +injekti +inklino +inokuli +insekto +inteligenta +inundi +inviti +ioma +ionosfero +iperito +ipomeo +irana +irejo +irigacio +ironio +isato +islamo +istempo +itinero +itrio +iuloke +iumaniere +iutempe +izolita +jado +jaguaro +jakto +jama +januaro +japano +jarringo +jazo +jenoj +jesulo +jetavio +jezuito +jodli +joviala +juano +jubileo +judismo +jufto +juki +julio +juneca +jupo +juristo +juste +juvelo +kabineto +kadrato +kafo +kahelo +kajako +kakao +kalkuli +kampo +kanti +kapitalo +karaktero +kaserolo +katapulto +kaverna +kazino +kebabo +kefiro +keglo +kejlo +kekso +kelka +kemio +kerno +kesto +kiamaniere +kibuco +kidnapi +kielo +kikero +kilogramo +kimono +kinejo +kiosko +kirurgo +kisi +kitelo +kivio +klavaro +klerulo +klini +klopodi +klubo +knabo +knedi +koalo +kobalto +kodigi +kofro +kohera +koincidi +kojoto +kokoso +koloro +komenci +kontrakto +kopio +korekte +kosti +kotono +kovri +krajono +kredi +krii +krom +kruco +ksantino +ksenono +ksilofono +ksosa +kubuto +kudri +kuglo +kuiri +kuko +kulero +kumuluso +kuneco +kupro +kuri +kuseno +kutimo +kuvo +kuzino +kvalito +kverko +kvin +kvoto +labori +laculo +ladbotelo +lafo +laguno +laikino +laktobovino +lampolumo +landkarto +laosa +lapono +larmoguto +lastjare +latitudo +lavejo +lazanjo +leciono +ledosako +leganto +lekcio +lemura +lentuga +leopardo +leporo +lerni +lesivo +letero +levilo +lezi +liano +libera +liceo +lieno +lifto +ligilo +likvoro +lila +limono +lingvo +lipo +lirika +listo +literatura +liveri +lobio +logika +lojala +lokalo +longa +lordo +lotado +loza +luanto +lubriki +lucida +ludema +luigi +lukso +luli +lumbilda +lunde +lupago +lustro +lutilo +luzerno +maato +maceri +madono +mafiano +magazeno +mahometano +maizo +majstro +maketo +malgranda +mamo +mandareno +maorio +mapigi +marini +masko +mateno +mazuto +meandro +meblo +mecenato +medialo +mefito +megafono +mejlo +mekanika +melodia +membro +mendi +mergi +mespilo +metoda +mevo +mezuri +miaflanke +micelio +mielo +migdalo +mikrofilmo +militi +mimiko +mineralo +miopa +miri +mistera +mitralo +mizeri +mjelo +mnemoniko +mobilizi +mocio +moderna +mohajro +mokadi +molaro +momento +monero +mopso +mordi +moskito +motoro +movimento +mozaiko +mueli +mukozo +muldi +mumio +munti +muro +muskolo +mutacio +muzikisto +nabo +nacio +nadlo +nafto +naiva +najbaro +nanometro +napo +narciso +naski +naturo +navigi +naztruo +neatendite +nebulo +necesa +nedankinde +neebla +nefari +negoco +nehavi +neimagebla +nektaro +nelonga +nematura +nenia +neordinara +nepra +nervuro +nesto +nete +neulo +nevino +nifo +nigra +nihilisto +nikotino +nilono +nimfeo +nitrogeno +nivelo +nobla +nocio +nodozo +nokto +nomkarto +norda +nostalgio +notbloko +novico +nuanco +nuboza +nuda +nugato +nuklea +nuligi +numero +nuntempe +nupto +nura +nutri +oazo +obei +objekto +oblikva +obolo +observi +obtuza +obuso +oceano +odekolono +odori +oferti +oficiala +ofsajdo +ofte +ogivo +ogro +ojstredoj +okaze +okcidenta +okro +oksido +oktobro +okulo +oldulo +oleo +olivo +omaro +ombro +omego +omikrono +omleto +omnibuso +onagro +ondo +oneco +onidire +onklino +onlajna +onomatopeo +ontologio +opaka +operacii +opinii +oportuna +opresi +optimisto +oratoro +orbito +ordinara +orelo +orfino +organizi +orienta +orkestro +orlo +orminejo +ornami +ortangulo +orumi +oscedi +osmozo +ostocerbo +ovalo +ovingo +ovoblanko +ovri +ovulado +ozono +pacama +padeli +pafilo +pagigi +pajlo +paketo +palaco +pampelmo +pantalono +papero +paroli +pasejo +patro +pavimo +peco +pedalo +peklita +pelikano +pensiono +peplomo +pesilo +petanto +pezoforto +piano +picejo +piede +pigmento +pikema +pilkoludo +pimento +pinglo +pioniro +pipromento +pirato +pistolo +pitoreska +piulo +pivoti +pizango +planko +plektita +plibonigi +ploradi +plurlingva +pobo +podio +poeto +pogranda +pohora +pokalo +politekniko +pomarbo +ponevosto +populara +porcelana +postkompreno +poteto +poviga +pozitiva +prapatroj +precize +pridemandi +probable +pruntanto +psalmo +psikologio +psoriazo +pterido +publiko +pudro +pufo +pugnobato +pulovero +pumpi +punkto +pupo +pureo +puso +putrema +puzlo +rabate +racionala +radiko +rafinado +raguo +rajto +rakonti +ralio +rampi +rando +rapida +rastruma +ratifiki +raviolo +razeno +reakcio +rebildo +recepto +redakti +reenigi +reformi +regiono +rehavi +reinspekti +rejesi +reklamo +relativa +rememori +renkonti +reorganizado +reprezenti +respondi +retumilo +reuzebla +revidi +rezulti +rialo +ribeli +ricevi +ridiga +rifuginto +rigardi +rikolti +rilati +rimarki +rinocero +ripozi +riski +ritmo +rivero +rizokampo +roboto +rododendro +rojo +rokmuziko +rolvorto +romantika +ronroni +rosino +rotondo +rovero +rozeto +rubando +rudimenta +rufa +rugbeo +ruino +ruleto +rumoro +runo +rupio +rura +rustimuna +ruzulo +sabato +sadismo +safario +sagaca +sakfluto +salti +samtage +sandalo +sapejo +sarongo +satelito +savano +sbiro +sciado +seanco +sebo +sedativo +segligno +sekretario +selektiva +semajno +senpeza +separeo +servilo +sesangulo +setli +seurigi +severa +sezono +sfagno +sfero +sfinkso +siatempe +siblado +sidejo +siesto +sifono +signalo +siklo +silenti +simpla +sinjoro +siropo +sistemo +situacio +siverto +sizifa +skatolo +skemo +skianto +sklavo +skorpio +skribisto +skulpti +skvamo +slango +sledeto +sliparo +smeraldo +smirgi +smokingo +smuto +snoba +snufegi +sobra +sociano +sodakvo +sofo +soifi +sojlo +soklo +soldato +somero +sonilo +sopiri +sorto +soulo +soveto +sparkado +speciala +spiri +splito +sporto +sprita +spuro +stabila +stelfiguro +stimulo +stomako +strato +studanto +subgrupo +suden +suferanta +sugesti +suito +sukero +sulko +sume +sunlumo +super +surskribeto +suspekti +suturo +svati +svenfali +svingi +svopo +tabako +taglumo +tajloro +taksimetro +talento +tamen +tanko +taoismo +tapioko +tarifo +tasko +tatui +taverno +teatro +tedlaboro +tegmento +tehoro +teknika +telefono +tempo +tenisejo +teorie +teraso +testudo +tetablo +teujo +tezo +tialo +tibio +tielnomata +tifono +tigro +tikli +timida +tinkturo +tiom +tiparo +tirkesto +titolo +tiutempe +tizano +tobogano +tofeo +togo +toksa +tolerema +tombolo +tondri +topografio +tordeti +tosti +totalo +traduko +tredi +triangulo +tropika +trumpeto +tualeto +tubisto +tufgrebo +tuja +tukano +tulipo +tumulto +tunelo +turisto +tusi +tutmonda +tvisto +udono +uesto +ukazo +ukelelo +ulcero +ulmo +ultimato +ululi +umbiliko +unco +ungego +uniformo +unkti +unukolora +uragano +urbano +uretro +urino +ursido +uskleco +usonigi +utero +utila +utopia +uverturo +uzadi +uzeblo +uzino +uzkutimo +uzofini +uzurpi +uzvaloro +vadejo +vafleto +vagono +vahabismo +vajco +vakcino +valoro +vampiro +vangharoj +vaporo +varma +vasta +vato +vazaro +veaspekta +vedismo +vegetalo +vehiklo +vejno +vekita +velstango +vemieno +vendi +vepro +verando +vespero +veturi +veziko +viando +vibri +vico +videbla +vifio +vigla +viktimo +vila +vimeno +vintro +violo +vippuno +virtuala +viskoza +vitro +viveca +viziti +vobli +vodko +vojeto +vokegi +volbo +vomema +vono +vortaro +vosto +voti +vrako +vringi +vualo +vulkano +vundo +vuvuzelo +zamenhofa +zapi +zebro +zefiro +zeloto +zenismo +zeolito +zepelino +zeto +zigzagi +zinko +zipo +zirkonio +zodiako +zoeto +zombio +zono +zoologio +zorgi +zukino +zumilo diff --git a/src/mnemonics/languages/french.txt b/src/mnemonics/languages/french.txt new file mode 100644 index 000000000..13b8996b9 --- /dev/null +++ b/src/mnemonics/languages/french.txt @@ -0,0 +1,1629 @@ +French +Français +4 +abandon +abattre +aboi +abolir +aborder +abri +absence +absolu +abuser +acacia +acajou +accent +accord +accrocher +accuser +acerbe +achat +acheter +acide +acier +acquis +acte +action +adage +adepte +adieu +admettre +admis +adorer +adresser +aduler +affaire +affirmer +afin +agacer +agent +agir +agiter +agonie +agrafe +agrume +aider +aigle +aigre +aile +ailleurs +aimant +aimer +ainsi +aise +ajouter +alarme +album +alcool +alerte +algue +alibi +aller +allumer +alors +amande +amener +amie +amorcer +amour +ample +amuser +ananas +ancien +anglais +angoisse +animal +anneau +annoncer +apercevoir +apparence +appel +apporter +apprendre +appuyer +arbre +arcade +arceau +arche +ardeur +argent +argile +aride +arme +armure +arracher +arriver +article +asile +aspect +assaut +assez +assister +assurer +astre +astuce +atlas +atroce +attacher +attente +attirer +aube +aucun +audace +auparavant +auquel +aurore +aussi +autant +auteur +autoroute +autre +aval +avant +avec +avenir +averse +aveu +avide +avion +avis +avoir +avouer +avril +azote +azur +badge +bagage +bague +bain +baisser +balai +balcon +balise +balle +bambou +banane +banc +bandage +banjo +banlieue +bannir +banque +baobab +barbe +barque +barrer +bassine +bataille +bateau +battre +baver +bavoir +bazar +beau +beige +berger +besoin +beurre +biais +biceps +bidule +bien +bijou +bilan +billet +blanc +blason +bleu +bloc +blond +bocal +boire +boiserie +boiter +bonbon +bondir +bonheur +bordure +borgne +borner +bosse +bouche +bouder +bouger +boule +bourse +bout +boxe +brader +braise +branche +braquer +bras +brave +brebis +brevet +brider +briller +brin +brique +briser +broche +broder +bronze +brosser +brouter +bruit +brute +budget +buffet +bulle +bureau +buriner +buste +buter +butiner +cabas +cabinet +cabri +cacao +cacher +cadeau +cadre +cage +caisse +caler +calme +camarade +camion +campagne +canal +canif +capable +capot +carat +caresser +carie +carpe +cartel +casier +casque +casserole +cause +cavale +cave +ceci +cela +celui +cendre +cent +cependant +cercle +cerise +cerner +certes +cerveau +cesser +chacun +chair +chaleur +chamois +chanson +chaque +charge +chasse +chat +chaud +chef +chemin +cheveu +chez +chicane +chien +chiffre +chiner +chiot +chlore +choc +choix +chose +chou +chute +cibler +cidre +ciel +cigale +cinq +cintre +cirage +cirque +ciseau +citation +citer +citron +civet +clairon +clan +classe +clavier +clef +climat +cloche +cloner +clore +clos +clou +club +cobra +cocon +coiffer +coin +colline +colon +combat +comme +compte +conclure +conduire +confier +connu +conseil +contre +convenir +copier +cordial +cornet +corps +cosmos +coton +couche +coude +couler +coupure +cour +couteau +couvrir +crabe +crainte +crampe +cran +creuser +crever +crier +crime +crin +crise +crochet +croix +cruel +cuisine +cuite +culot +culte +cumul +cure +curieux +cuve +dame +danger +dans +davantage +debout +dedans +dehors +delta +demain +demeurer +demi +dense +dent +depuis +dernier +descendre +dessus +destin +dette +deuil +deux +devant +devenir +devin +devoir +dicton +dieu +difficile +digestion +digue +diluer +dimanche +dinde +diode +dire +diriger +discours +disposer +distance +divan +divers +docile +docteur +dodu +dogme +doigt +dominer +donation +donjon +donner +dopage +dorer +dormir +doseur +douane +double +douche +douleur +doute +doux +douzaine +draguer +drame +drap +dresser +droit +duel +dune +duper +durant +durcir +durer +eaux +effacer +effet +effort +effrayant +elle +embrasser +emmener +emparer +empire +employer +emporter +enclos +encore +endive +endormir +endroit +enduit +enfant +enfermer +enfin +enfler +enfoncer +enfuir +engager +engin +enjeu +enlever +ennemi +ennui +ensemble +ensuite +entamer +entendre +entier +entourer +entre +envelopper +envie +envoyer +erreur +escalier +espace +espoir +esprit +essai +essor +essuyer +estimer +exact +examiner +excuse +exemple +exiger +exil +exister +exode +expliquer +exposer +exprimer +extase +fable +facette +facile +fade +faible +faim +faire +fait +falloir +famille +faner +farce +farine +fatigue +faucon +faune +faute +faux +faveur +favori +faxer +feinter +femme +fendre +fente +ferme +festin +feuille +feutre +fiable +fibre +ficher +fier +figer +figure +filet +fille +filmer +fils +filtre +final +finesse +finir +fiole +firme +fixe +flacon +flair +flamme +flan +flaque +fleur +flocon +flore +flot +flou +fluide +fluor +flux +focus +foin +foire +foison +folie +fonction +fondre +fonte +force +forer +forger +forme +fort +fosse +fouet +fouine +foule +four +foyer +frais +franc +frapper +freiner +frimer +friser +frite +froid +froncer +fruit +fugue +fuir +fuite +fumer +fureur +furieux +fuser +fusil +futile +futur +gagner +gain +gala +galet +galop +gamme +gant +garage +garde +garer +gauche +gaufre +gaule +gaver +gazon +geler +genou +genre +gens +gercer +germer +geste +gibier +gicler +gilet +girafe +givre +glace +glisser +globe +gloire +gluant +gober +golf +gommer +gorge +gosier +goutte +grain +gramme +grand +gras +grave +gredin +griffure +griller +gris +gronder +gros +grotte +groupe +grue +guerrier +guetter +guider +guise +habiter +hache +haie +haine +halte +hamac +hanche +hangar +hanter +haras +hareng +harpe +hasard +hausse +haut +havre +herbe +heure +hibou +hier +histoire +hiver +hochet +homme +honneur +honte +horde +horizon +hormone +houle +housse +hublot +huile +huit +humain +humble +humide +humour +hurler +idole +igloo +ignorer +illusion +image +immense +immobile +imposer +impression +incapable +inconnu +index +indiquer +infime +injure +inox +inspirer +instant +intention +intime +inutile +inventer +inviter +iode +iris +issue +ivre +jade +jadis +jamais +jambe +janvier +jardin +jauge +jaunisse +jeter +jeton +jeudi +jeune +joie +joindre +joli +joueur +journal +judo +juge +juillet +juin +jument +jungle +jupe +jupon +jurer +juron +jury +jusque +juste +kayak +ketchup +kilo +kiwi +koala +label +lacet +lacune +laine +laisse +lait +lame +lancer +lande +laque +lard +largeur +larme +larve +lasso +laver +lendemain +lentement +lequel +lettre +leur +lever +levure +liane +libre +lien +lier +lieutenant +ligne +ligoter +liguer +limace +limer +limite +lingot +lion +lire +lisser +litre +livre +lobe +local +logis +loin +loisir +long +loque +lors +lotus +louer +loup +lourd +louve +loyer +lubie +lucide +lueur +luge +luire +lundi +lune +lustre +lutin +lutte +luxe +machine +madame +magie +magnifique +magot +maigre +main +mairie +maison +malade +malheur +malin +manche +manger +manier +manoir +manquer +marche +mardi +marge +mariage +marquer +mars +masque +masse +matin +mauvais +meilleur +melon +membre +menacer +mener +mensonge +mentir +menu +merci +merlu +mesure +mettre +meuble +meunier +meute +miche +micro +midi +miel +miette +mieux +milieu +mille +mimer +mince +mineur +ministre +minute +mirage +miroir +miser +mite +mixte +mobile +mode +module +moins +mois +moment +momie +monde +monsieur +monter +moquer +moral +morceau +mordre +morose +morse +mortier +morue +motif +motte +moudre +moule +mourir +mousse +mouton +mouvement +moyen +muer +muette +mugir +muguet +mulot +multiple +munir +muret +muse +musique +muter +nacre +nager +nain +naissance +narine +narrer +naseau +nasse +nation +nature +naval +navet +naviguer +navrer +neige +nerf +nerveux +neuf +neutre +neuve +neveu +niche +nier +niveau +noble +noce +nocif +noir +nomade +nombre +nommer +nord +norme +notaire +notice +notre +nouer +nougat +nourrir +nous +nouveau +novice +noyade +noyer +nuage +nuance +nuire +nuit +nulle +nuque +oasis +objet +obliger +obscur +observer +obtenir +obus +occasion +occuper +ocre +octet +odeur +odorat +offense +officier +offrir +ogive +oiseau +olive +ombre +onctueux +onduler +ongle +onze +opter +option +orageux +oral +orange +orbite +ordinaire +ordre +oreille +organe +orgie +orgueil +orient +origan +orner +orteil +ortie +oser +osselet +otage +otarie +ouate +oublier +ouest +ours +outil +outre +ouvert +ouvrir +ovale +ozone +pacte +page +paille +pain +paire +paix +palace +palissade +palmier +palpiter +panda +panneau +papa +papier +paquet +parc +pardi +parfois +parler +parmi +parole +partir +parvenir +passer +pastel +patin +patron +paume +pause +pauvre +paver +pavot +payer +pays +peau +peigne +peinture +pelage +pelote +pencher +pendre +penser +pente +percer +perdu +perle +permettre +personne +perte +peser +pesticide +petit +peuple +peur +phase +photo +phrase +piano +pied +pierre +pieu +pile +pilier +pilote +pilule +piment +pincer +pinson +pinte +pion +piquer +pirate +pire +piste +piton +pitre +pivot +pizza +placer +plage +plaire +plan +plaque +plat +plein +pleurer +pliage +plier +plonger +plot +pluie +plume +plus +pneu +poche +podium +poids +poil +point +poire +poison +poitrine +poivre +police +pollen +pomme +pompier +poncer +pondre +pont +portion +poser +position +possible +poste +potage +potin +pouce +poudre +poulet +poumon +poupe +pour +pousser +poutre +pouvoir +prairie +premier +prendre +presque +preuve +prier +primeur +prince +prison +priver +prix +prochain +produire +profond +proie +projet +promener +prononcer +propre +prose +prouver +prune +public +puce +pudeur +puiser +pull +pulpe +puma +punir +purge +putois +quand +quartier +quasi +quatre +quel +question +queue +quiche +quille +quinze +quitter +quoi +rabais +raboter +race +racheter +racine +racler +raconter +radar +radio +rafale +rage +ragot +raideur +raie +rail +raison +ramasser +ramener +rampe +rance +rang +rapace +rapide +rapport +rarement +rasage +raser +rasoir +rassurer +rater +ratio +rature +ravage +ravir +rayer +rayon +rebond +recevoir +recherche +record +reculer +redevenir +refuser +regard +regretter +rein +rejeter +rejoindre +relation +relever +religion +remarquer +remettre +remise +remonter +remplir +remuer +rencontre +rendre +renier +renoncer +rentrer +renverser +repas +repli +reposer +reproche +requin +respect +ressembler +reste +retard +retenir +retirer +retour +retrouver +revenir +revoir +revue +rhume +ricaner +riche +rideau +ridicule +rien +rigide +rincer +rire +risquer +rituel +rivage +rive +robe +robot +robuste +rocade +roche +rodeur +rogner +roman +rompre +ronce +rondeur +ronger +roque +rose +rosir +rotation +rotule +roue +rouge +rouler +route +ruban +rubis +ruche +rude +ruelle +ruer +rugby +rugir +ruine +rumeur +rural +ruse +rustre +sable +sabot +sabre +sacre +sage +saint +saisir +salade +salive +salle +salon +salto +salut +salve +samba +sandale +sanguin +sapin +sarcasme +satisfaire +sauce +sauf +sauge +saule +sauna +sauter +sauver +savoir +science +scoop +score +second +secret +secte +seigneur +sein +seize +selle +selon +semaine +sembler +semer +semis +sensuel +sentir +sept +serpe +serrer +sertir +service +seuil +seulement +short +sien +sigle +signal +silence +silo +simple +singe +sinon +sinus +sioux +sirop +site +situation +skier +snob +sobre +social +socle +sodium +soigner +soir +soixante +soja +solaire +soldat +soleil +solide +solo +solvant +sombre +somme +somnoler +sondage +songeur +sonner +sorte +sosie +sottise +souci +soudain +souffrir +souhaiter +soulever +soumettre +soupe +sourd +soustraire +soutenir +souvent +soyeux +spectacle +sport +stade +stagiaire +stand +star +statue +stock +stop +store +style +suave +subir +sucre +suer +suffire +suie +suite +suivre +sujet +sulfite +supposer +surf +surprendre +surtout +surveiller +tabac +table +tabou +tache +tacler +tacot +tact +taie +taille +taire +talon +talus +tandis +tango +tanin +tant +taper +tapis +tard +tarif +tarot +tarte +tasse +taureau +taux +taverne +taxer +taxi +tellement +temple +tendre +tenir +tenter +tenu +terme +ternir +terre +test +texte +thym +tibia +tiers +tige +tipi +tique +tirer +tissu +titre +toast +toge +toile +toiser +toiture +tomber +tome +tonne +tonte +toque +torse +tortue +totem +toucher +toujours +tour +tousser +tout +toux +trace +train +trame +tranquille +travail +trembler +trente +tribu +trier +trio +tripe +triste +troc +trois +tromper +tronc +trop +trotter +trouer +truc +truite +tuba +tuer +tuile +turbo +tutu +tuyau +type +union +unique +unir +unisson +untel +urne +usage +user +usiner +usure +utile +vache +vague +vaincre +valeur +valoir +valser +valve +vampire +vaseux +vaste +veau +veille +veine +velours +velu +vendre +venir +vent +venue +verbe +verdict +version +vertige +verve +veste +veto +vexer +vice +victime +vide +vieil +vieux +vigie +vigne +ville +vingt +violent +virer +virus +visage +viser +visite +visuel +vitamine +vitrine +vivant +vivre +vocal +vodka +vogue +voici +voile +voir +voisin +voiture +volaille +volcan +voler +volt +votant +votre +vouer +vouloir +vous +voyage +voyou +vrac +vrai +yacht +yeti +yeux +yoga +zeste +zinc +zone +zoom diff --git a/src/mnemonics/languages/german.txt b/src/mnemonics/languages/german.txt new file mode 100644 index 000000000..bec652510 --- /dev/null +++ b/src/mnemonics/languages/german.txt @@ -0,0 +1,1629 @@ +German +Deutsch +4 +Abakus +Abart +abbilden +Abbruch +Abdrift +Abendrot +Abfahrt +abfeuern +Abflug +abfragen +Abglanz +abhärten +abheben +Abhilfe +Abitur +Abkehr +Ablauf +ablecken +Ablösung +Abnehmer +abnutzen +Abonnent +Abrasion +Abrede +abrüsten +Absicht +Absprung +Abstand +absuchen +Abteil +Abundanz +abwarten +Abwurf +Abzug +Achse +Achtung +Acker +Aderlass +Adler +Admiral +Adresse +Affe +Affront +Afrika +Aggregat +Agilität +ähneln +Ahnung +Ahorn +Akazie +Akkord +Akrobat +Aktfoto +Aktivist +Albatros +Alchimie +Alemanne +Alibi +Alkohol +Allee +Allüre +Almosen +Almweide +Aloe +Alpaka +Alpental +Alphabet +Alpinist +Alraune +Altbier +Alter +Altflöte +Altruist +Alublech +Aludose +Amateur +Amazonas +Ameise +Amnesie +Amok +Ampel +Amphibie +Ampulle +Amsel +Amulett +Anakonda +Analogie +Ananas +Anarchie +Anatomie +Anbau +Anbeginn +anbieten +Anblick +ändern +andocken +Andrang +anecken +Anflug +Anfrage +Anführer +Angebot +Angler +Anhalter +Anhöhe +Animator +Anis +Anker +ankleben +Ankunft +Anlage +anlocken +Anmut +Annahme +Anomalie +Anonymus +Anorak +anpeilen +Anrecht +Anruf +Ansage +Anschein +Ansicht +Ansporn +Anteil +Antlitz +Antrag +Antwort +Anwohner +Aorta +Apfel +Appetit +Applaus +Aquarium +Arbeit +Arche +Argument +Arktis +Armband +Aroma +Asche +Askese +Asphalt +Asteroid +Ästhetik +Astronom +Atelier +Athlet +Atlantik +Atmung +Audienz +aufatmen +Auffahrt +aufholen +aufregen +Aufsatz +Auftritt +Aufwand +Augapfel +Auktion +Ausbruch +Ausflug +Ausgabe +Aushilfe +Ausland +Ausnahme +Aussage +Autobahn +Avocado +Axthieb +Bach +backen +Badesee +Bahnhof +Balance +Balkon +Ballett +Balsam +Banane +Bandage +Bankett +Barbar +Barde +Barett +Bargeld +Barkasse +Barriere +Bart +Bass +Bastler +Batterie +Bauch +Bauer +Bauholz +Baujahr +Baum +Baustahl +Bauteil +Bauweise +Bazar +beachten +Beatmung +beben +Becher +Becken +bedanken +beeilen +beenden +Beere +befinden +Befreier +Begabung +Begierde +begrüßen +Beiboot +Beichte +Beifall +Beigabe +Beil +Beispiel +Beitrag +beizen +bekommen +beladen +Beleg +bellen +belohnen +Bemalung +Bengel +Benutzer +Benzin +beraten +Bereich +Bergluft +Bericht +Bescheid +Besitz +besorgen +Bestand +Besuch +betanken +beten +betören +Bett +Beule +Beute +Bewegung +bewirken +Bewohner +bezahlen +Bezug +biegen +Biene +Bierzelt +bieten +Bikini +Bildung +Billard +binden +Biobauer +Biologe +Bionik +Biotop +Birke +Bison +Bitte +Biwak +Bizeps +blasen +Blatt +Blauwal +Blende +Blick +Blitz +Blockade +Blödelei +Blondine +Blues +Blume +Blut +Bodensee +Bogen +Boje +Bollwerk +Bonbon +Bonus +Boot +Bordarzt +Börse +Böschung +Boudoir +Boxkampf +Boykott +Brahms +Brandung +Brauerei +Brecher +Breitaxt +Bremse +brennen +Brett +Brief +Brigade +Brillanz +bringen +brodeln +Brosche +Brötchen +Brücke +Brunnen +Brüste +Brutofen +Buch +Büffel +Bugwelle +Bühne +Buletten +Bullauge +Bumerang +bummeln +Buntglas +Bürde +Burgherr +Bursche +Busen +Buslinie +Bussard +Butangas +Butter +Cabrio +campen +Captain +Cartoon +Cello +Chalet +Charisma +Chefarzt +Chiffon +Chipsatz +Chirurg +Chor +Chronik +Chuzpe +Clubhaus +Cockpit +Codewort +Cognac +Coladose +Computer +Coupon +Cousin +Cracking +Crash +Curry +Dach +Dackel +daddeln +daliegen +Dame +Dammbau +Dämon +Dampflok +Dank +Darm +Datei +Datsche +Datteln +Datum +Dauer +Daunen +Deckel +Decoder +Defekt +Degen +Dehnung +Deiche +Dekade +Dekor +Delfin +Demut +denken +Deponie +Design +Desktop +Dessert +Detail +Detektiv +Dezibel +Diadem +Diagnose +Dialekt +Diamant +Dichter +Dickicht +Diesel +Diktat +Diplom +Direktor +Dirne +Diskurs +Distanz +Docht +Dohle +Dolch +Domäne +Donner +Dorade +Dorf +Dörrobst +Dorsch +Dossier +Dozent +Drachen +Draht +Drama +Drang +Drehbuch +Dreieck +Dressur +Drittel +Drossel +Druck +Duell +Duft +Düne +Dünung +dürfen +Duschbad +Düsenjet +Dynamik +Ebbe +Echolot +Echse +Eckball +Edding +Edelweiß +Eden +Edition +Efeu +Effekte +Egoismus +Ehre +Eiablage +Eiche +Eidechse +Eidotter +Eierkopf +Eigelb +Eiland +Eilbote +Eimer +einatmen +Einband +Eindruck +Einfall +Eingang +Einkauf +einladen +Einöde +Einrad +Eintopf +Einwurf +Einzug +Eisbär +Eisen +Eishöhle +Eismeer +Eiweiß +Ekstase +Elan +Elch +Elefant +Eleganz +Element +Elfe +Elite +Elixier +Ellbogen +Eloquenz +Emigrant +Emission +Emotion +Empathie +Empfang +Endzeit +Energie +Engpass +Enkel +Enklave +Ente +entheben +Entität +entladen +Entwurf +Episode +Epoche +erachten +Erbauer +erblühen +Erdbeere +Erde +Erdgas +Erdkunde +Erdnuss +Erdöl +Erdteil +Ereignis +Eremit +erfahren +Erfolg +erfreuen +erfüllen +Ergebnis +erhitzen +erkalten +erkennen +erleben +Erlösung +ernähren +erneuern +Ernte +Eroberer +eröffnen +Erosion +Erotik +Erpel +erraten +Erreger +erröten +Ersatz +Erstflug +Ertrag +Eruption +erwarten +erwidern +Erzbau +Erzeuger +erziehen +Esel +Eskimo +Eskorte +Espe +Espresso +essen +Etage +Etappe +Etat +Ethik +Etikett +Etüde +Eule +Euphorie +Europa +Everest +Examen +Exil +Exodus +Extrakt +Fabel +Fabrik +Fachmann +Fackel +Faden +Fagott +Fahne +Faible +Fairness +Fakt +Fakultät +Falke +Fallobst +Fälscher +Faltboot +Familie +Fanclub +Fanfare +Fangarm +Fantasie +Farbe +Farmhaus +Farn +Fasan +Faser +Fassung +fasten +Faulheit +Fauna +Faust +Favorit +Faxgerät +Fazit +fechten +Federboa +Fehler +Feier +Feige +feilen +Feinripp +Feldbett +Felge +Fellpony +Felswand +Ferien +Ferkel +Fernweh +Ferse +Fest +Fettnapf +Feuer +Fiasko +Fichte +Fiktion +Film +Filter +Filz +Finanzen +Findling +Finger +Fink +Finnwal +Fisch +Fitness +Fixpunkt +Fixstern +Fjord +Flachbau +Flagge +Flamenco +Flanke +Flasche +Flaute +Fleck +Flegel +flehen +Fleisch +fliegen +Flinte +Flirt +Flocke +Floh +Floskel +Floß +Flöte +Flugzeug +Flunder +Flusstal +Flutung +Fockmast +Fohlen +Föhnlage +Fokus +folgen +Foliant +Folklore +Fontäne +Förde +Forelle +Format +Forscher +Fortgang +Forum +Fotograf +Frachter +Fragment +Fraktion +fräsen +Frauenpo +Freak +Fregatte +Freiheit +Freude +Frieden +Frohsinn +Frosch +Frucht +Frühjahr +Fuchs +Fügung +fühlen +Füller +Fundbüro +Funkboje +Funzel +Furnier +Fürsorge +Fusel +Fußbad +Futteral +Gabelung +gackern +Gage +gähnen +Galaxie +Galeere +Galopp +Gameboy +Gamsbart +Gandhi +Gang +Garage +Gardine +Garküche +Garten +Gasthaus +Gattung +gaukeln +Gazelle +Gebäck +Gebirge +Gebräu +Geburt +Gedanke +Gedeck +Gedicht +Gefahr +Gefieder +Geflügel +Gefühl +Gegend +Gehirn +Gehöft +Gehweg +Geige +Geist +Gelage +Geld +Gelenk +Gelübde +Gemälde +Gemeinde +Gemüse +genesen +Genuss +Gepäck +Geranie +Gericht +Germane +Geruch +Gesang +Geschenk +Gesetz +Gesindel +Gesöff +Gespan +Gestade +Gesuch +Getier +Getränk +Getümmel +Gewand +Geweih +Gewitter +Gewölbe +Geysir +Giftzahn +Gipfel +Giraffe +Gitarre +glänzen +Glasauge +Glatze +Gleis +Globus +Glück +glühen +Glutofen +Goldzahn +Gondel +gönnen +Gottheit +graben +Grafik +Grashalm +Graugans +greifen +Grenze +grillen +Groschen +Grotte +Grube +Grünalge +Gruppe +gruseln +Gulasch +Gummibär +Gurgel +Gürtel +Güterzug +Haarband +Habicht +hacken +hadern +Hafen +Hagel +Hähnchen +Haifisch +Haken +Halbaffe +Halsader +halten +Halunke +Handbuch +Hanf +Harfe +Harnisch +härten +Harz +Hasenohr +Haube +hauchen +Haupt +Haut +Havarie +Hebamme +hecheln +Heck +Hedonist +Heiler +Heimat +Heizung +Hektik +Held +helfen +Helium +Hemd +hemmen +Hengst +Herd +Hering +Herkunft +Hermelin +Herrchen +Herzdame +Heulboje +Hexe +Hilfe +Himbeere +Himmel +Hingabe +hinhören +Hinweis +Hirsch +Hirte +Hitzkopf +Hobel +Hochform +Hocker +hoffen +Hofhund +Hofnarr +Höhenzug +Hohlraum +Hölle +Holzboot +Honig +Honorar +horchen +Hörprobe +Höschen +Hotel +Hubraum +Hufeisen +Hügel +huldigen +Hülle +Humbug +Hummer +Humor +Hund +Hunger +Hupe +Hürde +Hurrikan +Hydrant +Hypnose +Ibis +Idee +Idiot +Igel +Illusion +Imitat +impfen +Import +Inferno +Ingwer +Inhalte +Inland +Insekt +Ironie +Irrfahrt +Irrtum +Isolator +Istwert +Jacke +Jade +Jagdhund +Jäger +Jaguar +Jahr +Jähzorn +Jazzfest +Jetpilot +jobben +Jochbein +jodeln +Jodsalz +Jolle +Journal +Jubel +Junge +Junimond +Jupiter +Jutesack +Juwel +Kabarett +Kabine +Kabuff +Käfer +Kaffee +Kahlkopf +Kaimauer +Kajüte +Kaktus +Kaliber +Kaltluft +Kamel +kämmen +Kampagne +Kanal +Känguru +Kanister +Kanone +Kante +Kanu +kapern +Kapitän +Kapuze +Karneval +Karotte +Käsebrot +Kasper +Kastanie +Katalog +Kathode +Katze +kaufen +Kaugummi +Kauz +Kehle +Keilerei +Keksdose +Kellner +Keramik +Kerze +Kessel +Kette +keuchen +kichern +Kielboot +Kindheit +Kinnbart +Kinosaal +Kiosk +Kissen +Klammer +Klang +Klapprad +Klartext +kleben +Klee +Kleinod +Klima +Klingel +Klippe +Klischee +Kloster +Klugheit +Klüngel +kneten +Knie +Knöchel +knüpfen +Kobold +Kochbuch +Kohlrabi +Koje +Kokosöl +Kolibri +Kolumne +Kombüse +Komiker +kommen +Konto +Konzept +Kopfkino +Kordhose +Korken +Korsett +Kosename +Krabbe +Krach +Kraft +Krähe +Kralle +Krapfen +Krater +kraulen +Kreuz +Krokodil +Kröte +Kugel +Kuhhirt +Kühnheit +Künstler +Kurort +Kurve +Kurzfilm +kuscheln +küssen +Kutter +Labor +lachen +Lackaffe +Ladeluke +Lagune +Laib +Lakritze +Lammfell +Land +Langmut +Lappalie +Last +Laterne +Latzhose +Laubsäge +laufen +Laune +Lausbub +Lavasee +Leben +Leder +Leerlauf +Lehm +Lehrer +leihen +Lektüre +Lenker +Lerche +Leseecke +Leuchter +Lexikon +Libelle +Libido +Licht +Liebe +liefern +Liftboy +Limonade +Lineal +Linoleum +List +Liveband +Lobrede +locken +Löffel +Logbuch +Logik +Lohn +Loipe +Lokal +Lorbeer +Lösung +löten +Lottofee +Löwe +Luchs +Luder +Luftpost +Luke +Lümmel +Lunge +lutschen +Luxus +Macht +Magazin +Magier +Magnet +mähen +Mahlzeit +Mahnmal +Maibaum +Maisbrei +Makel +malen +Mammut +Maniküre +Mantel +Marathon +Marder +Marine +Marke +Marmor +Märzluft +Maske +Maßanzug +Maßkrug +Mastkorb +Material +Matratze +Mauerbau +Maulkorb +Mäuschen +Mäzen +Medium +Meinung +melden +Melodie +Mensch +Merkmal +Messe +Metall +Meteor +Methode +Metzger +Mieze +Milchkuh +Mimose +Minirock +Minute +mischen +Missetat +mitgehen +Mittag +Mixtape +Möbel +Modul +mögen +Möhre +Molch +Moment +Monat +Mondflug +Monitor +Monokini +Monster +Monument +Moorhuhn +Moos +Möpse +Moral +Mörtel +Motiv +Motorrad +Möwe +Mühe +Mulatte +Müller +Mumie +Mund +Münze +Muschel +Muster +Mythos +Nabel +Nachtzug +Nackedei +Nagel +Nähe +Nähnadel +Namen +Narbe +Narwal +Nasenbär +Natur +Nebel +necken +Neffe +Neigung +Nektar +Nenner +Neptun +Nerz +Nessel +Nestbau +Netz +Neubau +Neuerung +Neugier +nicken +Niere +Nilpferd +nisten +Nocke +Nomade +Nordmeer +Notdurft +Notstand +Notwehr +Nudismus +Nuss +Nutzhanf +Oase +Obdach +Oberarzt +Objekt +Oboe +Obsthain +Ochse +Odyssee +Ofenholz +öffnen +Ohnmacht +Ohrfeige +Ohrwurm +Ökologie +Oktave +Ölberg +Olive +Ölkrise +Omelett +Onkel +Oper +Optiker +Orange +Orchidee +ordnen +Orgasmus +Orkan +Ortskern +Ortung +Ostasien +Ozean +Paarlauf +Packeis +paddeln +Paket +Palast +Pandabär +Panik +Panorama +Panther +Papagei +Papier +Paprika +Paradies +Parka +Parodie +Partner +Passant +Patent +Patzer +Pause +Pavian +Pedal +Pegel +peilen +Perle +Person +Pfad +Pfau +Pferd +Pfleger +Physik +Pier +Pilotwal +Pinzette +Piste +Plakat +Plankton +Platin +Plombe +plündern +Pobacke +Pokal +polieren +Popmusik +Porträt +Posaune +Postamt +Pottwal +Pracht +Pranke +Preis +Primat +Prinzip +Protest +Proviant +Prüfung +Pubertät +Pudding +Pullover +Pulsader +Punkt +Pute +Putsch +Puzzle +Python +quaken +Qualle +Quark +Quellsee +Querkopf +Quitte +Quote +Rabauke +Rache +Radclub +Radhose +Radio +Radtour +Rahmen +Rampe +Randlage +Ranzen +Rapsöl +Raserei +rasten +Rasur +Rätsel +Raubtier +Raumzeit +Rausch +Reaktor +Realität +Rebell +Rede +Reetdach +Regatta +Regen +Rehkitz +Reifen +Reim +Reise +Reizung +Rekord +Relevanz +Rennboot +Respekt +Restmüll +retten +Reue +Revolte +Rhetorik +Rhythmus +Richtung +Riegel +Rindvieh +Rippchen +Ritter +Robbe +Roboter +Rockband +Rohdaten +Roller +Roman +röntgen +Rose +Rosskur +Rost +Rotahorn +Rotglut +Rotznase +Rubrik +Rückweg +Rufmord +Ruhe +Ruine +Rumpf +Runde +Rüstung +rütteln +Saaltür +Saatguts +Säbel +Sachbuch +Sack +Saft +sagen +Sahneeis +Salat +Salbe +Salz +Sammlung +Samt +Sandbank +Sanftmut +Sardine +Satire +Sattel +Satzbau +Sauerei +Saum +Säure +Schall +Scheitel +Schiff +Schlager +Schmied +Schnee +Scholle +Schrank +Schulbus +Schwan +Seeadler +Seefahrt +Seehund +Seeufer +segeln +Sehnerv +Seide +Seilzug +Senf +Sessel +Seufzer +Sexgott +Sichtung +Signal +Silber +singen +Sinn +Sirup +Sitzbank +Skandal +Skikurs +Skipper +Skizze +Smaragd +Socke +Sohn +Sommer +Songtext +Sorte +Spagat +Spannung +Spargel +Specht +Speiseöl +Spiegel +Sport +spülen +Stadtbus +Stall +Stärke +Stativ +staunen +Stern +Stiftung +Stollen +Strömung +Sturm +Substanz +Südalpen +Sumpf +surfen +Tabak +Tafel +Tagebau +takeln +Taktung +Talsohle +Tand +Tanzbär +Tapir +Tarantel +Tarnname +Tasse +Tatnacht +Tatsache +Tatze +Taube +tauchen +Taufpate +Taumel +Teelicht +Teich +teilen +Tempo +Tenor +Terrasse +Testflug +Theater +Thermik +ticken +Tiefflug +Tierart +Tigerhai +Tinte +Tischler +toben +Toleranz +Tölpel +Tonband +Topf +Topmodel +Torbogen +Torlinie +Torte +Tourist +Tragesel +trampeln +Trapez +Traum +treffen +Trennung +Treue +Trick +trimmen +Trödel +Trost +Trumpf +tüfteln +Turban +Turm +Übermut +Ufer +Uhrwerk +umarmen +Umbau +Umfeld +Umgang +Umsturz +Unart +Unfug +Unimog +Unruhe +Unwucht +Uranerz +Urlaub +Urmensch +Utopie +Vakuum +Valuta +Vandale +Vase +Vektor +Ventil +Verb +Verdeck +Verfall +Vergaser +verhexen +Verlag +Vers +Vesper +Vieh +Viereck +Vinyl +Virus +Vitrine +Vollblut +Vorbote +Vorrat +Vorsicht +Vulkan +Wachstum +Wade +Wagemut +Wahlen +Wahrheit +Wald +Walhai +Wallach +Walnuss +Walzer +wandeln +Wanze +wärmen +Warnruf +Wäsche +Wasser +Weberei +wechseln +Wegegeld +wehren +Weiher +Weinglas +Weißbier +Weitwurf +Welle +Weltall +Werkbank +Werwolf +Wetter +wiehern +Wildgans +Wind +Wohl +Wohnort +Wolf +Wollust +Wortlaut +Wrack +Wunder +Wurfaxt +Wurst +Yacht +Yeti +Zacke +Zahl +zähmen +Zahnfee +Zäpfchen +Zaster +Zaumzeug +Zebra +zeigen +Zeitlupe +Zellkern +Zeltdach +Zensor +Zerfall +Zeug +Ziege +Zielfoto +Zimteis +Zobel +Zollhund +Zombie +Zöpfe +Zucht +Zufahrt +Zugfahrt +Zugvogel +Zündung +Zweck +Zyklop diff --git a/src/mnemonics/languages/italian.txt b/src/mnemonics/languages/italian.txt new file mode 100644 index 000000000..16613df41 --- /dev/null +++ b/src/mnemonics/languages/italian.txt @@ -0,0 +1,1629 @@ +Italian +Italiano +4 +abbinare +abbonato +abisso +abitare +abominio +accadere +accesso +acciaio +accordo +accumulo +acido +acqua +acrobata +acustico +adattare +addetto +addio +addome +adeguato +aderire +adorare +adottare +adozione +adulto +aereo +aerobica +affare +affetto +affidare +affogato +affronto +africano +afrodite +agenzia +aggancio +aggeggio +aggiunta +agio +agire +agitare +aglio +agnello +agosto +aiutare +albero +albo +alce +alchimia +alcool +alfabeto +algebra +alimento +allarme +alleanza +allievo +alloggio +alluce +alpi +alterare +altro +aluminio +amante +amarezza +ambiente +ambrosia +america +amico +ammalare +ammirare +amnesia +amnistia +amore +ampliare +amputare +analisi +anamnesi +ananas +anarchia +anatra +anca +ancorato +andare +androide +aneddoto +anello +angelo +angolino +anguilla +anidride +anima +annegare +anno +annuncio +anomalia +antenna +anticipo +aperto +apostolo +appalto +appello +appiglio +applauso +appoggio +appurare +aprile +aquila +arabo +arachidi +aragosta +arancia +arbitrio +archivio +arco +argento +argilla +aria +ariete +arma +armonia +aroma +arrivare +arrosto +arsenale +arte +artiglio +asfalto +asfissia +asino +asparagi +aspirina +assalire +assegno +assolto +assurdo +asta +astratto +atlante +atletica +atomo +atropina +attacco +attesa +attico +atto +attrarre +auguri +aula +aumento +aurora +auspicio +autista +auto +autunno +avanzare +avarizia +avere +aviatore +avido +avorio +avvenire +avviso +avvocato +azienda +azione +azzardo +azzurro +babbuino +bacio +badante +baffi +bagaglio +bagliore +bagno +balcone +balena +ballare +balordo +balsamo +bambola +bancomat +banda +barato +barba +barista +barriera +basette +basilico +bassista +bastare +battello +bavaglio +beccare +beduino +bellezza +bene +benzina +berretto +bestia +bevitore +bianco +bibbia +biberon +bibita +bici +bidone +bilancia +biliardo +binario +binocolo +biologia +biondina +biopsia +biossido +birbante +birra +biscotto +bisogno +bistecca +bivio +blindare +bloccare +bocca +bollire +bombola +bonifico +borghese +borsa +bottino +botulino +braccio +bradipo +branco +bravo +bresaola +bretelle +brevetto +briciola +brigante +brillare +brindare +brivido +broccoli +brontolo +bruciare +brufolo +bucare +buddista +budino +bufera +buffo +bugiardo +buio +buono +burrone +bussola +bustina +buttare +cabernet +cabina +cacao +cacciare +cactus +cadavere +caffe +calamari +calcio +caldaia +calmare +calunnia +calvario +calzone +cambiare +camera +camion +cammello +campana +canarino +cancello +candore +cane +canguro +cannone +canoa +cantare +canzone +caos +capanna +capello +capire +capo +capperi +capra +capsula +caraffa +carbone +carciofo +cardigan +carenza +caricare +carota +carrello +carta +casa +cascare +caserma +cashmere +casino +cassetta +castello +catalogo +catena +catorcio +cattivo +causa +cauzione +cavallo +caverna +caviglia +cavo +cazzotto +celibato +cemento +cenare +centrale +ceramica +cercare +ceretta +cerniera +certezza +cervello +cessione +cestino +cetriolo +chiave +chiedere +chilo +chimera +chiodo +chirurgo +chitarra +chiudere +ciabatta +ciao +cibo +ciccia +cicerone +ciclone +cicogna +cielo +cifra +cigno +ciliegia +cimitero +cinema +cinque +cintura +ciondolo +ciotola +cipolla +cippato +circuito +cisterna +citofono +ciuccio +civetta +civico +clausola +cliente +clima +clinica +cobra +coccole +cocktail +cocomero +codice +coesione +cogliere +cognome +colla +colomba +colpire +coltello +comando +comitato +commedia +comodino +compagna +comune +concerto +condotto +conforto +congiura +coniglio +consegna +conto +convegno +coperta +copia +coprire +corazza +corda +corleone +cornice +corona +corpo +corrente +corsa +cortesia +corvo +coso +costume +cotone +cottura +cozza +crampo +cratere +cravatta +creare +credere +crema +crescere +crimine +criterio +croce +crollare +cronaca +crostata +croupier +cubetto +cucciolo +cucina +cultura +cuoco +cuore +cupido +cupola +cura +curva +cuscino +custode +danzare +data +decennio +decidere +decollo +dedicare +dedurre +definire +delegare +delfino +delitto +demone +dentista +denuncia +deposito +derivare +deserto +designer +destino +detonare +dettagli +diagnosi +dialogo +diamante +diario +diavolo +dicembre +difesa +digerire +digitare +diluvio +dinamica +dipinto +diploma +diramare +dire +dirigere +dirupo +discesa +disdetta +disegno +disporre +dissenso +distacco +dito +ditta +diva +divenire +dividere +divorare +docente +dolcetto +dolore +domatore +domenica +dominare +donatore +donna +dorato +dormire +dorso +dosaggio +dottore +dovere +download +dragone +dramma +dubbio +dubitare +duetto +durata +ebbrezza +eccesso +eccitare +eclissi +economia +edera +edificio +editore +edizione +educare +effetto +egitto +egiziano +elastico +elefante +eleggere +elemento +elenco +elezione +elmetto +elogio +embrione +emergere +emettere +eminenza +emisfero +emozione +empatia +energia +enfasi +enigma +entrare +enzima +epidemia +epilogo +episodio +epoca +equivoco +erba +erede +eroe +erotico +errore +eruzione +esaltare +esame +esaudire +eseguire +esempio +esigere +esistere +esito +esperto +espresso +essere +estasi +esterno +estrarre +eterno +etica +euforico +europa +evacuare +evasione +evento +evidenza +evitare +evolvere +fabbrica +facciata +fagiano +fagotto +falco +fame +famiglia +fanale +fango +fantasia +farfalla +farmacia +faro +fase +fastidio +faticare +fatto +favola +febbre +femmina +femore +fenomeno +fermata +feromoni +ferrari +fessura +festa +fiaba +fiamma +fianco +fiat +fibbia +fidare +fieno +figa +figlio +figura +filetto +filmato +filosofo +filtrare +finanza +finestra +fingere +finire +finta +finzione +fiocco +fioraio +firewall +firmare +fisico +fissare +fittizio +fiume +flacone +flagello +flirtare +flusso +focaccia +foglio +fognario +follia +fonderia +fontana +forbici +forcella +foresta +forgiare +formare +fornace +foro +fortuna +forzare +fosforo +fotoni +fracasso +fragola +frantumi +fratello +frazione +freccia +freddo +frenare +fresco +friggere +frittata +frivolo +frizione +fronte +frullato +frumento +frusta +frutto +fucile +fuggire +fulmine +fumare +funzione +fuoco +furbizia +furgone +furia +furore +fusibile +fuso +futuro +gabbiano +galassia +gallina +gamba +gancio +garanzia +garofano +gasolio +gatto +gazebo +gazzetta +gelato +gemelli +generare +genitori +gennaio +geologia +germania +gestire +gettare +ghepardo +ghiaccio +giaccone +giaguaro +giallo +giappone +giardino +gigante +gioco +gioiello +giorno +giovane +giraffa +giudizio +giurare +giusto +globo +gloria +glucosio +gnocca +gocciola +godere +gomito +gomma +gonfiare +gorilla +governo +gradire +graffiti +granchio +grappolo +grasso +grattare +gridare +grissino +grondaia +grugnito +gruppo +guadagno +guaio +guancia +guardare +gufo +guidare +guscio +gusto +icona +idea +identico +idolo +idoneo +idrante +idrogeno +igiene +ignoto +imbarco +immagine +immobile +imparare +impedire +impianto +importo +impresa +impulso +incanto +incendio +incidere +incontro +incrocia +incubo +indagare +indice +indotto +infanzia +inferno +infinito +infranto +ingerire +inglese +ingoiare +ingresso +iniziare +innesco +insalata +inserire +insicuro +insonnia +insulto +interno +introiti +invasori +inverno +invito +invocare +ipnosi +ipocrita +ipotesi +ironia +irrigare +iscritto +isola +ispirare +isterico +istinto +istruire +italiano +jazz +labbra +labrador +ladro +lago +lamento +lampone +lancetta +lanterna +lapide +larva +lasagne +lasciare +lastra +latte +laurea +lavagna +lavorare +leccare +legare +leggere +lenzuolo +leone +lepre +letargo +lettera +levare +levitare +lezione +liberare +libidine +libro +licenza +lievito +limite +lince +lingua +liquore +lire +listino +litigare +litro +locale +lottare +lucciola +lucidare +luglio +luna +macchina +madama +madre +maestro +maggio +magico +maglione +magnolia +mago +maialino +maionese +malattia +male +malloppo +mancare +mandorla +mangiare +manico +manopola +mansarda +mantello +manubrio +manzo +mappa +mare +margine +marinaio +marmotta +marocco +martello +marzo +maschera +matrice +maturare +mazzetta +meandri +medaglia +medico +medusa +megafono +melone +membrana +menta +mercato +meritare +merluzzo +mese +mestiere +metafora +meteo +metodo +mettere +miele +miglio +miliardo +mimetica +minatore +minuto +miracolo +mirtillo +missile +mistero +misura +mito +mobile +moda +moderare +moglie +molecola +molle +momento +moneta +mongolia +monologo +montagna +morale +morbillo +mordere +mosaico +mosca +mostro +motivare +moto +mulino +mulo +muovere +muraglia +muscolo +museo +musica +mutande +nascere +nastro +natale +natura +nave +navigare +negare +negozio +nemico +nero +nervo +nessuno +nettare +neutroni +neve +nevicare +nicotina +nido +nipote +nocciola +noleggio +nome +nonno +norvegia +notare +notizia +nove +nucleo +nuda +nuotare +nutrire +obbligo +occhio +occupare +oceano +odissea +odore +offerta +officina +offrire +oggetto +oggi +olfatto +olio +oliva +ombelico +ombrello +omuncolo +ondata +onore +opera +opinione +opuscolo +opzione +orario +orbita +orchidea +ordine +orecchio +orgasmo +orgoglio +origine +orologio +oroscopo +orso +oscurare +ospedale +ospite +ossigeno +ostacolo +ostriche +ottenere +ottimo +ottobre +ovest +pacco +pace +pacifico +padella +pagare +pagina +pagnotta +palazzo +palestra +palpebre +pancetta +panfilo +panino +pannello +panorama +papa +paperino +paradiso +parcella +parente +parlare +parodia +parrucca +partire +passare +pasta +patata +patente +patogeno +patriota +pausa +pazienza +peccare +pecora +pedalare +pelare +pena +pendenza +penisola +pennello +pensare +pentirsi +percorso +perdono +perfetto +perizoma +perla +permesso +persona +pesare +pesce +peso +petardo +petrolio +pezzo +piacere +pianeta +piastra +piatto +piazza +piccolo +piede +piegare +pietra +pigiama +pigliare +pigrizia +pilastro +pilota +pinguino +pioggia +piombo +pionieri +piovra +pipa +pirata +pirolisi +piscina +pisolino +pista +pitone +piumino +pizza +plastica +platino +poesia +poiana +polaroid +polenta +polimero +pollo +polmone +polpetta +poltrona +pomodoro +pompa +popolo +porco +porta +porzione +possesso +postino +potassio +potere +poverino +pranzo +prato +prefisso +prelievo +premio +prendere +prestare +pretesa +prezzo +primario +privacy +problema +processo +prodotto +profeta +progetto +promessa +pronto +proposta +proroga +prossimo +proteina +prova +prudenza +pubblico +pudore +pugilato +pulire +pulsante +puntare +pupazzo +puzzle +quaderno +qualcuno +quarzo +quercia +quintale +rabbia +racconto +radice +raffica +ragazza +ragione +rammento +ramo +rana +randagio +rapace +rapinare +rapporto +rasatura +ravioli +reagire +realista +reattore +reazione +recitare +recluso +record +recupero +redigere +regalare +regina +regola +relatore +reliquia +remare +rendere +reparto +resina +resto +rete +retorica +rettile +revocare +riaprire +ribadire +ribelle +ricambio +ricetta +richiamo +ricordo +ridurre +riempire +riferire +riflesso +righello +rilancio +rilevare +rilievo +rimanere +rimborso +rinforzo +rinuncia +riparo +ripetere +riposare +ripulire +risalita +riscatto +riserva +riso +rispetto +ritaglio +ritmo +ritorno +ritratto +rituale +riunione +riuscire +riva +robotica +rondine +rosa +rospo +rosso +rotonda +rotta +roulotte +rubare +rubrica +ruffiano +rumore +ruota +ruscello +sabbia +sacco +saggio +sale +salire +salmone +salto +salutare +salvia +sangue +sanzioni +sapere +sapienza +sarcasmo +sardine +sartoria +sbalzo +sbarcare +sberla +sborsare +scadenza +scafo +scala +scambio +scappare +scarpa +scatola +scelta +scena +sceriffo +scheggia +schiuma +sciarpa +scienza +scimmia +sciopero +scivolo +sclerare +scolpire +sconto +scopa +scordare +scossa +scrivere +scrupolo +scuderia +scultore +scuola +scusare +sdraiare +secolo +sedativo +sedere +sedia +segare +segreto +seguire +semaforo +seme +senape +seno +sentiero +separare +sepolcro +sequenza +serata +serpente +servizio +sesso +seta +settore +sfamare +sfera +sfidare +sfiorare +sfogare +sgabello +sicuro +siepe +sigaro +silenzio +silicone +simbiosi +simpatia +simulare +sinapsi +sindrome +sinergia +sinonimo +sintonia +sirena +siringa +sistema +sito +smalto +smentire +smontare +soccorso +socio +soffitto +software +soggetto +sogliola +sognare +soldi +sole +sollievo +solo +sommario +sondare +sonno +sorpresa +sorriso +sospiro +sostegno +sovrano +spaccare +spada +spagnolo +spalla +sparire +spavento +spazio +specchio +spedire +spegnere +spendere +speranza +spessore +spezzare +spiaggia +spiccare +spiegare +spiffero +spingere +sponda +sporcare +spostare +spremuta +spugna +spumante +spuntare +squadra +squillo +staccare +stadio +stagione +stallone +stampa +stancare +starnuto +statura +stella +stendere +sterzo +stilista +stimolo +stinco +stiva +stoffa +storia +strada +stregone +striscia +studiare +stufa +stupendo +subire +successo +sudare +suono +superare +supporto +surfista +sussurro +svelto +svenire +sviluppo +svolta +svuotare +tabacco +tabella +tabu +tacchino +tacere +taglio +talento +tangente +tappeto +tartufo +tassello +tastiera +tavolo +tazza +teatro +tedesco +telaio +telefono +tema +temere +tempo +tendenza +tenebre +tensione +tentare +teologia +teorema +termica +terrazzo +teschio +tesi +tesoro +tessera +testa +thriller +tifoso +tigre +timbrare +timido +tinta +tirare +tisana +titano +titolo +toccare +togliere +topolino +torcia +torrente +tovaglia +traffico +tragitto +training +tramonto +transito +trapezio +trasloco +trattore +trazione +treccia +tregua +treno +triciclo +tridente +trilogia +tromba +troncare +trota +trovare +trucco +tubo +tulipano +tumulto +tunisia +tuono +turista +tuta +tutelare +tutore +ubriaco +uccello +udienza +udito +uffa +umanoide +umore +unghia +unguento +unicorno +unione +universo +uomo +uragano +uranio +urlare +uscire +utente +utilizzo +vacanza +vacca +vaglio +vagonata +valle +valore +valutare +valvola +vampiro +vaniglia +vanto +vapore +variante +vasca +vaselina +vassoio +vedere +vegetale +veglia +veicolo +vela +veleno +velivolo +velluto +vendere +venerare +venire +vento +veranda +verbo +verdura +vergine +verifica +vernice +vero +verruca +versare +vertebra +vescica +vespaio +vestito +vesuvio +veterano +vetro +vetta +viadotto +viaggio +vibrare +vicenda +vichingo +vietare +vigilare +vigneto +villa +vincere +violino +vipera +virgola +virtuoso +visita +vita +vitello +vittima +vivavoce +vivere +viziato +voglia +volare +volpe +volto +volume +vongole +voragine +vortice +votare +vulcano +vuotare +zabaione +zaffiro +zainetto +zampa +zanzara +zattera +zavorra +zenzero +zero +zingaro +zittire +zoccolo +zolfo +zombie +zucchero diff --git a/src/mnemonics/languages/japanese.txt b/src/mnemonics/languages/japanese.txt new file mode 100644 index 000000000..bbf1f4c4d --- /dev/null +++ b/src/mnemonics/languages/japanese.txt @@ -0,0 +1,1629 @@ +Japanese +日本語 +3 +あいこくしん +あいさつ +あいだ +あおぞら +あかちゃん +あきる +あけがた +あける +あこがれる +あさい +あさひ +あしあと +あじわう +あずかる +あずき +あそぶ +あたえる +あたためる +あたりまえ +あたる +あつい +あつかう +あっしゅく +あつまり +あつめる +あてな +あてはまる +あひる +あぶら +あぶる +あふれる +あまい +あまど +あまやかす +あまり +あみもの +あめりか +あやまる +あゆむ +あらいぐま +あらし +あらすじ +あらためる +あらゆる +あらわす +ありがとう +あわせる +あわてる +あんい +あんがい +あんこ +あんぜん +あんてい +あんない +あんまり +いいだす +いおん +いがい +いがく +いきおい +いきなり +いきもの +いきる +いくじ +いくぶん +いけばな +いけん +いこう +いこく +いこつ +いさましい +いさん +いしき +いじゅう +いじょう +いじわる +いずみ +いずれ +いせい +いせえび +いせかい +いせき +いぜん +いそうろう +いそがしい +いだい +いだく +いたずら +いたみ +いたりあ +いちおう +いちじ +いちど +いちば +いちぶ +いちりゅう +いつか +いっしゅん +いっせい +いっそう +いったん +いっち +いってい +いっぽう +いてざ +いてん +いどう +いとこ +いない +いなか +いねむり +いのち +いのる +いはつ +いばる +いはん +いびき +いひん +いふく +いへん +いほう +いみん +いもうと +いもたれ +いもり +いやがる +いやす +いよかん +いよく +いらい +いらすと +いりぐち +いりょう +いれい +いれもの +いれる +いろえんぴつ +いわい +いわう +いわかん +いわば +いわゆる +いんげんまめ +いんさつ +いんしょう +いんよう +うえき +うえる +うおざ +うがい +うかぶ +うかべる +うきわ +うくらいな +うくれれ +うけたまわる +うけつけ +うけとる +うけもつ +うける +うごかす +うごく +うこん +うさぎ +うしなう +うしろがみ +うすい +うすぎ +うすぐらい +うすめる +うせつ +うちあわせ +うちがわ +うちき +うちゅう +うっかり +うつくしい +うったえる +うつる +うどん +うなぎ +うなじ +うなずく +うなる +うねる +うのう +うぶげ +うぶごえ +うまれる +うめる +うもう +うやまう +うよく +うらがえす +うらぐち +うらない +うりあげ +うりきれ +うるさい +うれしい +うれゆき +うれる +うろこ +うわき +うわさ +うんこう +うんちん +うんてん +うんどう +えいえん +えいが +えいきょう +えいご +えいせい +えいぶん +えいよう +えいわ +えおり +えがお +えがく +えきたい +えくせる +えしゃく +えすて +えつらん +えのぐ +えほうまき +えほん +えまき +えもじ +えもの +えらい +えらぶ +えりあ +えんえん +えんかい +えんぎ +えんげき +えんしゅう +えんぜつ +えんそく +えんちょう +えんとつ +おいかける +おいこす +おいしい +おいつく +おうえん +おうさま +おうじ +おうせつ +おうたい +おうふく +おうべい +おうよう +おえる +おおい +おおう +おおどおり +おおや +おおよそ +おかえり +おかず +おがむ +おかわり +おぎなう +おきる +おくさま +おくじょう +おくりがな +おくる +おくれる +おこす +おこなう +おこる +おさえる +おさない +おさめる +おしいれ +おしえる +おじぎ +おじさん +おしゃれ +おそらく +おそわる +おたがい +おたく +おだやか +おちつく +おっと +おつり +おでかけ +おとしもの +おとなしい +おどり +おどろかす +おばさん +おまいり +おめでとう +おもいで +おもう +おもたい +おもちゃ +おやつ +おやゆび +およぼす +おらんだ +おろす +おんがく +おんけい +おんしゃ +おんせん +おんだん +おんちゅう +おんどけい +かあつ +かいが +がいき +がいけん +がいこう +かいさつ +かいしゃ +かいすいよく +かいぜん +かいぞうど +かいつう +かいてん +かいとう +かいふく +がいへき +かいほう +かいよう +がいらい +かいわ +かえる +かおり +かかえる +かがく +かがし +かがみ +かくご +かくとく +かざる +がぞう +かたい +かたち +がちょう +がっきゅう +がっこう +がっさん +がっしょう +かなざわし +かのう +がはく +かぶか +かほう +かほご +かまう +かまぼこ +かめれおん +かゆい +かようび +からい +かるい +かろう +かわく +かわら +がんか +かんけい +かんこう +かんしゃ +かんそう +かんたん +かんち +がんばる +きあい +きあつ +きいろ +ぎいん +きうい +きうん +きえる +きおう +きおく +きおち +きおん +きかい +きかく +きかんしゃ +ききて +きくばり +きくらげ +きけんせい +きこう +きこえる +きこく +きさい +きさく +きさま +きさらぎ +ぎじかがく +ぎしき +ぎじたいけん +ぎじにってい +ぎじゅつしゃ +きすう +きせい +きせき +きせつ +きそう +きぞく +きぞん +きたえる +きちょう +きつえん +ぎっちり +きつつき +きつね +きてい +きどう +きどく +きない +きなが +きなこ +きぬごし +きねん +きのう +きのした +きはく +きびしい +きひん +きふく +きぶん +きぼう +きほん +きまる +きみつ +きむずかしい +きめる +きもだめし +きもち +きもの +きゃく +きやく +ぎゅうにく +きよう +きょうりゅう +きらい +きらく +きりん +きれい +きれつ +きろく +ぎろん +きわめる +ぎんいろ +きんかくじ +きんじょ +きんようび +ぐあい +くいず +くうかん +くうき +くうぐん +くうこう +ぐうせい +くうそう +ぐうたら +くうふく +くうぼ +くかん +くきょう +くげん +ぐこう +くさい +くさき +くさばな +くさる +くしゃみ +くしょう +くすのき +くすりゆび +くせげ +くせん +ぐたいてき +くださる +くたびれる +くちこみ +くちさき +くつした +ぐっすり +くつろぐ +くとうてん +くどく +くなん +くねくね +くのう +くふう +くみあわせ +くみたてる +くめる +くやくしょ +くらす +くらべる +くるま +くれる +くろう +くわしい +ぐんかん +ぐんしょく +ぐんたい +ぐんて +けあな +けいかく +けいけん +けいこ +けいさつ +げいじゅつ +けいたい +げいのうじん +けいれき +けいろ +けおとす +けおりもの +げきか +げきげん +げきだん +げきちん +げきとつ +げきは +げきやく +げこう +げこくじょう +げざい +けさき +げざん +けしき +けしごむ +けしょう +げすと +けたば +けちゃっぷ +けちらす +けつあつ +けつい +けつえき +けっこん +けつじょ +けっせき +けってい +けつまつ +げつようび +げつれい +けつろん +げどく +けとばす +けとる +けなげ +けなす +けなみ +けぬき +げねつ +けねん +けはい +げひん +けぶかい +げぼく +けまり +けみかる +けむし +けむり +けもの +けらい +けろけろ +けわしい +けんい +けんえつ +けんお +けんか +げんき +けんげん +けんこう +けんさく +けんしゅう +けんすう +げんそう +けんちく +けんてい +けんとう +けんない +けんにん +げんぶつ +けんま +けんみん +けんめい +けんらん +けんり +こあくま +こいぬ +こいびと +ごうい +こうえん +こうおん +こうかん +ごうきゅう +ごうけい +こうこう +こうさい +こうじ +こうすい +ごうせい +こうそく +こうたい +こうちゃ +こうつう +こうてい +こうどう +こうない +こうはい +ごうほう +ごうまん +こうもく +こうりつ +こえる +こおり +ごかい +ごがつ +ごかん +こくご +こくさい +こくとう +こくない +こくはく +こぐま +こけい +こける +ここのか +こころ +こさめ +こしつ +こすう +こせい +こせき +こぜん +こそだて +こたい +こたえる +こたつ +こちょう +こっか +こつこつ +こつばん +こつぶ +こてい +こてん +ことがら +ことし +ことば +ことり +こなごな +こねこね +このまま +このみ +このよ +ごはん +こひつじ +こふう +こふん +こぼれる +ごまあぶら +こまかい +ごますり +こまつな +こまる +こむぎこ +こもじ +こもち +こもの +こもん +こやく +こやま +こゆう +こゆび +こよい +こよう +こりる +これくしょん +ころっけ +こわもて +こわれる +こんいん +こんかい +こんき +こんしゅう +こんすい +こんだて +こんとん +こんなん +こんびに +こんぽん +こんまけ +こんや +こんれい +こんわく +ざいえき +さいかい +さいきん +ざいげん +ざいこ +さいしょ +さいせい +ざいたく +ざいちゅう +さいてき +ざいりょう +さうな +さかいし +さがす +さかな +さかみち +さがる +さぎょう +さくし +さくひん +さくら +さこく +さこつ +さずかる +ざせき +さたん +さつえい +ざつおん +ざっか +ざつがく +さっきょく +ざっし +さつじん +ざっそう +さつたば +さつまいも +さてい +さといも +さとう +さとおや +さとし +さとる +さのう +さばく +さびしい +さべつ +さほう +さほど +さます +さみしい +さみだれ +さむけ +さめる +さやえんどう +さゆう +さよう +さよく +さらだ +ざるそば +さわやか +さわる +さんいん +さんか +さんきゃく +さんこう +さんさい +ざんしょ +さんすう +さんせい +さんそ +さんち +さんま +さんみ +さんらん +しあい +しあげ +しあさって +しあわせ +しいく +しいん +しうち +しえい +しおけ +しかい +しかく +じかん +しごと +しすう +じだい +したうけ +したぎ +したて +したみ +しちょう +しちりん +しっかり +しつじ +しつもん +してい +してき +してつ +じてん +じどう +しなぎれ +しなもの +しなん +しねま +しねん +しのぐ +しのぶ +しはい +しばかり +しはつ +しはらい +しはん +しひょう +しふく +じぶん +しへい +しほう +しほん +しまう +しまる +しみん +しむける +じむしょ +しめい +しめる +しもん +しゃいん +しゃうん +しゃおん +じゃがいも +しやくしょ +しゃくほう +しゃけん +しゃこ +しゃざい +しゃしん +しゃせん +しゃそう +しゃたい +しゃちょう +しゃっきん +じゃま +しゃりん +しゃれい +じゆう +じゅうしょ +しゅくはく +じゅしん +しゅっせき +しゅみ +しゅらば +じゅんばん +しょうかい +しょくたく +しょっけん +しょどう +しょもつ +しらせる +しらべる +しんか +しんこう +じんじゃ +しんせいじ +しんちく +しんりん +すあげ +すあし +すあな +ずあん +すいえい +すいか +すいとう +ずいぶん +すいようび +すうがく +すうじつ +すうせん +すおどり +すきま +すくう +すくない +すける +すごい +すこし +ずさん +すずしい +すすむ +すすめる +すっかり +ずっしり +ずっと +すてき +すてる +すねる +すのこ +すはだ +すばらしい +ずひょう +ずぶぬれ +すぶり +すふれ +すべて +すべる +ずほう +すぼん +すまい +すめし +すもう +すやき +すらすら +するめ +すれちがう +すろっと +すわる +すんぜん +すんぽう +せあぶら +せいかつ +せいげん +せいじ +せいよう +せおう +せかいかん +せきにん +せきむ +せきゆ +せきらんうん +せけん +せこう +せすじ +せたい +せたけ +せっかく +せっきゃく +ぜっく +せっけん +せっこつ +せっさたくま +せつぞく +せつだん +せつでん +せっぱん +せつび +せつぶん +せつめい +せつりつ +せなか +せのび +せはば +せびろ +せぼね +せまい +せまる +せめる +せもたれ +せりふ +ぜんあく +せんい +せんえい +せんか +せんきょ +せんく +せんげん +ぜんご +せんさい +せんしゅ +せんすい +せんせい +せんぞ +せんたく +せんちょう +せんてい +せんとう +せんぬき +せんねん +せんぱい +ぜんぶ +ぜんぽう +せんむ +せんめんじょ +せんもん +せんやく +せんゆう +せんよう +ぜんら +ぜんりゃく +せんれい +せんろ +そあく +そいとげる +そいね +そうがんきょう +そうき +そうご +そうしん +そうだん +そうなん +そうび +そうめん +そうり +そえもの +そえん +そがい +そげき +そこう +そこそこ +そざい +そしな +そせい +そせん +そそぐ +そだてる +そつう +そつえん +そっかん +そつぎょう +そっけつ +そっこう +そっせん +そっと +そとがわ +そとづら +そなえる +そなた +そふぼ +そぼく +そぼろ +そまつ +そまる +そむく +そむりえ +そめる +そもそも +そよかぜ +そらまめ +そろう +そんかい +そんけい +そんざい +そんしつ +そんぞく +そんちょう +ぞんび +ぞんぶん +そんみん +たあい +たいいん +たいうん +たいえき +たいおう +だいがく +たいき +たいぐう +たいけん +たいこ +たいざい +だいじょうぶ +だいすき +たいせつ +たいそう +だいたい +たいちょう +たいてい +だいどころ +たいない +たいねつ +たいのう +たいはん +だいひょう +たいふう +たいへん +たいほ +たいまつばな +たいみんぐ +たいむ +たいめん +たいやき +たいよう +たいら +たいりょく +たいる +たいわん +たうえ +たえる +たおす +たおる +たおれる +たかい +たかね +たきび +たくさん +たこく +たこやき +たさい +たしざん +だじゃれ +たすける +たずさわる +たそがれ +たたかう +たたく +ただしい +たたみ +たちばな +だっかい +だっきゃく +だっこ +だっしゅつ +だったい +たてる +たとえる +たなばた +たにん +たぬき +たのしみ +たはつ +たぶん +たべる +たぼう +たまご +たまる +だむる +ためいき +ためす +ためる +たもつ +たやすい +たよる +たらす +たりきほんがん +たりょう +たりる +たると +たれる +たれんと +たろっと +たわむれる +だんあつ +たんい +たんおん +たんか +たんき +たんけん +たんご +たんさん +たんじょうび +だんせい +たんそく +たんたい +だんち +たんてい +たんとう +だんな +たんにん +だんねつ +たんのう +たんぴん +だんぼう +たんまつ +たんめい +だんれつ +だんろ +だんわ +ちあい +ちあん +ちいき +ちいさい +ちえん +ちかい +ちから +ちきゅう +ちきん +ちけいず +ちけん +ちこく +ちさい +ちしき +ちしりょう +ちせい +ちそう +ちたい +ちたん +ちちおや +ちつじょ +ちてき +ちてん +ちぬき +ちぬり +ちのう +ちひょう +ちへいせん +ちほう +ちまた +ちみつ +ちみどろ +ちめいど +ちゃんこなべ +ちゅうい +ちゆりょく +ちょうし +ちょさくけん +ちらし +ちらみ +ちりがみ +ちりょう +ちるど +ちわわ +ちんたい +ちんもく +ついか +ついたち +つうか +つうじょう +つうはん +つうわ +つかう +つかれる +つくね +つくる +つけね +つける +つごう +つたえる +つづく +つつじ +つつむ +つとめる +つながる +つなみ +つねづね +つのる +つぶす +つまらない +つまる +つみき +つめたい +つもり +つもる +つよい +つるぼ +つるみく +つわもの +つわり +てあし +てあて +てあみ +ていおん +ていか +ていき +ていけい +ていこく +ていさつ +ていし +ていせい +ていたい +ていど +ていねい +ていひょう +ていへん +ていぼう +てうち +ておくれ +てきとう +てくび +でこぼこ +てさぎょう +てさげ +てすり +てそう +てちがい +てちょう +てつがく +てつづき +でっぱ +てつぼう +てつや +でぬかえ +てぬき +てぬぐい +てのひら +てはい +てぶくろ +てふだ +てほどき +てほん +てまえ +てまきずし +てみじか +てみやげ +てらす +てれび +てわけ +てわたし +でんあつ +てんいん +てんかい +てんき +てんぐ +てんけん +てんごく +てんさい +てんし +てんすう +でんち +てんてき +てんとう +てんない +てんぷら +てんぼうだい +てんめつ +てんらんかい +でんりょく +でんわ +どあい +といれ +どうかん +とうきゅう +どうぐ +とうし +とうむぎ +とおい +とおか +とおく +とおす +とおる +とかい +とかす +ときおり +ときどき +とくい +とくしゅう +とくてん +とくに +とくべつ +とけい +とける +とこや +とさか +としょかん +とそう +とたん +とちゅう +とっきゅう +とっくん +とつぜん +とつにゅう +とどける +ととのえる +とない +となえる +となり +とのさま +とばす +どぶがわ +とほう +とまる +とめる +ともだち +ともる +どようび +とらえる +とんかつ +どんぶり +ないかく +ないこう +ないしょ +ないす +ないせん +ないそう +なおす +ながい +なくす +なげる +なこうど +なさけ +なたでここ +なっとう +なつやすみ +ななおし +なにごと +なにもの +なにわ +なのか +なふだ +なまいき +なまえ +なまみ +なみだ +なめらか +なめる +なやむ +ならう +ならび +ならぶ +なれる +なわとび +なわばり +にあう +にいがた +にうけ +におい +にかい +にがて +にきび +にくしみ +にくまん +にげる +にさんかたんそ +にしき +にせもの +にちじょう +にちようび +にっか +にっき +にっけい +にっこう +にっさん +にっしょく +にっすう +にっせき +にってい +になう +にほん +にまめ +にもつ +にやり +にゅういん +にりんしゃ +にわとり +にんい +にんか +にんき +にんげん +にんしき +にんずう +にんそう +にんたい +にんち +にんてい +にんにく +にんぷ +にんまり +にんむ +にんめい +にんよう +ぬいくぎ +ぬかす +ぬぐいとる +ぬぐう +ぬくもり +ぬすむ +ぬまえび +ぬめり +ぬらす +ぬんちゃく +ねあげ +ねいき +ねいる +ねいろ +ねぐせ +ねくたい +ねくら +ねこぜ +ねこむ +ねさげ +ねすごす +ねそべる +ねだん +ねつい +ねっしん +ねつぞう +ねったいぎょ +ねぶそく +ねふだ +ねぼう +ねほりはほり +ねまき +ねまわし +ねみみ +ねむい +ねむたい +ねもと +ねらう +ねわざ +ねんいり +ねんおし +ねんかん +ねんきん +ねんぐ +ねんざ +ねんし +ねんちゃく +ねんど +ねんぴ +ねんぶつ +ねんまつ +ねんりょう +ねんれい +のいず +のおづま +のがす +のきなみ +のこぎり +のこす +のこる +のせる +のぞく +のぞむ +のたまう +のちほど +のっく +のばす +のはら +のべる +のぼる +のみもの +のやま +のらいぬ +のらねこ +のりもの +のりゆき +のれん +のんき +ばあい +はあく +ばあさん +ばいか +ばいく +はいけん +はいご +はいしん +はいすい +はいせん +はいそう +はいち +ばいばい +はいれつ +はえる +はおる +はかい +ばかり +はかる +はくしゅ +はけん +はこぶ +はさみ +はさん +はしご +ばしょ +はしる +はせる +ぱそこん +はそん +はたん +はちみつ +はつおん +はっかく +はづき +はっきり +はっくつ +はっけん +はっこう +はっさん +はっしん +はったつ +はっちゅう +はってん +はっぴょう +はっぽう +はなす +はなび +はにかむ +はぶらし +はみがき +はむかう +はめつ +はやい +はやし +はらう +はろうぃん +はわい +はんい +はんえい +はんおん +はんかく +はんきょう +ばんぐみ +はんこ +はんしゃ +はんすう +はんだん +ぱんち +ぱんつ +はんてい +はんとし +はんのう +はんぱ +はんぶん +はんぺん +はんぼうき +はんめい +はんらん +はんろん +ひいき +ひうん +ひえる +ひかく +ひかり +ひかる +ひかん +ひくい +ひけつ +ひこうき +ひこく +ひさい +ひさしぶり +ひさん +びじゅつかん +ひしょ diff --git a/src/mnemonics/languages/lojban.txt b/src/mnemonics/languages/lojban.txt new file mode 100644 index 000000000..a8b41c12b --- /dev/null +++ b/src/mnemonics/languages/lojban.txt @@ -0,0 +1,1629 @@ +Lojban +Lojban +4 +backi +bacru +badna +badri +bajra +bakfu +bakni +bakri +baktu +balji +balni +balre +balvi +bambu +bancu +bandu +banfi +bangu +banli +banro +banxa +banzu +bapli +barda +bargu +barja +barna +bartu +basfa +basna +basti +batci +batke +bavmi +baxso +bebna +bekpi +bemro +bende +bengo +benji +benre +benzo +bergu +bersa +berti +besna +besto +betfu +betri +bevri +bidju +bifce +bikla +bilga +bilma +bilni +bindo +binra +binxo +birje +birka +birti +bisli +bitmu +bitni +blabi +blaci +blanu +bliku +bloti +bolci +bongu +boske +botpi +boxfo +boxna +bradi +brano +bratu +brazo +bredi +bridi +brife +briju +brito +brivo +broda +bruna +budjo +bukpu +bumru +bunda +bunre +burcu +burna +cabna +cabra +cacra +cadga +cadzu +cafne +cagna +cakla +calku +calse +canci +cando +cange +canja +canko +canlu +canpa +canre +canti +carce +carfu +carmi +carna +cartu +carvi +casnu +catke +catlu +catni +catra +caxno +cecla +cecmu +cedra +cenba +censa +centi +cerda +cerni +certu +cevni +cfale +cfari +cfika +cfila +cfine +cfipu +ciblu +cicna +cidja +cidni +cidro +cifnu +cigla +cikna +cikre +ciksi +cilce +cilfu +cilmo +cilre +cilta +cimde +cimni +cinba +cindu +cinfo +cinje +cinki +cinla +cinmo +cinri +cinse +cinta +cinza +cipni +cipra +cirko +cirla +ciska +cisma +cisni +ciste +citka +citno +citri +citsi +civla +cizra +ckabu +ckafi +ckaji +ckana +ckape +ckasu +ckeji +ckiku +ckilu +ckini +ckire +ckule +ckunu +cladu +clani +claxu +cletu +clika +clinu +clira +clite +cliva +clupa +cmaci +cmalu +cmana +cmavo +cmene +cmeta +cmevo +cmila +cmima +cmoni +cnano +cnebo +cnemu +cnici +cnino +cnisa +cnita +cokcu +condi +conka +corci +cortu +cpacu +cpana +cpare +cpedu +cpina +cradi +crane +creka +crepu +cribe +crida +crino +cripu +crisa +critu +ctaru +ctebi +cteki +ctile +ctino +ctuca +cukla +cukre +cukta +culno +cumki +cumla +cunmi +cunso +cuntu +cupra +curmi +curnu +curve +cusku +cusna +cutci +cutne +cuxna +dacru +dacti +dadjo +dakfu +dakli +damba +damri +dandu +danfu +danlu +danmo +danre +dansu +danti +daplu +dapma +darca +dargu +darlu +darno +darsi +darxi +daski +dasni +daspo +dasri +datka +datni +datro +decti +degji +dejni +dekpu +dekto +delno +dembi +denci +denmi +denpa +dertu +derxi +desku +detri +dicma +dicra +didni +digno +dikca +diklo +dikni +dilcu +dilma +dilnu +dimna +dindi +dinju +dinko +dinso +dirba +dirce +dirgo +disko +ditcu +divzi +dizlo +djacu +djedi +djica +djine +djuno +donri +dotco +draci +drani +drata +drudi +dugri +dukse +dukti +dunda +dunja +dunku +dunli +dunra +dutso +dzena +dzipo +facki +fadni +fagri +falnu +famti +fancu +fange +fanmo +fanri +fanta +fanva +fanza +fapro +farka +farlu +farna +farvi +fasnu +fatci +fatne +fatri +febvi +fegli +femti +fendi +fengu +fenki +fenra +fenso +fepni +fepri +ferti +festi +fetsi +figre +filso +finpe +finti +firca +fisli +fizbu +flaci +flalu +flani +flecu +flese +fliba +flira +foldi +fonmo +fonxa +forca +forse +fraso +frati +fraxu +frica +friko +frili +frinu +friti +frumu +fukpi +fulta +funca +fusra +fuzme +gacri +gadri +galfi +galtu +galxe +ganlo +ganra +ganse +ganti +ganxo +ganzu +gapci +gapru +garna +gasnu +gaspo +gasta +genja +gento +genxu +gerku +gerna +gidva +gigdo +ginka +girzu +gismu +glare +gleki +gletu +glico +glife +glosa +gluta +gocti +gomsi +gotro +gradu +grafu +grake +grana +grasu +grava +greku +grusi +grute +gubni +gugde +gugle +gumri +gundi +gunka +gunma +gunro +gunse +gunta +gurni +guska +gusni +gusta +gutci +gutra +guzme +jabre +jadni +jakne +jalge +jalna +jalra +jamfu +jamna +janbe +janco +janli +jansu +janta +jarbu +jarco +jarki +jaspu +jatna +javni +jbama +jbari +jbena +jbera +jbini +jdari +jdice +jdika +jdima +jdini +jduli +jecta +jeftu +jegvo +jelca +jemna +jenca +jendu +jenmi +jensi +jerna +jersi +jerxo +jesni +jetce +jetnu +jgalu +jganu +jgari +jgena +jgina +jgira +jgita +jibni +jibri +jicla +jicmu +jijnu +jikca +jikfi +jikni +jikru +jilka +jilra +jimca +jimpe +jimte +jinci +jinda +jinga +jinku +jinme +jinru +jinsa +jinto +jinvi +jinzi +jipci +jipno +jirna +jisra +jitfa +jitro +jivbu +jivna +jmaji +jmifa +jmina +jmive +jonse +jordo +jorne +jubme +judri +jufra +jukni +jukpa +julne +julro +jundi +jungo +junla +junri +junta +jurme +jursa +jutsi +juxre +jvinu +jviso +kabri +kacma +kadno +kafke +kagni +kajde +kajna +kakne +kakpa +kalci +kalri +kalsa +kalte +kamju +kamni +kampu +kamre +kanba +kancu +kandi +kanji +kanla +kanpe +kanro +kansa +kantu +kanxe +karbi +karce +karda +kargu +karli +karni +katci +katna +kavbu +kazra +kecti +kekli +kelci +kelvo +kenka +kenra +kensa +kerfa +kerlo +kesri +ketco +ketsu +kevna +kibro +kicne +kijno +kilto +kinda +kinli +kisto +klaji +klaku +klama +klani +klesi +kliki +klina +kliru +kliti +klupe +kluza +kobli +kogno +kojna +kokso +kolme +komcu +konju +korbi +korcu +korka +korvo +kosmu +kosta +krali +kramu +krasi +krati +krefu +krici +krili +krinu +krixa +kruca +kruji +kruvi +kubli +kucli +kufra +kukte +kulnu +kumfa +kumte +kunra +kunti +kurfa +kurji +kurki +kuspe +kusru +labno +lacni +lacpu +lacri +ladru +lafti +lakne +lakse +laldo +lalxu +lamji +lanbi +lanci +landa +lanka +lanli +lanme +lante +lanxe +lanzu +larcu +larva +lasna +lastu +latmo +latna +lazni +lebna +lelxe +lenga +lenjo +lenku +lerci +lerfu +libjo +lidne +lifri +lijda +limfa +limna +lince +lindi +linga +linji +linsi +linto +lisri +liste +litce +litki +litru +livga +livla +logji +loglo +lojbo +loldi +lorxu +lubno +lujvo +luksi +lumci +lunbe +lunra +lunsa +luska +lusto +mabla +mabru +macnu +majga +makcu +makfa +maksi +malsi +mamta +manci +manfo +mango +manku +manri +mansa +manti +mapku +mapni +mapra +mapti +marbi +marce +marde +margu +marji +marna +marxa +masno +masti +matci +matli +matne +matra +mavji +maxri +mebri +megdo +mekso +melbi +meljo +melmi +menli +menre +mensi +mentu +merko +merli +metfo +mexno +midju +mifra +mikce +mikri +milti +milxe +minde +minji +minli +minra +mintu +mipri +mirli +misno +misro +mitre +mixre +mlana +mlatu +mleca +mledi +mluni +mogle +mokca +moklu +molki +molro +morji +morko +morna +morsi +mosra +mraji +mrilu +mruli +mucti +mudri +mugle +mukti +mulno +munje +mupli +murse +murta +muslo +mutce +muvdu +muzga +nabmi +nakni +nalci +namcu +nanba +nanca +nandu +nanla +nanmu +nanvi +narge +narju +natfe +natmi +natsi +navni +naxle +nazbi +nejni +nelci +nenri +nerde +nibli +nicfa +nicte +nikle +nilce +nimre +ninja +ninmu +nirna +nitcu +nivji +nixli +nobli +norgo +notci +nudle +nukni +nunmu +nupre +nurma +nusna +nutka +nutli +nuzba +nuzlo +pacna +pagbu +pagre +pajni +palci +palku +palma +palne +palpi +palta +pambe +pamga +panci +pandi +panje +panka +panlo +panpi +panra +pante +panzi +papri +parbi +pardu +parji +pastu +patfu +patlu +patxu +paznu +pelji +pelxu +pemci +penbi +pencu +pendo +penmi +pensi +pentu +perli +pesxu +petso +pevna +pezli +picti +pijne +pikci +pikta +pilda +pilji +pilka +pilno +pimlu +pinca +pindi +pinfu +pinji +pinka +pinsi +pinta +pinxe +pipno +pixra +plana +platu +pleji +plibu +plini +plipe +plise +plita +plixa +pluja +pluka +pluta +pocli +polje +polno +ponjo +ponse +poplu +porpi +porsi +porto +prali +prami +prane +preja +prenu +preri +preti +prije +prina +pritu +proga +prosa +pruce +pruni +pruri +pruxi +pulce +pulji +pulni +punji +punli +pupsu +purci +purdi +purmo +racli +ractu +radno +rafsi +ragbi +ragve +rakle +rakso +raktu +ralci +ralju +ralte +randa +rango +ranji +ranmi +ransu +ranti +ranxi +rapli +rarna +ratcu +ratni +rebla +rectu +rekto +remna +renro +renvi +respa +rexsa +ricfu +rigni +rijno +rilti +rimni +rinci +rindo +rinju +rinka +rinsa +rirci +rirni +rirxe +rismi +risna +ritli +rivbi +rokci +romge +romlo +ronte +ropno +rorci +rotsu +rozgu +ruble +rufsu +runme +runta +rupnu +rusko +rutni +sabji +sabnu +sacki +saclu +sadjo +sakci +sakli +sakta +salci +salpo +salri +salta +samcu +sampu +sanbu +sance +sanga +sanji +sanli +sanmi +sanso +santa +sarcu +sarji +sarlu +sarni +sarxe +saske +satci +satre +savru +sazri +sefsi +sefta +sekre +selci +selfu +semto +senci +sengi +senpi +senta +senva +sepli +serti +sesre +setca +sevzi +sfani +sfasa +sfofa +sfubu +sibli +siclu +sicni +sicpi +sidbo +sidju +sigja +sigma +sikta +silka +silna +simlu +simsa +simxu +since +sinma +sinso +sinxa +sipna +sirji +sirxo +sisku +sisti +sitna +sivni +skaci +skami +skapi +skari +skicu +skiji +skina +skori +skoto +skuba +skuro +slabu +slaka +slami +slanu +slari +slasi +sligu +slilu +sliri +slovo +sluji +sluni +smacu +smadi +smaji +smaka +smani +smela +smoka +smuci +smuni +smusu +snada +snanu +snidu +snime +snipa +snuji +snura +snuti +sobde +sodna +sodva +softo +solji +solri +sombo +sonci +sorcu +sorgu +sorni +sorta +sovda +spaji +spali +spano +spati +speni +spero +spisa +spita +spofu +spoja +spuda +sputu +sraji +sraku +sralo +srana +srasu +srera +srito +sruma +sruri +stace +stagi +staku +stali +stani +stapa +stasu +stati +steba +steci +stedu +stela +stero +stici +stidi +stika +stizu +stodi +stuna +stura +stuzi +sucta +sudga +sufti +suksa +sumji +sumne +sumti +sunga +sunla +surla +sutra +tabno +tabra +tadji +tadni +tagji +taksi +talsa +tamca +tamji +tamne +tanbo +tance +tanjo +tanko +tanru +tansi +tanxe +tapla +tarbi +tarci +tarla +tarmi +tarti +taske +tasmi +tasta +tatpi +tatru +tavla +taxfu +tcaci +tcadu +tcana +tcati +tcaxe +tcena +tcese +tcica +tcidu +tcika +tcila +tcima +tcini +tcita +temci +temse +tende +tenfa +tengu +terdi +terpa +terto +tifri +tigni +tigra +tikpa +tilju +tinbe +tinci +tinsa +tirna +tirse +tirxu +tisna +titla +tivni +tixnu +toknu +toldi +tonga +tordu +torni +torso +traji +trano +trati +trene +tricu +trina +trixe +troci +tsaba +tsali +tsani +tsapi +tsiju +tsina +tsuku +tubnu +tubra +tugni +tujli +tumla +tunba +tunka +tunlo +tunta +tuple +turko +turni +tutci +tutle +tutra +vacri +vajni +valsi +vamji +vamtu +vanbi +vanci +vanju +vasru +vasxu +vecnu +vedli +venfu +vensa +vente +vepre +verba +vibna +vidni +vidru +vifne +vikmi +viknu +vimcu +vindu +vinji +vinta +vipsi +virnu +viska +vitci +vitke +vitno +vlagi +vlile +vlina +vlipa +vofli +voksa +volve +vorme +vraga +vreji +vreta +vrici +vrude +vrusi +vubla +vujnu +vukna +vukro +xabju +xadba +xadji +xadni +xagji +xagri +xajmi +xaksu +xalbo +xalka +xalni +xamgu +xampo +xamsi +xance +xango +xanka +xanri +xansa +xanto +xarci +xarju +xarnu +xasli +xasne +xatra +xatsi +xazdo +xebni +xebro +xecto +xedja +xekri +xelso +xendo +xenru +xexso +xigzo +xindo +xinmo +xirma +xislu +xispo +xlali +xlura +xorbo +xorlo +xotli +xrabo +xrani +xriso +xrotu +xruba +xruki +xrula +xruti +xukmi +xulta +xunre +xurdo +xusra +xutla +zabna +zajba +zalvi +zanru +zarci +zargu +zasni +zasti +zbabu +zbani +zbasu +zbepi +zdani +zdile +zekri +zenba +zepti +zetro +zevla +zgadi +zgana +zgike +zifre +zinki +zirpu +zivle +zmadu +zmiku +zucna +zukte +zumri +zungi +zunle +zunti +zutse +zvati +zviki +jbobau +jbopre +karsna +cabdei +zunsna +gendra +glibau +nintadni +pavyseljirna +vlaste +selbri +latro'a +zdakemkulgu'a +mriste +selsku +fu'ivla +tolmo'i +snavei +xagmau +retsku +ckupau +skudji +smudra +prulamdei +vokta'a +tinju'i +jefyfa'o +bavlamdei +kinzga +jbocre +jbovla +xauzma +selkei +xuncku +spusku +jbogu'e +pampe'o +bripre +jbosnu +zi'evla +gimste +tolzdi +velski +samselpla +cnegau +velcki +selja'e +fasybau +zanfri +reisku +favgau +jbota'a +rejgau +malgli +zilkai +keidji +tersu'i +jbofi'e +cnima'o +mulgau +ningau +ponbau +mrobi'o +rarbau +zmanei +famyma'o +vacysai +jetmlu +jbonunsla +nunpe'i +fa'orma'o +crezenzu'e +jbojbe +cmicu'a +zilcmi +tolcando +zukcfu +depybu'i +mencre +matmau +nunctu +selma'o +titnanba +naldra +jvajvo +nunsnu +nerkla +cimjvo +muvgau +zipcpi +runbau +faumlu +terbri +balcu'e +dragau +smuvelcki +piksku +selpli +bregau +zvafa'i +ci'izra +noltruti'u +samtci +snaxa'a diff --git a/src/mnemonics/languages/portuguese.txt b/src/mnemonics/languages/portuguese.txt new file mode 100644 index 000000000..a50b53adf --- /dev/null +++ b/src/mnemonics/languages/portuguese.txt @@ -0,0 +1,1629 @@ +Portuguese +Português +4 +abaular +abdominal +abeto +abissinio +abjeto +ablucao +abnegar +abotoar +abrutalhar +absurdo +abutre +acautelar +accessorios +acetona +achocolatado +acirrar +acne +acovardar +acrostico +actinomicete +acustico +adaptavel +adeus +adivinho +adjunto +admoestar +adnominal +adotivo +adquirir +adriatico +adsorcao +adutora +advogar +aerossol +afazeres +afetuoso +afixo +afluir +afortunar +afrouxar +aftosa +afunilar +agentes +agito +aglutinar +aiatola +aimore +aino +aipo +airoso +ajeitar +ajoelhar +ajudante +ajuste +alazao +albumina +alcunha +alegria +alexandre +alforriar +alguns +alhures +alivio +almoxarife +alotropico +alpiste +alquimista +alsaciano +altura +aluviao +alvura +amazonico +ambulatorio +ametodico +amizades +amniotico +amovivel +amurada +anatomico +ancorar +anexo +anfora +aniversario +anjo +anotar +ansioso +anturio +anuviar +anverso +anzol +aonde +apaziguar +apito +aplicavel +apoteotico +aprimorar +aprumo +apto +apuros +aquoso +arauto +arbusto +arduo +aresta +arfar +arguto +aritmetico +arlequim +armisticio +aromatizar +arpoar +arquivo +arrumar +arsenio +arturiano +aruaque +arvores +asbesto +ascorbico +aspirina +asqueroso +assustar +astuto +atazanar +ativo +atletismo +atmosferico +atormentar +atroz +aturdir +audivel +auferir +augusto +aula +aumento +aurora +autuar +avatar +avexar +avizinhar +avolumar +avulso +axiomatico +azerbaijano +azimute +azoto +azulejo +bacteriologista +badulaque +baforada +baixote +bajular +balzaquiana +bambuzal +banzo +baoba +baqueta +barulho +bastonete +batuta +bauxita +bavaro +bazuca +bcrepuscular +beato +beduino +begonia +behaviorista +beisebol +belzebu +bemol +benzido +beocio +bequer +berro +besuntar +betume +bexiga +bezerro +biatlon +biboca +bicuspide +bidirecional +bienio +bifurcar +bigorna +bijuteria +bimotor +binormal +bioxido +bipolarizacao +biquini +birutice +bisturi +bituca +biunivoco +bivalve +bizarro +blasfemo +blenorreia +blindar +bloqueio +blusao +boazuda +bofete +bojudo +bolso +bombordo +bonzo +botina +boquiaberto +bostoniano +botulismo +bourbon +bovino +boximane +bravura +brevidade +britar +broxar +bruno +bruxuleio +bubonico +bucolico +buda +budista +bueiro +buffer +bugre +bujao +bumerangue +burundines +busto +butique +buzios +caatinga +cabuqui +cacunda +cafuzo +cajueiro +camurca +canudo +caquizeiro +carvoeiro +casulo +catuaba +cauterizar +cebolinha +cedula +ceifeiro +celulose +cerzir +cesto +cetro +ceus +cevar +chavena +cheroqui +chita +chovido +chuvoso +ciatico +cibernetico +cicuta +cidreira +cientistas +cifrar +cigarro +cilio +cimo +cinzento +cioso +cipriota +cirurgico +cisto +citrico +ciumento +civismo +clavicula +clero +clitoris +cluster +coaxial +cobrir +cocota +codorniz +coexistir +cogumelo +coito +colusao +compaixao +comutativo +contentamento +convulsivo +coordenativa +coquetel +correto +corvo +costureiro +cotovia +covil +cozinheiro +cretino +cristo +crivo +crotalo +cruzes +cubo +cucuia +cueiro +cuidar +cujo +cultural +cunilingua +cupula +curvo +custoso +cutucar +czarismo +dablio +dacota +dados +daguerreotipo +daiquiri +daltonismo +damista +dantesco +daquilo +darwinista +dasein +dativo +deao +debutantes +decurso +deduzir +defunto +degustar +dejeto +deltoide +demover +denunciar +deputado +deque +dervixe +desvirtuar +deturpar +deuteronomio +devoto +dextrose +dezoito +diatribe +dicotomico +didatico +dietista +difuso +digressao +diluvio +diminuto +dinheiro +dinossauro +dioxido +diplomatico +dique +dirimivel +disturbio +diurno +divulgar +dizivel +doar +dobro +docura +dodoi +doer +dogue +doloso +domo +donzela +doping +dorsal +dossie +dote +doutro +doze +dravidico +dreno +driver +dropes +druso +dubnio +ducto +dueto +dulija +dundum +duodeno +duquesa +durou +duvidoso +duzia +ebano +ebrio +eburneo +echarpe +eclusa +ecossistema +ectoplasma +ecumenismo +eczema +eden +editorial +edredom +edulcorar +efetuar +efigie +efluvio +egiptologo +egresso +egua +einsteiniano +eira +eivar +eixos +ejetar +elastomero +eldorado +elixir +elmo +eloquente +elucidativo +emaranhar +embutir +emerito +emfa +emitir +emotivo +empuxo +emulsao +enamorar +encurvar +enduro +enevoar +enfurnar +enguico +enho +enigmista +enlutar +enormidade +enpreendimento +enquanto +enriquecer +enrugar +entusiastico +enunciar +envolvimento +enxuto +enzimatico +eolico +epiteto +epoxi +epura +equivoco +erario +erbio +ereto +erguido +erisipela +ermo +erotizar +erros +erupcao +ervilha +esburacar +escutar +esfuziante +esguio +esloveno +esmurrar +esoterismo +esperanca +espirito +espurio +essencialmente +esturricar +esvoacar +etario +eterno +etiquetar +etnologo +etos +etrusco +euclidiano +euforico +eugenico +eunuco +europio +eustaquio +eutanasia +evasivo +eventualidade +evitavel +evoluir +exaustor +excursionista +exercito +exfoliado +exito +exotico +expurgo +exsudar +extrusora +exumar +fabuloso +facultativo +fado +fagulha +faixas +fajuto +faltoso +famoso +fanzine +fapesp +faquir +fartura +fastio +faturista +fausto +favorito +faxineira +fazer +fealdade +febril +fecundo +fedorento +feerico +feixe +felicidade +felpudo +feltro +femur +fenotipo +fervura +festivo +feto +feudo +fevereiro +fezinha +fiasco +fibra +ficticio +fiduciario +fiesp +fifa +figurino +fijiano +filtro +finura +fiorde +fiquei +firula +fissurar +fitoteca +fivela +fixo +flavio +flexor +flibusteiro +flotilha +fluxograma +fobos +foco +fofura +foguista +foie +foliculo +fominha +fonte +forum +fosso +fotossintese +foxtrote +fraudulento +frevo +frivolo +frouxo +frutose +fuba +fucsia +fugitivo +fuinha +fujao +fulustreco +fumo +funileiro +furunculo +fustigar +futurologo +fuxico +fuzue +gabriel +gado +gaelico +gafieira +gaguejo +gaivota +gajo +galvanoplastico +gamo +ganso +garrucha +gastronomo +gatuno +gaussiano +gaviao +gaxeta +gazeteiro +gear +geiser +geminiano +generoso +genuino +geossinclinal +gerundio +gestual +getulista +gibi +gigolo +gilete +ginseng +giroscopio +glaucio +glacial +gleba +glifo +glote +glutonia +gnostico +goela +gogo +goitaca +golpista +gomo +gonzo +gorro +gostou +goticula +gourmet +governo +gozo +graxo +grevista +grito +grotesco +gruta +guaxinim +gude +gueto +guizo +guloso +gume +guru +gustativo +grelhado +gutural +habitue +haitiano +halterofilista +hamburguer +hanseniase +happening +harpista +hastear +haveres +hebreu +hectometro +hedonista +hegira +helena +helminto +hemorroidas +henrique +heptassilabo +hertziano +hesitar +heterossexual +heuristico +hexagono +hiato +hibrido +hidrostatico +hieroglifo +hifenizar +higienizar +hilario +himen +hino +hippie +hirsuto +historiografia +hitlerista +hodometro +hoje +holograma +homus +honroso +hoquei +horto +hostilizar +hotentote +huguenote +humilde +huno +hurra +hutu +iaia +ialorixa +iambico +iansa +iaque +iara +iatista +iberico +ibis +icar +iceberg +icosagono +idade +ideologo +idiotice +idoso +iemenita +iene +igarape +iglu +ignorar +igreja +iguaria +iidiche +ilativo +iletrado +ilharga +ilimitado +ilogismo +ilustrissimo +imaturo +imbuzeiro +imerso +imitavel +imovel +imputar +imutavel +inaveriguavel +incutir +induzir +inextricavel +infusao +ingua +inhame +iniquo +injusto +inning +inoxidavel +inquisitorial +insustentavel +intumescimento +inutilizavel +invulneravel +inzoneiro +iodo +iogurte +ioio +ionosfera +ioruba +iota +ipsilon +irascivel +iris +irlandes +irmaos +iroques +irrupcao +isca +isento +islandes +isotopo +isqueiro +israelita +isso +isto +iterbio +itinerario +itrio +iuane +iugoslavo +jabuticabeira +jacutinga +jade +jagunco +jainista +jaleco +jambo +jantarada +japones +jaqueta +jarro +jasmim +jato +jaula +javel +jazz +jegue +jeitoso +jejum +jenipapo +jeova +jequitiba +jersei +jesus +jetom +jiboia +jihad +jilo +jingle +jipe +jocoso +joelho +joguete +joio +jojoba +jorro +jota +joule +joviano +jubiloso +judoca +jugular +juizo +jujuba +juliano +jumento +junto +jururu +justo +juta +juventude +labutar +laguna +laico +lajota +lanterninha +lapso +laquear +lastro +lauto +lavrar +laxativo +lazer +leasing +lebre +lecionar +ledo +leguminoso +leitura +lele +lemure +lento +leonardo +leopardo +lepton +leque +leste +letreiro +leucocito +levitico +lexicologo +lhama +lhufas +liame +licoroso +lidocaina +liliputiano +limusine +linotipo +lipoproteina +liquidos +lirismo +lisura +liturgico +livros +lixo +lobulo +locutor +lodo +logro +lojista +lombriga +lontra +loop +loquaz +lorota +losango +lotus +louvor +luar +lubrificavel +lucros +lugubre +luis +luminoso +luneta +lustroso +luto +luvas +luxuriante +luzeiro +maduro +maestro +mafioso +magro +maiuscula +majoritario +malvisto +mamute +manutencao +mapoteca +maquinista +marzipa +masturbar +matuto +mausoleu +mavioso +maxixe +mazurca +meandro +mecha +medusa +mefistofelico +megera +meirinho +melro +memorizar +menu +mequetrefe +mertiolate +mestria +metroviario +mexilhao +mezanino +miau +microssegundo +midia +migratorio +mimosa +minuto +miosotis +mirtilo +misturar +mitzvah +miudos +mixuruca +mnemonico +moagem +mobilizar +modulo +moer +mofo +mogno +moita +molusco +monumento +moqueca +morubixaba +mostruario +motriz +mouse +movivel +mozarela +muarra +muculmano +mudo +mugir +muitos +mumunha +munir +muon +muquira +murros +musselina +nacoes +nado +naftalina +nago +naipe +naja +nalgum +namoro +nanquim +napolitano +naquilo +nascimento +nautilo +navios +nazista +nebuloso +nectarina +nefrologo +negus +nelore +nenufar +nepotismo +nervura +neste +netuno +neutron +nevoeiro +newtoniano +nexo +nhenhenhem +nhoque +nigeriano +niilista +ninho +niobio +niponico +niquelar +nirvana +nisto +nitroglicerina +nivoso +nobreza +nocivo +noel +nogueira +noivo +nojo +nominativo +nonuplo +noruegues +nostalgico +noturno +nouveau +nuanca +nublar +nucleotideo +nudista +nulo +numismatico +nunquinha +nupcias +nutritivo +nuvens +oasis +obcecar +obeso +obituario +objetos +oblongo +obnoxio +obrigatorio +obstruir +obtuso +obus +obvio +ocaso +occipital +oceanografo +ocioso +oclusivo +ocorrer +ocre +octogono +odalisca +odisseia +odorifico +oersted +oeste +ofertar +ofidio +oftalmologo +ogiva +ogum +oigale +oitavo +oitocentos +ojeriza +olaria +oleoso +olfato +olhos +oliveira +olmo +olor +olvidavel +ombudsman +omeleteira +omitir +omoplata +onanismo +ondular +oneroso +onomatopeico +ontologico +onus +onze +opalescente +opcional +operistico +opio +oposto +oprobrio +optometrista +opusculo +oratorio +orbital +orcar +orfao +orixa +orla +ornitologo +orquidea +ortorrombico +orvalho +osculo +osmotico +ossudo +ostrogodo +otario +otite +ouro +ousar +outubro +ouvir +ovario +overnight +oviparo +ovni +ovoviviparo +ovulo +oxala +oxente +oxiuro +oxossi +ozonizar +paciente +pactuar +padronizar +paete +pagodeiro +paixao +pajem +paludismo +pampas +panturrilha +papudo +paquistanes +pastoso +patua +paulo +pauzinhos +pavoroso +paxa +pazes +peao +pecuniario +pedunculo +pegaso +peixinho +pejorativo +pelvis +penuria +pequno +petunia +pezada +piauiense +pictorico +pierro +pigmeu +pijama +pilulas +pimpolho +pintura +piorar +pipocar +piqueteiro +pirulito +pistoleiro +pituitaria +pivotar +pixote +pizzaria +plistoceno +plotar +pluviometrico +pneumonico +poco +podridao +poetisa +pogrom +pois +polvorosa +pomposo +ponderado +pontudo +populoso +poquer +porvir +posudo +potro +pouso +povoar +prazo +prezar +privilegios +proximo +prussiano +pseudopode +psoriase +pterossauros +ptialina +ptolemaico +pudor +pueril +pufe +pugilista +puir +pujante +pulverizar +pumba +punk +purulento +pustula +putsch +puxe +quatrocentos +quetzal +quixotesco +quotizavel +rabujice +racista +radonio +rafia +ragu +rajado +ralo +rampeiro +ranzinza +raptor +raquitismo +raro +rasurar +ratoeira +ravioli +razoavel +reavivar +rebuscar +recusavel +reduzivel +reexposicao +refutavel +regurgitar +reivindicavel +rejuvenescimento +relva +remuneravel +renunciar +reorientar +repuxo +requisito +resumo +returno +reutilizar +revolvido +rezonear +riacho +ribossomo +ricota +ridiculo +rifle +rigoroso +rijo +rimel +rins +rios +riqueza +respeito +rissole +ritualistico +rivalizar +rixa +robusto +rococo +rodoviario +roer +rogo +rojao +rolo +rompimento +ronronar +roqueiro +rorqual +rosto +rotundo +rouxinol +roxo +royal +ruas +rucula +rudimentos +ruela +rufo +rugoso +ruivo +rule +rumoroso +runico +ruptura +rural +rustico +rutilar +saariano +sabujo +sacudir +sadomasoquista +safra +sagui +sais +samurai +santuario +sapo +saquear +sartriano +saturno +saude +sauva +saveiro +saxofonista +sazonal +scherzo +script +seara +seborreia +secura +seduzir +sefardim +seguro +seja +selvas +sempre +senzala +sepultura +sequoia +sestercio +setuplo +seus +seviciar +sezonismo +shalom +siames +sibilante +sicrano +sidra +sifilitico +signos +silvo +simultaneo +sinusite +sionista +sirio +sisudo +situar +sivan +slide +slogan +soar +sobrio +socratico +sodomizar +soerguer +software +sogro +soja +solver +somente +sonso +sopro +soquete +sorveteiro +sossego +soturno +sousafone +sovinice +sozinho +suavizar +subverter +sucursal +sudoriparo +sufragio +sugestoes +suite +sujo +sultao +sumula +suntuoso +suor +supurar +suruba +susto +suturar +suvenir +tabuleta +taco +tadjique +tafeta +tagarelice +taitiano +talvez +tampouco +tanzaniano +taoista +tapume +taquion +tarugo +tascar +tatuar +tautologico +tavola +taxionomista +tchecoslovaco +teatrologo +tectonismo +tedioso +teflon +tegumento +teixo +telurio +temporas +tenue +teosofico +tepido +tequila +terrorista +testosterona +tetrico +teutonico +teve +texugo +tiara +tibia +tiete +tifoide +tigresa +tijolo +tilintar +timpano +tintureiro +tiquete +tiroteio +tisico +titulos +tive +toar +toboga +tofu +togoles +toicinho +tolueno +tomografo +tontura +toponimo +toquio +torvelinho +tostar +toto +touro +toxina +trazer +trezentos +trivialidade +trovoar +truta +tuaregue +tubular +tucano +tudo +tufo +tuiste +tulipa +tumultuoso +tunisino +tupiniquim +turvo +tutu +ucraniano +udenista +ufanista +ufologo +ugaritico +uiste +uivo +ulceroso +ulema +ultravioleta +umbilical +umero +umido +umlaut +unanimidade +unesco +ungulado +unheiro +univoco +untuoso +urano +urbano +urdir +uretra +urgente +urinol +urna +urologo +urro +ursulina +urtiga +urupe +usavel +usbeque +usei +usineiro +usurpar +utero +utilizar +utopico +uvular +uxoricidio +vacuo +vadio +vaguear +vaivem +valvula +vampiro +vantajoso +vaporoso +vaquinha +varziano +vasto +vaticinio +vaudeville +vazio +veado +vedico +veemente +vegetativo +veio +veja +veludo +venusiano +verdade +verve +vestuario +vetusto +vexatorio +vezes +viavel +vibratorio +victor +vicunha +vidros +vietnamita +vigoroso +vilipendiar +vime +vintem +violoncelo +viquingue +virus +visualizar +vituperio +viuvo +vivo +vizir +voar +vociferar +vodu +vogar +voile +volver +vomito +vontade +vortice +vosso +voto +vovozinha +voyeuse +vozes +vulva +vupt +western +xadrez +xale +xampu +xango +xarope +xaual +xavante +xaxim +xenonio +xepa +xerox +xicara +xifopago +xiita +xilogravura +xinxim +xistoso +xixi +xodo +xogum +xucro +zabumba +zagueiro +zambiano +zanzar +zarpar +zebu +zefiro +zeloso +zenite +zumbi diff --git a/src/mnemonics/languages/russian.txt b/src/mnemonics/languages/russian.txt new file mode 100644 index 000000000..18abd4a1b --- /dev/null +++ b/src/mnemonics/languages/russian.txt @@ -0,0 +1,1629 @@ +Russian +русский язык +3 +абажур +абзац +абонент +абрикос +абсурд +авангард +август +авиация +авоська +автор +агат +агент +агитатор +агнец +агония +агрегат +адвокат +адмирал +адрес +ажиотаж +азарт +азбука +азот +аист +айсберг +академия +аквариум +аккорд +акробат +аксиома +актер +акула +акция +алгоритм +алебарда +аллея +алмаз +алтарь +алфавит +алхимик +алый +альбом +алюминий +амбар +аметист +амнезия +ампула +амфора +анализ +ангел +анекдот +анимация +анкета +аномалия +ансамбль +антенна +апатия +апельсин +апофеоз +аппарат +апрель +аптека +арабский +арбуз +аргумент +арест +ария +арка +армия +аромат +арсенал +артист +архив +аршин +асбест +аскетизм +аспект +ассорти +астроном +асфальт +атака +ателье +атлас +атом +атрибут +аудитор +аукцион +аура +афера +афиша +ахинея +ацетон +аэропорт +бабушка +багаж +бадья +база +баклажан +балкон +бампер +банк +барон +бассейн +батарея +бахрома +башня +баян +бегство +бедро +бездна +бекон +белый +бензин +берег +беседа +бетонный +биатлон +библия +бивень +бигуди +бидон +бизнес +бикини +билет +бинокль +биология +биржа +бисер +битва +бицепс +благо +бледный +близкий +блок +блуждать +блюдо +бляха +бобер +богатый +бодрый +боевой +бокал +большой +борьба +босой +ботинок +боцман +бочка +боярин +брать +бревно +бригада +бросать +брызги +брюки +бублик +бугор +будущее +буква +бульвар +бумага +бунт +бурный +бусы +бутылка +буфет +бухта +бушлат +бывалый +быль +быстрый +быть +бюджет +бюро +бюст +вагон +важный +ваза +вакцина +валюта +вампир +ванная +вариант +вассал +вата +вафля +вахта +вдова +вдыхать +ведущий +веер +вежливый +везти +веко +великий +вена +верить +веселый +ветер +вечер +вешать +вещь +веяние +взаимный +взбучка +взвод +взгляд +вздыхать +взлетать +взмах +взнос +взор +взрыв +взывать +взятка +вибрация +визит +вилка +вино +вирус +висеть +витрина +вихрь +вишневый +включать +вкус +власть +влечь +влияние +влюблять +внешний +внимание +внук +внятный +вода +воевать +вождь +воздух +войти +вокзал +волос +вопрос +ворота +восток +впадать +впускать +врач +время +вручать +всадник +всеобщий +вспышка +встреча +вторник +вулкан +вурдалак +входить +въезд +выбор +вывод +выгодный +выделять +выезжать +выживать +вызывать +выигрыш +вылезать +выносить +выпивать +высокий +выходить +вычет +вышка +выяснять +вязать +вялый +гавань +гадать +газета +гаишник +галстук +гамма +гарантия +гастроли +гвардия +гвоздь +гектар +гель +генерал +геолог +герой +гешефт +гибель +гигант +гильза +гимн +гипотеза +гитара +глаз +глина +глоток +глубокий +глыба +глядеть +гнать +гнев +гнить +гном +гнуть +говорить +годовой +голова +гонка +город +гость +готовый +граница +грех +гриб +громкий +группа +грызть +грязный +губа +гудеть +гулять +гуманный +густой +гуща +давать +далекий +дама +данные +дарить +дать +дача +дверь +движение +двор +дебют +девушка +дедушка +дежурный +дезертир +действие +декабрь +дело +демократ +день +депутат +держать +десяток +детский +дефицит +дешевый +деятель +джаз +джинсы +джунгли +диалог +диван +диета +дизайн +дикий +динамика +диплом +директор +диск +дитя +дичь +длинный +дневник +добрый +доверие +договор +дождь +доза +документ +должен +домашний +допрос +дорога +доход +доцент +дочь +дощатый +драка +древний +дрожать +друг +дрянь +дубовый +дуга +дудка +дукат +дуло +думать +дупло +дурак +дуть +духи +душа +дуэт +дымить +дыня +дыра +дыханье +дышать +дьявол +дюжина +дюйм +дюна +дядя +дятел +егерь +единый +едкий +ежевика +ежик +езда +елка +емкость +ерунда +ехать +жадный +жажда +жалеть +жанр +жара +жать +жгучий +ждать +жевать +желание +жемчуг +женщина +жертва +жесткий +жечь +живой +жидкость +жизнь +жилье +жирный +житель +журнал +жюри +забывать +завод +загадка +задача +зажечь +зайти +закон +замечать +занимать +западный +зарплата +засыпать +затрата +захват +зацепка +зачет +защита +заявка +звать +звезда +звонить +звук +здание +здешний +здоровье +зебра +зевать +зеленый +земля +зенит +зеркало +зефир +зигзаг +зима +зиять +злак +злой +змея +знать +зной +зодчий +золотой +зомби +зона +зоопарк +зоркий +зрачок +зрение +зритель +зубной +зыбкий +зять +игла +иголка +играть +идея +идиот +идол +идти +иерархия +избрать +известие +изгонять +издание +излагать +изменять +износ +изоляция +изрядный +изучать +изымать +изящный +икона +икра +иллюзия +имбирь +иметь +имидж +иммунный +империя +инвестор +индивид +инерция +инженер +иномарка +институт +интерес +инфекция +инцидент +ипподром +ирис +ирония +искать +история +исходить +исчезать +итог +июль +июнь +кабинет +кавалер +кадр +казарма +кайф +кактус +калитка +камень +канал +капитан +картина +касса +катер +кафе +качество +каша +каюта +квартира +квинтет +квота +кедр +кекс +кенгуру +кепка +керосин +кетчуп +кефир +кибитка +кивнуть +кидать +километр +кино +киоск +кипеть +кирпич +кисть +китаец +класс +клетка +клиент +клоун +клуб +клык +ключ +клятва +книга +кнопка +кнут +князь +кобура +ковер +коготь +кодекс +кожа +козел +койка +коктейль +колено +компания +конец +копейка +короткий +костюм +котел +кофе +кошка +красный +кресло +кричать +кровь +крупный +крыша +крючок +кубок +кувшин +кудрявый +кузов +кукла +культура +кумир +купить +курс +кусок +кухня +куча +кушать +кювет +лабиринт +лавка +лагерь +ладонь +лазерный +лайнер +лакей +лампа +ландшафт +лапа +ларек +ласковый +лауреат +лачуга +лаять +лгать +лебедь +левый +легкий +ледяной +лежать +лекция +лента +лепесток +лесной +лето +лечь +леший +лживый +либерал +ливень +лига +лидер +ликовать +лиловый +лимон +линия +липа +лирика +лист +литр +лифт +лихой +лицо +личный +лишний +лобовой +ловить +логика +лодка +ложка +лозунг +локоть +ломать +лоно +лопата +лорд +лось +лоток +лохматый +лошадь +лужа +лукавый +луна +лупить +лучший +лыжный +лысый +львиный +льгота +льдина +любить +людской +люстра +лютый +лягушка +магазин +мадам +мазать +майор +максимум +мальчик +манера +март +масса +мать +мафия +махать +мачта +машина +маэстро +маяк +мгла +мебель +медведь +мелкий +мемуары +менять +мера +место +метод +механизм +мечтать +мешать +миграция +мизинец +микрофон +миллион +минута +мировой +миссия +митинг +мишень +младший +мнение +мнимый +могила +модель +мозг +мойка +мокрый +молодой +момент +монах +море +мост +мотор +мохнатый +мочь +мошенник +мощный +мрачный +мстить +мудрый +мужчина +музыка +мука +мумия +мундир +муравей +мусор +мутный +муфта +муха +мучить +мушкетер +мыло +мысль +мыть +мычать +мышь +мэтр +мюзикл +мягкий +мякиш +мясо +мятый +мячик +набор +навык +нагрузка +надежда +наемный +нажать +называть +наивный +накрыть +налог +намерен +наносить +написать +народ +натура +наука +нация +начать +небо +невеста +негодяй +неделя +нежный +незнание +нелепый +немалый +неправда +нервный +нести +нефть +нехватка +нечистый +неясный +нива +нижний +низкий +никель +нирвана +нить +ничья +ниша +нищий +новый +нога +ножницы +ноздря +ноль +номер +норма +нота +ночь +ноша +ноябрь +нрав +нужный +нутро +нынешний +нырнуть +ныть +нюанс +нюхать +няня +оазис +обаяние +обвинять +обгонять +обещать +обжигать +обзор +обида +область +обмен +обнимать +оборона +образ +обучение +обходить +обширный +общий +объект +обычный +обязать +овальный +овес +овощи +овраг +овца +овчарка +огненный +огонь +огромный +огурец +одежда +одинокий +одобрить +ожидать +ожог +озарение +озеро +означать +оказать +океан +оклад +окно +округ +октябрь +окурок +олень +опасный +операция +описать +оплата +опора +оппонент +опрос +оптимизм +опускать +опыт +орать +орбита +орган +орден +орел +оригинал +оркестр +орнамент +оружие +осадок +освещать +осень +осина +осколок +осмотр +основной +особый +осуждать +отбор +отвечать +отдать +отец +отзыв +открытие +отмечать +относить +отпуск +отрасль +отставка +оттенок +отходить +отчет +отъезд +офицер +охапка +охота +охрана +оценка +очаг +очередь +очищать +очки +ошейник +ошибка +ощущение +павильон +падать +паек +пакет +палец +память +панель +папка +партия +паспорт +патрон +пауза +пафос +пахнуть +пациент +пачка +пашня +певец +педагог +пейзаж +пельмень +пенсия +пепел +период +песня +петля +пехота +печать +пешеход +пещера +пианист +пиво +пиджак +пиковый +пилот +пионер +пирог +писать +пить +пицца +пишущий +пища +план +плечо +плита +плохой +плыть +плюс +пляж +победа +повод +погода +подумать +поехать +пожимать +позиция +поиск +покой +получать +помнить +пони +поощрять +попадать +порядок +пост +поток +похожий +поцелуй +почва +пощечина +поэт +пояснить +право +предмет +проблема +пруд +прыгать +прямой +психолог +птица +публика +пугать +пудра +пузырь +пуля +пункт +пурга +пустой +путь +пухлый +пучок +пушистый +пчела +пшеница +пыль +пытка +пыхтеть +пышный +пьеса +пьяный +пятно +работа +равный +радость +развитие +район +ракета +рамка +ранний +рапорт +рассказ +раунд +рация +рвать +реальный +ребенок +реветь +регион +редакция +реестр +режим +резкий +рейтинг +река +религия +ремонт +рента +реплика +ресурс +реформа +рецепт +речь +решение +ржавый +рисунок +ритм +рифма +робкий +ровный +рогатый +родитель +рождение +розовый +роковой +роль +роман +ронять +рост +рота +роща +рояль +рубль +ругать +руда +ружье +руины +рука +руль +румяный +русский +ручка +рыба +рывок +рыдать +рыжий +рынок +рысь +рыть +рыхлый +рыцарь +рычаг +рюкзак +рюмка +рябой +рядовой +сабля +садовый +сажать +салон +самолет +сани +сапог +сарай +сатира +сауна +сахар +сбегать +сбивать +сбор +сбыт +свадьба +свет +свидание +свобода +связь +сгорать +сдвигать +сеанс +северный +сегмент +седой +сезон +сейф +секунда +сельский +семья +сентябрь +сердце +сеть +сечение +сеять +сигнал +сидеть +сизый +сила +символ +синий +сирота +система +ситуация +сиять +сказать +скважина +скелет +скидка +склад +скорый +скрывать +скучный +слава +слеза +слияние +слово +случай +слышать +слюна +смех +смирение +смотреть +смутный +смысл +смятение +снаряд +снег +снижение +сносить +снять +событие +совет +согласие +сожалеть +сойти +сокол +солнце +сомнение +сонный +сообщать +соперник +сорт +состав +сотня +соус +социолог +сочинять +союз +спать +спешить +спина +сплошной +способ +спутник +средство +срок +срывать +стать +ствол +стена +стихи +сторона +страна +студент +стыд +субъект +сувенир +сугроб +судьба +суета +суждение +сукно +сулить +сумма +сунуть +супруг +суровый +сустав +суть +сухой +суша +существо +сфера +схема +сцена +счастье +счет +считать +сшивать +съезд +сынок +сыпать +сырье +сытый +сыщик +сюжет +сюрприз +таблица +таежный +таинство +тайна +такси +талант +таможня +танец +тарелка +таскать +тахта +тачка +таять +тварь +твердый +творить +театр +тезис +текст +тело +тема +тень +теория +теплый +терять +тесный +тетя +техника +течение +тигр +типичный +тираж +титул +тихий +тишина +ткань +товарищ +толпа +тонкий +топливо +торговля +тоска +точка +тощий +традиция +тревога +трибуна +трогать +труд +трюк +тряпка +туалет +тугой +туловище +туман +тундра +тупой +турнир +тусклый +туфля +туча +туша +тыкать +тысяча +тьма +тюльпан +тюрьма +тяга +тяжелый +тянуть +убеждать +убирать +убогий +убыток +уважение +уверять +увлекать +угнать +угол +угроза +удар +удивлять +удобный +уезд +ужас +ужин +узел +узкий +узнавать +узор +уйма +уклон +укол +уксус +улетать +улица +улучшать +улыбка +уметь +умиление +умный +умолять +умысел +унижать +уносить +уныние +упасть +уплата +упор +упрекать +упускать +уран +урна +уровень +усадьба +усердие +усилие +ускорять +условие +усмешка +уснуть +успеть +усыпать +утешать +утка +уточнять +утро +утюг +уходить +уцелеть +участие +ученый +учитель +ушко +ущерб +уютный +уяснять +фабрика +фаворит +фаза +файл +факт +фамилия +фантазия +фара +фасад +февраль +фельдшер +феномен +ферма +фигура +физика +фильм +финал +фирма +фишка +флаг +флейта +флот +фокус +фольклор +фонд +форма +фото +фраза +фреска +фронт +фрукт +функция +фуражка +футбол +фыркать +халат +хамство +хаос +характер +хата +хватать +хвост +хижина +хилый +химия +хирург +хитрый +хищник +хлам +хлеб +хлопать +хмурый +ходить +хозяин +хоккей +холодный +хороший +хотеть +хохотать +храм +хрен +хриплый +хроника +хрупкий +художник +хулиган +хутор +царь +цвет +цель +цемент +центр +цепь +церковь +цикл +цилиндр +циничный +цирк +цистерна +цитата +цифра +цыпленок +чадо +чайник +часть +чашка +человек +чемодан +чепуха +черный +честь +четкий +чехол +чиновник +число +читать +членство +чреватый +чтение +чувство +чугунный +чудо +чужой +чукча +чулок +чума +чуткий +чучело +чушь +шаблон +шагать +шайка +шакал +шалаш +шампунь +шанс +шапка +шарик +шасси +шатер +шахта +шашлык +швейный +швырять +шевелить +шедевр +шейка +шелковый +шептать +шерсть +шестерка +шикарный +шинель +шипеть +широкий +шить +шишка +шкаф +школа +шкура +шланг +шлем +шлюпка +шляпа +шнур +шоколад +шорох +шоссе +шофер +шпага +шпион +шприц +шрам +шрифт +штаб +штора +штраф +штука +штык +шуба +шуметь +шуршать +шутка +щадить +щедрый +щека +щель +щенок +щепка +щетка +щука +эволюция +эгоизм +экзамен +экипаж +экономия +экран +эксперт +элемент +элита +эмблема +эмигрант +эмоция +энергия +эпизод +эпоха +эскиз +эссе +эстрада +этап +этика +этюд +эфир +эффект +эшелон +юбилей +юбка +южный +юмор +юноша +юрист +яблоко +явление +ягода +ядерный +ядовитый +ядро +язва +язык +яйцо +якорь +январь +японец +яркий +ярмарка +ярость +ярус +ясный +яхта +ячейка +ящик diff --git a/src/mnemonics/languages/spanish.txt b/src/mnemonics/languages/spanish.txt new file mode 100644 index 000000000..a5891ebb6 --- /dev/null +++ b/src/mnemonics/languages/spanish.txt @@ -0,0 +1,1629 @@ +Spanish +Español +4 +ábaco +abdomen +abeja +abierto +abogado +abono +aborto +abrazo +abrir +abuelo +abuso +acabar +academia +acceso +acción +aceite +acelga +acento +aceptar +ácido +aclarar +acné +acoger +acoso +activo +acto +actriz +actuar +acudir +acuerdo +acusar +adicto +admitir +adoptar +adorno +aduana +adulto +aéreo +afectar +afición +afinar +afirmar +ágil +agitar +agonía +agosto +agotar +agregar +agrio +agua +agudo +águila +aguja +ahogo +ahorro +aire +aislar +ajedrez +ajeno +ajuste +alacrán +alambre +alarma +alba +álbum +alcalde +aldea +alegre +alejar +alerta +aleta +alfiler +alga +algodón +aliado +aliento +alivio +alma +almeja +almíbar +altar +alteza +altivo +alto +altura +alumno +alzar +amable +amante +amapola +amargo +amasar +ámbar +ámbito +ameno +amigo +amistad +amor +amparo +amplio +ancho +anciano +ancla +andar +andén +anemia +ángulo +anillo +ánimo +anís +anotar +antena +antiguo +antojo +anual +anular +anuncio +añadir +añejo +año +apagar +aparato +apetito +apio +aplicar +apodo +aporte +apoyo +aprender +aprobar +apuesta +apuro +arado +araña +arar +árbitro +árbol +arbusto +archivo +arco +arder +ardilla +arduo +área +árido +aries +armonía +arnés +aroma +arpa +arpón +arreglo +arroz +arruga +arte +artista +asa +asado +asalto +ascenso +asegurar +aseo +asesor +asiento +asilo +asistir +asno +asombro +áspero +astilla +astro +astuto +asumir +asunto +atajo +ataque +atar +atento +ateo +ático +atleta +átomo +atraer +atroz +atún +audaz +audio +auge +aula +aumento +ausente +autor +aval +avance +avaro +ave +avellana +avena +avestruz +avión +aviso +ayer +ayuda +ayuno +azafrán +azar +azote +azúcar +azufre +azul +baba +babor +bache +bahía +baile +bajar +balanza +balcón +balde +bambú +banco +banda +baño +barba +barco +barniz +barro +báscula +bastón +basura +batalla +batería +batir +batuta +baúl +bazar +bebé +bebida +bello +besar +beso +bestia +bicho +bien +bingo +blanco +bloque +blusa +boa +bobina +bobo +boca +bocina +boda +bodega +boina +bola +bolero +bolsa +bomba +bondad +bonito +bono +bonsái +borde +borrar +bosque +bote +botín +bóveda +bozal +bravo +brazo +brecha +breve +brillo +brinco +brisa +broca +broma +bronce +brote +bruja +brusco +bruto +buceo +bucle +bueno +buey +bufanda +bufón +búho +buitre +bulto +burbuja +burla +burro +buscar +butaca +buzón +caballo +cabeza +cabina +cabra +cacao +cadáver +cadena +caer +café +caída +caimán +caja +cajón +cal +calamar +calcio +caldo +calidad +calle +calma +calor +calvo +cama +cambio +camello +camino +campo +cáncer +candil +canela +canguro +canica +canto +caña +cañón +caoba +caos +capaz +capitán +capote +captar +capucha +cara +carbón +cárcel +careta +carga +cariño +carne +carpeta +carro +carta +casa +casco +casero +caspa +castor +catorce +catre +caudal +causa +cazo +cebolla +ceder +cedro +celda +célebre +celoso +célula +cemento +ceniza +centro +cerca +cerdo +cereza +cero +cerrar +certeza +césped +cetro +chacal +chaleco +champú +chancla +chapa +charla +chico +chiste +chivo +choque +choza +chuleta +chupar +ciclón +ciego +cielo +cien +cierto +cifra +cigarro +cima +cinco +cine +cinta +ciprés +circo +ciruela +cisne +cita +ciudad +clamor +clan +claro +clase +clave +cliente +clima +clínica +cobre +cocción +cochino +cocina +coco +código +codo +cofre +coger +cohete +cojín +cojo +cola +colcha +colegio +colgar +colina +collar +colmo +columna +combate +comer +comida +cómodo +compra +conde +conejo +conga +conocer +consejo +contar +copa +copia +corazón +corbata +corcho +cordón +corona +correr +coser +cosmos +costa +cráneo +cráter +crear +crecer +creído +crema +cría +crimen +cripta +crisis +cromo +crónica +croqueta +crudo +cruz +cuadro +cuarto +cuatro +cubo +cubrir +cuchara +cuello +cuento +cuerda +cuesta +cueva +cuidar +culebra +culpa +culto +cumbre +cumplir +cuna +cuneta +cuota +cupón +cúpula +curar +curioso +curso +curva +cutis +dama +danza +dar +dardo +dátil +deber +débil +década +decir +dedo +defensa +definir +dejar +delfín +delgado +delito +demora +denso +dental +deporte +derecho +derrota +desayuno +deseo +desfile +desnudo +destino +desvío +detalle +detener +deuda +día +diablo +diadema +diamante +diana +diario +dibujo +dictar +diente +dieta +diez +difícil +digno +dilema +diluir +dinero +directo +dirigir +disco +diseño +disfraz +diva +divino +doble +doce +dolor +domingo +don +donar +dorado +dormir +dorso +dos +dosis +dragón +droga +ducha +duda +duelo +dueño +dulce +dúo +duque +durar +dureza +duro +ébano +ebrio +echar +eco +ecuador +edad +edición +edificio +editor +educar +efecto +eficaz +eje +ejemplo +elefante +elegir +elemento +elevar +elipse +élite +elixir +elogio +eludir +embudo +emitir +emoción +empate +empeño +empleo +empresa +enano +encargo +enchufe +encía +enemigo +enero +enfado +enfermo +engaño +enigma +enlace +enorme +enredo +ensayo +enseñar +entero +entrar +envase +envío +época +equipo +erizo +escala +escena +escolar +escribir +escudo +esencia +esfera +esfuerzo +espada +espejo +espía +esposa +espuma +esquí +estar +este +estilo +estufa +etapa +eterno +ética +etnia +evadir +evaluar +evento +evitar +exacto +examen +exceso +excusa +exento +exigir +exilio +existir +éxito +experto +explicar +exponer +extremo +fábrica +fábula +fachada +fácil +factor +faena +faja +falda +fallo +falso +faltar +fama +familia +famoso +faraón +farmacia +farol +farsa +fase +fatiga +fauna +favor +fax +febrero +fecha +feliz +feo +feria +feroz +fértil +fervor +festín +fiable +fianza +fiar +fibra +ficción +ficha +fideo +fiebre +fiel +fiera +fiesta +figura +fijar +fijo +fila +filete +filial +filtro +fin +finca +fingir +finito +firma +flaco +flauta +flecha +flor +flota +fluir +flujo +flúor +fobia +foca +fogata +fogón +folio +folleto +fondo +forma +forro +fortuna +forzar +fosa +foto +fracaso +frágil +franja +frase +fraude +freír +freno +fresa +frío +frito +fruta +fuego +fuente +fuerza +fuga +fumar +función +funda +furgón +furia +fusil +fútbol +futuro +gacela +gafas +gaita +gajo +gala +galería +gallo +gamba +ganar +gancho +ganga +ganso +garaje +garza +gasolina +gastar +gato +gavilán +gemelo +gemir +gen +género +genio +gente +geranio +gerente +germen +gesto +gigante +gimnasio +girar +giro +glaciar +globo +gloria +gol +golfo +goloso +golpe +goma +gordo +gorila +gorra +gota +goteo +gozar +grada +gráfico +grano +grasa +gratis +grave +grieta +grillo +gripe +gris +grito +grosor +grúa +grueso +grumo +grupo +guante +guapo +guardia +guerra +guía +guiño +guion +guiso +guitarra +gusano +gustar +haber +hábil +hablar +hacer +hacha +hada +hallar +hamaca +harina +haz +hazaña +hebilla +hebra +hecho +helado +helio +hembra +herir +hermano +héroe +hervir +hielo +hierro +hígado +higiene +hijo +himno +historia +hocico +hogar +hoguera +hoja +hombre +hongo +honor +honra +hora +hormiga +horno +hostil +hoyo +hueco +huelga +huerta +hueso +huevo +huida +huir +humano +húmedo +humilde +humo +hundir +huracán +hurto +icono +ideal +idioma +ídolo +iglesia +iglú +igual +ilegal +ilusión +imagen +imán +imitar +impar +imperio +imponer +impulso +incapaz +índice +inerte +infiel +informe +ingenio +inicio +inmenso +inmune +innato +insecto +instante +interés +íntimo +intuir +inútil +invierno +ira +iris +ironía +isla +islote +jabalí +jabón +jamón +jarabe +jardín +jarra +jaula +jazmín +jefe +jeringa +jinete +jornada +joroba +joven +joya +juerga +jueves +juez +jugador +jugo +juguete +juicio +junco +jungla +junio +juntar +júpiter +jurar +justo +juvenil +juzgar +kilo +koala +labio +lacio +lacra +lado +ladrón +lagarto +lágrima +laguna +laico +lamer +lámina +lámpara +lana +lancha +langosta +lanza +lápiz +largo +larva +lástima +lata +látex +latir +laurel +lavar +lazo +leal +lección +leche +lector +leer +legión +legumbre +lejano +lengua +lento +leña +león +leopardo +lesión +letal +letra +leve +leyenda +libertad +libro +licor +líder +lidiar +lienzo +liga +ligero +lima +límite +limón +limpio +lince +lindo +línea +lingote +lino +linterna +líquido +liso +lista +litera +litio +litro +llaga +llama +llanto +llave +llegar +llenar +llevar +llorar +llover +lluvia +lobo +loción +loco +locura +lógica +logro +lombriz +lomo +lonja +lote +lucha +lucir +lugar +lujo +luna +lunes +lupa +lustro +luto +luz +maceta +macho +madera +madre +maduro +maestro +mafia +magia +mago +maíz +maldad +maleta +malla +malo +mamá +mambo +mamut +manco +mando +manejar +manga +maniquí +manjar +mano +manso +manta +mañana +mapa +máquina +mar +marco +marea +marfil +margen +marido +mármol +marrón +martes +marzo +masa +máscara +masivo +matar +materia +matiz +matriz +máximo +mayor +mazorca +mecha +medalla +medio +médula +mejilla +mejor +melena +melón +memoria +menor +mensaje +mente +menú +mercado +merengue +mérito +mes +mesón +meta +meter +método +metro +mezcla +miedo +miel +miembro +miga +mil +milagro +militar +millón +mimo +mina +minero +mínimo +minuto +miope +mirar +misa +miseria +misil +mismo +mitad +mito +mochila +moción +moda +modelo +moho +mojar +molde +moler +molino +momento +momia +monarca +moneda +monja +monto +moño +morada +morder +moreno +morir +morro +morsa +mortal +mosca +mostrar +motivo +mover +móvil +mozo +mucho +mudar +mueble +muela +muerte +muestra +mugre +mujer +mula +muleta +multa +mundo +muñeca +mural +muro +músculo +museo +musgo +música +muslo +nácar +nación +nadar +naipe +naranja +nariz +narrar +nasal +natal +nativo +natural +náusea +naval +nave +navidad +necio +néctar +negar +negocio +negro +neón +nervio +neto +neutro +nevar +nevera +nicho +nido +niebla +nieto +niñez +niño +nítido +nivel +nobleza +noche +nómina +noria +norma +norte +nota +noticia +novato +novela +novio +nube +nuca +núcleo +nudillo +nudo +nuera +nueve +nuez +nulo +número +nutria +oasis +obeso +obispo +objeto +obra +obrero +observar +obtener +obvio +oca +ocaso +océano +ochenta +ocho +ocio +ocre +octavo +octubre +oculto +ocupar +ocurrir +odiar +odio +odisea +oeste +ofensa +oferta +oficio +ofrecer +ogro +oído +oír +ojo +ola +oleada +olfato +olivo +olla +olmo +olor +olvido +ombligo +onda +onza +opaco +opción +ópera +opinar +oponer +optar +óptica +opuesto +oración +orador +oral +órbita +orca +orden +oreja +órgano +orgía +orgullo +oriente +origen +orilla +oro +orquesta +oruga +osadía +oscuro +osezno +oso +ostra +otoño +otro +oveja +óvulo +óxido +oxígeno +oyente +ozono +pacto +padre +paella +página +pago +país +pájaro +palabra +palco +paleta +pálido +palma +paloma +palpar +pan +panal +pánico +pantera +pañuelo +papá +papel +papilla +paquete +parar +parcela +pared +parir +paro +párpado +parque +párrafo +parte +pasar +paseo +pasión +paso +pasta +pata +patio +patria +pausa +pauta +pavo +payaso +peatón +pecado +pecera +pecho +pedal +pedir +pegar +peine +pelar +peldaño +pelea +peligro +pellejo +pelo +peluca +pena +pensar +peñón +peón +peor +pepino +pequeño +pera +percha +perder +pereza +perfil +perico +perla +permiso +perro +persona +pesa +pesca +pésimo +pestaña +pétalo +petróleo +pez +pezuña +picar +pichón +pie +piedra +pierna +pieza +pijama +pilar +piloto +pimienta +pino +pintor +pinza +piña +piojo +pipa +pirata +pisar +piscina +piso +pista +pitón +pizca +placa +plan +plata +playa +plaza +pleito +pleno +plomo +pluma +plural +pobre +poco +poder +podio +poema +poesía +poeta +polen +policía +pollo +polvo +pomada +pomelo +pomo +pompa +poner +porción +portal +posada +poseer +posible +poste +potencia +potro +pozo +prado +precoz +pregunta +premio +prensa +preso +previo +primo +príncipe +prisión +privar +proa +probar +proceso +producto +proeza +profesor +programa +prole +promesa +pronto +propio +próximo +prueba +público +puchero +pudor +pueblo +puerta +puesto +pulga +pulir +pulmón +pulpo +pulso +puma +punto +puñal +puño +pupa +pupila +puré +quedar +queja +quemar +querer +queso +quieto +química +quince +quitar +rábano +rabia +rabo +ración +radical +raíz +rama +rampa +rancho +rango +rapaz +rápido +rapto +rasgo +raspa +rato +rayo +raza +razón +reacción +realidad +rebaño +rebote +recaer +receta +rechazo +recoger +recreo +recto +recurso +red +redondo +reducir +reflejo +reforma +refrán +refugio +regalo +regir +regla +regreso +rehén +reino +reír +reja +relato +relevo +relieve +relleno +reloj +remar +remedio +remo +rencor +rendir +renta +reparto +repetir +reposo +reptil +res +rescate +resina +respeto +resto +resumen +retiro +retorno +retrato +reunir +revés +revista +rey +rezar +rico +riego +rienda +riesgo +rifa +rígido +rigor +rincón +riñón +río +riqueza +risa +ritmo +rito diff --git a/src/mnemonics/mnemonics.cpp b/src/mnemonics/mnemonics.cpp new file mode 100644 index 000000000..6eab37a40 --- /dev/null +++ b/src/mnemonics/mnemonics.cpp @@ -0,0 +1,276 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session::mnemonics { + +using namespace oxen::log::literals; + +unknown_word_error::unknown_word_error(std::string word) : + std::invalid_argument{"Unknown mnemonic word: {}"_format(word)}, word_{std::move(word)} {} + +checksum_error::checksum_error() : + std::invalid_argument{"Seed phrase checksum word does not match"} {} + +unknown_language_error::unknown_language_error(std::string name) : + std::invalid_argument{"Unknown mnemonic language: {}"_format(name)}, + name_{std::move(name)} {} + +const Mnemonics* find_language(std::string_view name) { + for (auto lang : get_languages()) { + if (lang->english_name == name || lang->native_name == name) + return lang; + } + return nullptr; +} + +const Mnemonics& get_language(std::string_view name) { + auto* lang = find_language(name); + if (!lang) + throw unknown_language_error{std::string(name)}; + return *lang; +} + +namespace { + // CRC-32/ISO-HDLC -- the same function as zlib's crc32() and boost::crc_32_type, which is what + // the reference mnemonic implementations use to derive the checksum word. Implemented here + // rather than pulled in: the only input is a hundred-odd bytes of word prefixes, once per + // encode or decode, which does not justify a compression library as a link dependency. + // + // Deliberately not in session/hash.hpp: this detects accidental corruption and nothing more, + // and sitting it next to BLAKE2b would invite someone to use it as though it were a hash. + constexpr std::array crc32_table = [] { + std::array t{}; + for (uint32_t i = 0; i < 256; i++) { + uint32_t c = i; + for (int k = 0; k < 8; k++) + c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; + t[i] = c; + } + return t; + }(); + + constexpr uint32_t crc32(std::string_view data) { + uint32_t c = 0xFFFFFFFFu; + for (unsigned char b : data) + c = crc32_table[(c ^ b) & 0xFF] ^ (c >> 8); + return c ^ 0xFFFFFFFFu; + } + + // The standard CRC-32 check value; a mistyped table cannot compile. + static_assert(crc32("123456789") == 0xCBF43926u); + + // Returns the first `n_codepoints` codepoints of `s`, composed and case folded. + // + // Composition matters because the word lists are NFC and a decomposed input is a different byte + // sequence for the same word: `ö` typed as `o` + U+0308 puts the combining mark outside the + // prefix window, so the accent is dropped rather than mismatched. Mostly that fails to match + // anything, but Russian `тайна` decomposed truncates to `таи`, which *is* `таинство` -- the + // lookup silently succeeds against the wrong word. + // + // Case folding is done here rather than with towlower because towlower is locale-dependent: + // under LC_CTYPE=C it leaves U+00D6 alone, so case-insensitivity would work or not depending on + // the environment the process happens to run in. + // + // Both the word lists and user input go through this, so downstream comparisons are plain byte + // comparisons on canonical data. + std::string word_prefix(std::string_view s, int n_codepoints) { + utf8proc_uint8_t* folded = nullptr; + auto len = utf8proc_map( + reinterpret_cast(s.data()), + static_cast(s.size()), + &folded, + static_cast( + UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_CASEFOLD)); + if (len < 0) + // Not valid UTF-8, so it cannot be one of the words; the caller reports it as unknown. + return {}; + + std::unique_ptr owned{folded, &std::free}; + std::string_view canonical{reinterpret_cast(folded), static_cast(len)}; + + // Take n codepoints by skipping continuation bytes: canonical UTF-8 needs no decoding to + // find codepoint boundaries. + size_t end = 0; + for (int count = 0; end < canonical.size() && count < n_codepoints; count++) { + end++; + while (end < canonical.size() && + (static_cast(canonical[end]) & 0xC0) == 0x80) + end++; + } + return std::string{canonical.substr(0, end)}; + } + + using WordMap = std::unordered_map; + + const WordMap& get_word_map(const Mnemonics& lang) { + auto langs = get_languages(); + size_t idx = std::find(langs.begin(), langs.end(), &lang) - langs.begin(); + + static std::vector maps(langs.size()); + static std::vector flags(langs.size()); + + std::call_once(flags[idx], [&] { + for (int i = 0; i < static_cast(NWORDS); ++i) { + std::string prefix = word_prefix(lang.words[i], lang.prefix_len); + assert(!prefix.empty()); + maps[idx][prefix] = i; + } + }); + return maps[idx]; + } + + int get_word_index(const Mnemonics& lang, std::string_view word) { + const auto& wm = get_word_map(lang); + auto it = wm.find(word_prefix(word, lang.prefix_len)); + return it != wm.end() ? it->second : -1; + } + // Which of `words` is repeated as the checksum word: a CRC-32 over their concatenated prefixes, + // modulo the count. Only the prefix of each word participates, which is what makes a phrase + // survive a typo past the significant letters -- the same property that lets the words be + // recognised at all. `words` excludes the checksum word itself. + size_t checksum_index(std::span words, const Mnemonics& lang) { + std::string prefixes; + for (const auto& w : words) + prefixes += word_prefix(w, lang.prefix_len); + return crc32(prefixes) % words.size(); + } + +} // namespace + +// string_view objects stored in secure_mnemonic::storage are placement-new constructed below. +// We rely on string_view being trivially destructible so that secure_buffer can zero and free +// the memory without needing to call destructors. +static_assert(std::is_trivially_destructible_v); + +secure_mnemonic bytes_to_words( + std::span bytes, const Mnemonics& lang, bool checksum) { + if (bytes.size() % 4 != 0) + throw std::invalid_argument("Input length must be a multiple of 4 bytes"); + + size_t n = (bytes.size() / 4) * 3; + size_t total = n + checksum; + + secure_mnemonic result; + auto rw = result.storage.resize(total * sizeof(std::string_view)); + auto* out = reinterpret_cast(rw.buf.data()); + + for (size_t i = 0; i < bytes.size(); i += 4) { + uint32_t val = oxenc::load_little_to_host(&bytes[i]); + + uint32_t a = val % NWORDS; + uint32_t b = (val / NWORDS + a) % NWORDS; + uint32_t c = (val / NWORDS / NWORDS + b) % NWORDS; + + std::construct_at(out + i / 4 * 3 + 0, lang.words[a]); + std::construct_at(out + i / 4 * 3 + 1, lang.words[b]); + std::construct_at(out + i / 4 * 3 + 2, lang.words[c]); + } + + if (checksum) + std::construct_at(out + n, out[checksum_index({out, n}, lang)]); + + return result; +} + +secure_mnemonic bytes_to_words( + std::span bytes, std::string_view lang_name, bool checksum) { + return bytes_to_words(bytes, get_language(lang_name), checksum); +} + +// Validates the word count against `out.size()` and decodes words directly into `out`. +// out.size() must be a multiple of 4; words.size() must be (out.size()/4*3) or +1 with checksum. +static void words_to_bytes_impl( + std::span words, const Mnemonics& lang, std::span out) { + if (out.size() % 4 != 0) + throw std::invalid_argument( + "Output buffer size must be a multiple of 4 (got {})"_format(out.size())); + + size_t expected_seed_words = out.size() / 4 * 3; + size_t n = words.size(); + bool has_checksum = n == expected_seed_words + 1; + if (n != expected_seed_words && !has_checksum) + throw std::invalid_argument( + "Seed phrase word count ({}) does not match output buffer size ({} bytes, " + "expecting {} or {} words)"_format( + n, out.size(), expected_seed_words, expected_seed_words + 1)); + + uint32_t sum = 0; + for (size_t i = 0; i < expected_seed_words; i += 3) { + std::array w; + for (int j = 0; j < 3; j++) { + int idx = get_word_index(lang, words[i + j]); + if (idx < 0) + throw unknown_word_error{std::string(words[i + j])}; + w[j] = static_cast(idx); + } + auto [a, b, c] = w; + + uint32_t x = a + ((NWORDS - a + b) % NWORDS) * NWORDS + + ((NWORDS - b + c) % NWORDS) * (NWORDS * NWORDS); + + if (x % NWORDS != a) + throw std::invalid_argument("Seed phrase encodes an invalid value"); + + oxenc::write_host_as_little(x, &out[(i / 3) * 4]); + } + + if (has_checksum) { + int checksum_idx = get_word_index(lang, words[n - 1]); + if (checksum_idx < 0) + throw unknown_word_error{std::string(words[n - 1])}; + + // Compared by index, not by spelling: get_word_index() resolves a word by its prefix, so a + // phrase whose words differ only past the significant letters still validates -- which is + // the point of a prefix-based word list. + auto expected = words.first(expected_seed_words); + int expected_idx = get_word_index(lang, expected[checksum_index(expected, lang)]); + if (checksum_idx != expected_idx) + throw checksum_error{}; + } +} + +session::secure_buffer words_to_bytes( + std::span words, const Mnemonics& lang) { + size_t n = words.size(); + bool has_checksum = n % 3 == 1; + if (n % 3 != 0 && !has_checksum) + throw std::invalid_argument( + "Seed phrase word count must be a multiple of 3, or a multiple of 3 plus one " + "checksum word (got {})"_format(n)); + + size_t nbytes = ((n - has_checksum) / 3) * 4; + session::secure_buffer result; + auto rw = result.resize(nbytes); + words_to_bytes_impl(words, lang, rw.buf); + return result; +} + +session::secure_buffer words_to_bytes( + std::span words, std::string_view lang_name) { + return words_to_bytes(words, get_language(lang_name)); +} + +void words_to_bytes( + std::span words, const Mnemonics& lang, std::span out) { + words_to_bytes_impl(words, lang, out); +} + +void words_to_bytes( + std::span words, + std::string_view lang_name, + std::span out) { + words_to_bytes_impl(words, get_language(lang_name), out); +} + +} // namespace session::mnemonics diff --git a/src/multi_encrypt.cpp b/src/multi_encrypt.cpp index 9b1b7a5cd..41c8f4acb 100644 --- a/src/multi_encrypt.cpp +++ b/src/multi_encrypt.cpp @@ -2,120 +2,77 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include #include +#include #include +#include "session/hash.hpp" +#include "session/util.hpp" + namespace session { -const size_t encrypt_multiple_message_overhead = crypto_aead_xchacha20poly1305_ietf_ABYTES; +const size_t encrypt_multiple_message_overhead = encryption::XCHACHA20_ABYTES; namespace detail { void encrypt_multi_key( - std::array& key, - const unsigned char* a, - const unsigned char* A, - const unsigned char* B, + std::span key, + std::span a, + std::span A, + std::span B, bool encrypting, std::string_view domain) { - std::array buf; - if (0 != crypto_scalarmult_curve25519(buf.data(), a, B)) - throw std::invalid_argument{"Unable to compute shared encrypted key: invalid pubkey?"}; - - static_assert(crypto_aead_xchacha20poly1305_ietf_KEYBYTES == 32); - - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init( - &st, - reinterpret_cast(domain.data()), - std::min(domain.size(), crypto_generichash_blake2b_KEYBYTES_MAX), - 32); - - crypto_generichash_blake2b_update(&st, buf.data(), buf.size()); + auto buf = x25519::scalarmult(a, B); // If we're encrypting then a/A == sender, B = recipient // If we're decrypting then a/A = recipient, B = sender // We always need the same sR || S || R or rS || S || R, so if we're decrypting we need to // put B before A in the hash; - const auto* S = encrypting ? A : B; - const auto* R = encrypting ? B : A; - crypto_generichash_blake2b_update(&st, S, 32); - crypto_generichash_blake2b_update(&st, R, 32); - crypto_generichash_blake2b_final(&st, key.data(), 32); + const auto& S = encrypting ? A : B; + const auto& R = encrypting ? B : A; + hash::blake2b_key(key, domain, buf, S, R); } void encrypt_multi_impl( - std::vector& out, - std::span msg, - const unsigned char* key, - const unsigned char* nonce) { - - // auto key = encrypt_multi_key(a, A, B, true, domain); - - out.resize(msg.size() + crypto_aead_xchacha20poly1305_ietf_ABYTES); - if (0 != - crypto_aead_xchacha20poly1305_ietf_encrypt( - out.data(), nullptr, msg.data(), msg.size(), nullptr, 0, nullptr, nonce, key)) - throw std::runtime_error{"XChaCha20 encryption failed!"}; + std::vector& out, + std::span msg, + std::span key, + std::span nonce) { + + out.resize(msg.size() + encryption::XCHACHA20_ABYTES); + encryption::xchacha20poly1305_encrypt(out, msg, nonce, key); } bool decrypt_multi_impl( - std::vector& out, - std::span ciphertext, - const unsigned char* key, - const unsigned char* nonce) { + std::vector& out, + std::span ciphertext, + std::span key, + std::span nonce) { - if (ciphertext.size() < crypto_aead_xchacha20poly1305_ietf_ABYTES) + if (ciphertext.size() < encryption::XCHACHA20_ABYTES) return false; - out.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES); - return 0 == crypto_aead_xchacha20poly1305_ietf_decrypt( - out.data(), - nullptr, - nullptr, - ciphertext.data(), - ciphertext.size(), - nullptr, - 0, - nonce, - key); - } - - std::pair>, std::array> x_keys( - std::span ed25519_secret_key) { - if (ed25519_secret_key.size() != 64) - throw std::invalid_argument{"Ed25519 secret key is not the expected 64 bytes"}; - - std::pair>, std::array> ret; - auto& [x_priv, x_pub] = ret; - - crypto_sign_ed25519_sk_to_curve25519(x_priv.data(), ed25519_secret_key.data()); - if (0 != crypto_sign_ed25519_pk_to_curve25519(x_pub.data(), ed25519_secret_key.data() + 32)) - throw std::runtime_error{"Failed to convert Ed25519 key to X25519: invalid secret key"}; - - return ret; + out.resize(ciphertext.size() - encryption::XCHACHA20_ABYTES); + return encryption::xchacha20poly1305_decrypt(out, ciphertext, nonce, key); } } // namespace detail -std::optional> decrypt_for_multiple( - const std::vector>& ciphertexts, - std::span nonce, - std::span privkey, - std::span pubkey, - std::span sender_pubkey, +std::optional> decrypt_for_multiple( + const std::vector>& ciphertexts, + std::span nonce, + std::span privkey, + std::span pubkey, + std::span sender_pubkey, std::string_view domain) { auto it = ciphertexts.begin(); return decrypt_for_multiple( - [&]() -> std::optional> { + [&]() -> std::optional> { if (it == ciphertexts.end()) return std::nullopt; return *it++; @@ -127,21 +84,21 @@ std::optional> decrypt_for_multiple( domain); } -std::vector encrypt_for_multiple_simple( - const std::vector>& messages, - const std::vector>& recipients, - std::span privkey, - std::span pubkey, +std::vector encrypt_for_multiple_simple( + const std::vector>& messages, + const std::vector>& recipients, + std::span privkey, + std::span pubkey, std::string_view domain, - std::optional> nonce, + std::optional> nonce, int pad) { oxenc::bt_dict_producer d; - std::array random_nonce; + std::array random_nonce; if (!nonce) { - randombytes_buf(random_nonce.data(), random_nonce.size()); - nonce.emplace(random_nonce.data(), random_nonce.size()); + random::fill(random_nonce); + nonce.emplace(random_nonce); } else if (nonce->size() != 24) { throw std::invalid_argument{"Invalid nonce: nonce must be 24 bytes"}; } @@ -158,16 +115,16 @@ std::vector encrypt_for_multiple_simple( privkey, pubkey, domain, - [&](std::span encrypted) { + [&](std::span encrypted) { enc_list.append(encrypted); msg_count++; }); if (pad > 1 && !messages.empty()) { const auto pad_size = messages.front().size() + encrypt_multiple_message_overhead; - std::vector junk(pad_size); + std::vector junk(pad_size); for (; msg_count % pad != 0; msg_count++) { - randombytes_buf(junk.data(), junk.size()); + random::fill(junk); enc_list.append(to_string(junk)); } } @@ -176,38 +133,38 @@ std::vector encrypt_for_multiple_simple( return to_vector(d.span()); } -std::vector encrypt_for_multiple_simple( - const std::vector>& messages, - const std::vector>& recipients, - std::span ed25519_secret_key, +std::vector encrypt_for_multiple_simple( + const std::vector>& messages, + const std::vector>& recipients, + const ed25519::PrivKeySpan& ed25519_secret_key, std::string_view domain, - std::span nonce, + std::optional> nonce, int pad) { - auto [x_privkey, x_pubkey] = detail::x_keys(ed25519_secret_key); + auto [x_privkey, x_pubkey] = ed25519::x25519_keypair(ed25519_secret_key); return encrypt_for_multiple_simple( - messages, recipients, to_span(x_privkey), to_span(x_pubkey), domain, nonce, pad); + messages, recipients, x_privkey, x_pubkey, domain, nonce, pad); } -std::optional> decrypt_for_multiple_simple( - std::span encoded, - std::span privkey, - std::span pubkey, - std::span sender_pubkey, +std::optional> decrypt_for_multiple_simple( + std::span encoded, + std::span privkey, + std::span pubkey, + std::span sender_pubkey, std::string_view domain) { try { oxenc::bt_dict_consumer d{encoded}; - auto nonce = d.require>("#"); + auto nonce = d.require>("#"); if (nonce.size() != 24) return std::nullopt; auto enc_list = d.require("e"); return decrypt_for_multiple( - [&]() -> std::optional> { + [&]() -> std::optional> { if (enc_list.is_finished()) return std::nullopt; - return enc_list.consume>(); + return enc_list.consume>(); }, nonce, privkey, @@ -219,38 +176,33 @@ std::optional> decrypt_for_multiple_simple( } } -std::optional> decrypt_for_multiple_simple( - std::span encoded, - std::span ed25519_secret_key, - std::span sender_pubkey, +std::optional> decrypt_for_multiple_simple( + std::span encoded, + const ed25519::PrivKeySpan& ed25519_secret_key, + std::span sender_pubkey, std::string_view domain) { - auto [x_privkey, x_pubkey] = detail::x_keys(ed25519_secret_key); + auto [x_privkey, x_pubkey] = ed25519::x25519_keypair(ed25519_secret_key); - return decrypt_for_multiple_simple( - encoded, to_span(x_privkey), to_span(x_pubkey), sender_pubkey, domain); + return decrypt_for_multiple_simple(encoded, x_privkey, x_pubkey, sender_pubkey, domain); } -std::optional> decrypt_for_multiple_simple_ed25519( - std::span encoded, - std::span ed25519_secret_key, - std::span sender_ed25519_pubkey, +std::optional> decrypt_for_multiple_simple_ed25519( + std::span encoded, + const ed25519::PrivKeySpan& ed25519_secret_key, + std::span sender_ed25519_pubkey, std::string_view domain) { - std::array sender_pub; - if (sender_ed25519_pubkey.size() != 32) - throw std::invalid_argument{"Invalid sender Ed25519 pubkey: expected 32 bytes"}; - if (0 != crypto_sign_ed25519_pk_to_curve25519(sender_pub.data(), sender_ed25519_pubkey.data())) - throw std::runtime_error{"Failed to convert Ed25519 key to X25519: invalid secret key"}; + auto sender_pub = ed25519::pk_to_x25519(sender_ed25519_pubkey); - return decrypt_for_multiple_simple(encoded, ed25519_secret_key, to_span(sender_pub), domain); + return decrypt_for_multiple_simple(encoded, ed25519_secret_key, sender_pub, domain); } } // namespace session using namespace session; -static unsigned char* to_c_buffer(std::span x, size_t* out_len) { +static unsigned char* to_c_buffer(std::span x, size_t* out_len) { auto* ret = static_cast(malloc(x.size())); *out_len = x.size(); std::memcpy(ret, x.data(), x.size()); @@ -270,23 +222,23 @@ LIBSESSION_C_API unsigned char* session_encrypt_for_multiple_simple( const unsigned char* nonce, int pad) { - std::vector> msgs, recips; + std::vector> msgs, recips; msgs.reserve(n_messages); recips.reserve(n_recipients); for (size_t i = 0; i < n_messages; i++) - msgs.emplace_back(messages[i], message_lengths[i]); + msgs.emplace_back(to_byte_span(messages[i], message_lengths[i])); for (size_t i = 0; i < n_recipients; i++) - recips.emplace_back(recipients[i], 32); - std::optional> maybe_nonce; + recips.emplace_back(to_byte_span<32>(recipients[i])); + std::optional> maybe_nonce; if (nonce) - maybe_nonce.emplace(nonce, 24); + maybe_nonce.emplace(to_byte_span<24>(nonce)); try { auto encoded = session::encrypt_for_multiple_simple( msgs, recips, - std::span{x25519_privkey, 32}, - std::span{x25519_pubkey, 32}, + to_byte_span<32>(x25519_privkey), + to_byte_span<32>(x25519_pubkey), domain, std::move(maybe_nonce), pad); @@ -309,8 +261,7 @@ LIBSESSION_C_API unsigned char* session_encrypt_for_multiple_simple_ed25519( int pad) { try { - auto [priv, pub] = - session::detail::x_keys(std::span{ed25519_secret_key, 64}); + auto [priv, pub] = session::ed25519::x25519_keypair(to_byte_span<64>(ed25519_secret_key)); return session_encrypt_for_multiple_simple( out_len, messages, @@ -318,8 +269,8 @@ LIBSESSION_C_API unsigned char* session_encrypt_for_multiple_simple_ed25519( n_messages, recipients, n_recipients, - priv.data(), - pub.data(), + to_unsigned(priv.data()), + to_unsigned(pub.data()), domain, nonce, pad); @@ -339,10 +290,10 @@ LIBSESSION_C_API unsigned char* session_decrypt_for_multiple_simple( try { if (auto decrypted = session::decrypt_for_multiple_simple( - std::span{encoded, encoded_len}, - std::span{x25519_privkey, 32}, - std::span{x25519_pubkey, 32}, - std::span{sender_x25519_pubkey, 32}, + to_byte_span(encoded, encoded_len), + to_byte_span<32>(x25519_privkey), + to_byte_span<32>(x25519_pubkey), + to_byte_span<32>(sender_x25519_pubkey), domain)) { return to_c_buffer(*decrypted, out_len); } @@ -362,9 +313,9 @@ LIBSESSION_C_API unsigned char* session_decrypt_for_multiple_simple_ed25519_from try { if (auto decrypted = session::decrypt_for_multiple_simple( - std::span{encoded, encoded_len}, - std::span{ed25519_secret, 64}, - std::span{sender_x25519_pubkey, 32}, + to_byte_span(encoded, encoded_len), + to_byte_span<64>(ed25519_secret), + to_byte_span<32>(sender_x25519_pubkey), domain)) { return to_c_buffer(*decrypted, out_len); } @@ -384,9 +335,9 @@ LIBSESSION_C_API unsigned char* session_decrypt_for_multiple_simple_ed25519( try { if (auto decrypted = session::decrypt_for_multiple_simple_ed25519( - std::span{encoded, encoded_len}, - std::span{ed25519_secret, 64}, - std::span{sender_ed25519_pubkey, 32}, + to_byte_span(encoded, encoded_len), + to_byte_span<64>(ed25519_secret), + to_byte_span<32>(sender_ed25519_pubkey), domain)) { return to_c_buffer(*decrypted, out_len); } diff --git a/src/network/backends/quic_file_client.cpp b/src/network/backends/quic_file_client.cpp new file mode 100644 index 000000000..cdde5b687 --- /dev/null +++ b/src/network/backends/quic_file_client.cpp @@ -0,0 +1,666 @@ +#include "session/network/backends/quic_file_client.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "session/clock.hpp" +#include "session/crypto/ed25519.hpp" + +using namespace oxen; +using namespace std::literals; +using namespace oxen::log::literals; + +namespace session::network { + +namespace { + auto cat = log::Cat("quic-file-client"); +} + +// -- QuicFileClient -- + +QuicFileClient::QuicFileClient( + std::shared_ptr loop, + ed25519_pubkey ed_pubkey, + std::string address, + uint16_t port, + std::optional max_udp_payload, + ticket_store_cb ticket_store, + ticket_extract_cb ticket_extract) : + _loop{std::move(loop)}, + _ed_pubkey{std::move(ed_pubkey)}, + _address{std::move(address)}, + _port{port}, + _max_udp_payload{max_udp_payload}, + _ticket_store{std::move(ticket_store)}, + _ticket_extract{std::move(ticket_extract)}, + _last_activity{std::chrono::steady_clock::now()} { + + // Create a dedicated endpoint for file server connections + _ep = quic::Endpoint::endpoint( + *_loop, + quic::Address{}, + (_max_udp_payload ? std::make_optional(*_max_udp_payload) + : std::nullopt)); + + // Set up TLS credentials + auto [pk, sk] = ed25519::keypair(); + _creds = quic::GNUTLSCreds::make_from_ed_seckey( + std::string_view{reinterpret_cast(sk.data()), sk.size()}); + + // Enable 0RTT if callbacks are provided + if (_ticket_store && _ticket_extract) { + _creds->enable_outbound_0rtt( + [store = _ticket_store]( + quic::RemoteAddress remote, + std::vector data, + std::chrono::sys_seconds expiry) { + store(oxenc::to_hex(remote.view_remote_key()), std::move(data), expiry); + }, + [extract = _ticket_extract](const quic::RemoteAddress& remote) + -> std::optional> { + return extract(oxenc::to_hex(remote.view_remote_key())); + }); + } + + log::debug(cat, "QuicFileClient created for target {}:{}", _address, _port); +} + +QuicFileClient::~QuicFileClient() { + close(); +} + +void QuicFileClient::set_target(ed25519_pubkey ed_pubkey, std::string address, uint16_t port) { + if (_address != address || _port != port || _ed_pubkey != ed_pubkey) { + close(); + _ed_pubkey = std::move(ed_pubkey); + _address = std::move(address); + _port = port; + log::debug(cat, "Target updated to {}:{}", _address, _port); + } +} + +void QuicFileClient::close() { + _idle_timer.reset(); + _bt_stream.reset(); + if (_conn) { + _conn->close_connection(); + _conn.reset(); + } +} + +void QuicFileClient::_touch() { + _last_activity = std::chrono::steady_clock::now(); +} + +void QuicFileClient::_start_idle_timer() { + if (_idle_timer) + return; + + _idle_timer = _loop->call_every(IDLE_CHECK_INTERVAL, [this] { + if (!_conn) + return; + auto idle_duration = std::chrono::steady_clock::now() - _last_activity; + if (idle_duration >= IDLE_TIMEOUT) { + log::debug(cat, "Connection idle for {}s, closing.", idle_duration / 1s); + close(); + } + }); +} + +std::shared_ptr QuicFileClient::_ensure_connection() { + if (_conn) + return _conn; + + auto remote = quic::RemoteAddress{oxenc::from_hex(_ed_pubkey.hex()), _address, _port}; + + log::info(cat, "Connecting to QUIC file server at {}:{}", _address, _port); + + _conn = _ep->connect( + remote, + _creds, + quic::opt::outbound_alpn(QUIC_FILES_ALPN), + quic::opt::handshake_timeout{10s}, + quic::opt::keep_alive{10s}, + [this](quic::Connection&) { log::info(cat, "Connected to QUIC file server."); }, + [this](quic::Connection&, uint64_t ec) { + if (ec) + log::warning(cat, "Connection to QUIC file server failed (error {}).", ec); + else + log::debug(cat, "Connection to QUIC file server closed."); + _conn.reset(); + _bt_stream.reset(); + }); + + // Open stream 0 as BTRequestStream (required by file server protocol — subsequent file + // transfer streams get IDs 4, 8, etc.). + // TODO: use this stream for metadata requests (file info, extend TTL, etc.) + _bt_stream = _conn->open_stream(); + + _touch(); + _start_idle_timer(); + + return _conn; +} + +void QuicFileClient::upload( + std::vector data, + std::optional ttl, + std::function result)> on_complete) { + _loop->call([this, + data = std::make_shared>(std::move(data)), + ttl, + on_complete = std::move(on_complete)]() mutable { + try { + auto conn = _ensure_connection(); + if (!conn) { + on_complete(static_cast(ERROR_UNKNOWN)); + return; + } + + // State shared between the stream callbacks + struct upload_state { + int64_t upload_size; + std::string response_data; + std::function)> on_complete; + }; + auto state = std::make_shared(); + state->upload_size = static_cast(data->size()); + state->on_complete = std::move(on_complete); + + auto on_data = [state](quic::Stream&, std::span incoming) { + state->response_data += std::string_view{ + reinterpret_cast(incoming.data()), incoming.size()}; + }; + + auto on_close = [this, state](quic::Stream&, uint64_t error_code) { + _touch(); + + if (error_code != 0) { + log::warning(cat, "Upload stream closed with error {}.", error_code); + state->on_complete(static_cast(error_code)); + return; + } + + if (state->response_data.empty()) { + log::warning(cat, "Upload stream closed with no response data."); + state->on_complete(static_cast(ERROR_UNKNOWN)); + return; + } + + try { + // The upload response is a raw bt-dict, not size-prefixed. + log::trace( + cat, + "Upload response ({} bytes): {}", + state->response_data.size(), + state->response_data); + + oxenc::bt_dict_consumer resp{state->response_data}; + file_metadata metadata{}; + metadata.id = resp.require("#"); + metadata.size = state->upload_size; + metadata.uploaded = std::chrono::sys_seconds{ + std::chrono::seconds{resp.require("u")}}; + metadata.expiry = std::chrono::sys_seconds{ + std::chrono::seconds{resp.require("x")}}; + + log::info( + cat, + "Upload complete: file ID={}, expiry={}", + metadata.id, + metadata.expiry); + state->on_complete(std::move(metadata)); + } catch (const std::exception& e) { + log::error(cat, "Failed to parse upload response: {}", e.what()); + state->on_complete(static_cast(ERROR_UNKNOWN)); + } + }; + + auto str = conn->open_stream(on_data, on_close); + + // Build and send the PUT command + oxenc::bt_dict_producer cmd; + cmd.append("!", "PUT"); + cmd.append("s", static_cast(data->size())); + if (ttl) + cmd.append("t", static_cast(ttl->count())); + + auto cmd_view = cmd.view(); + str->send(fmt::format("{}:{}", cmd_view.size(), cmd_view)); + + // Send the file data, keeping the shared_ptr alive until the send completes + str->send(*data, data); + str->send_fin(); + + _touch(); + log::debug(cat, "Upload started: {} bytes.", data->size()); + + } catch (const std::exception& e) { + log::error(cat, "Upload failed: {}", e.what()); + on_complete(static_cast(ERROR_UNKNOWN)); + } + }); +} + +void QuicFileClient::download( + std::string file_id, + std::function data)> on_data, + std::function result)> on_complete) { + _loop->call([this, + file_id = std::move(file_id), + on_data = std::move(on_data), + on_complete = std::move(on_complete)]() mutable { + try { + auto conn = _ensure_connection(); + if (!conn) { + on_complete(static_cast(ERROR_UNKNOWN)); + return; + } + + // State shared between the stream callbacks + struct download_state { + std::string file_id; + file_metadata metadata{}; + bool metadata_parsed = false; + int meta_size = -1; + std::string partial; + std::vector meta_buf; + int64_t received = 0; + std::function)> on_data; + std::function)> on_complete; + }; + auto state = std::make_shared(); + state->file_id = file_id; + state->on_data = std::move(on_data); + state->on_complete = std::move(on_complete); + + auto data_cb = [this, state](quic::Stream& s, std::span data) { + _touch(); + + // Phase 1: parse the size prefix of the metadata block + if (state->meta_size < 0) { + try { + auto size = quic::prefix_accumulator(state->partial, data); + if (!size) + return; + if (*size == 0) + throw std::runtime_error{"Invalid 0-byte metadata block"}; + state->meta_size = static_cast(*size); + } catch (const std::exception& e) { + log::error(cat, "Download metadata prefix error: {}", e.what()); + s.close(400); + return; + } + state->meta_buf.reserve(state->meta_size); + } + + // Phase 2: accumulate metadata bytes + if (!state->metadata_parsed) { + try { + if (!quic::data_accumulator(state->meta_buf, data, state->meta_size)) + return; + } catch (const std::exception& e) { + log::error(cat, "Download metadata accumulation error: {}", e.what()); + s.close(400); + return; + } + + // Parse metadata dict + try { + oxenc::bt_dict_consumer d{state->meta_buf}; + auto file_size = d.require("s"); + if (file_size <= 0) + throw std::runtime_error{ + fmt::format("Invalid file size {}", file_size)}; + state->metadata.id = state->file_id; + state->metadata.size = file_size; + state->metadata.uploaded = std::chrono::sys_seconds{ + std::chrono::seconds{d.require("u")}}; + state->metadata.expiry = std::chrono::sys_seconds{ + std::chrono::seconds{d.require("x")}}; + d.finish(); + state->metadata_parsed = true; + + log::debug( + cat, + "Download metadata: {} bytes, expiry={}", + state->metadata.size, + state->metadata.expiry); + } catch (const std::exception& e) { + log::error(cat, "Download metadata parse error: {}", e.what()); + s.close(444); + return; + } + } + + // Phase 3: deliver file data + if (!data.empty()) { + state->received += data.size(); + if (state->on_data) { + try { + state->on_data(state->metadata, data); + } catch (const std::exception& e) { + log::warning(cat, "Download aborted by on_data callback: {}", e.what()); + s.close(QUIC_FILES_CLIENT_ABORT); + return; + } + } + } + }; + + auto close_cb = [this, state](quic::Stream&, uint64_t error_code) { + _touch(); + + if (error_code != 0) { + log::warning( + cat, + "Download stream for {} closed with error {}.", + state->file_id, + error_code); + state->on_complete(static_cast(error_code)); + return; + } + + if (!state->metadata_parsed) { + log::warning(cat, "Download stream closed before metadata received."); + state->on_complete(static_cast(ERROR_UNKNOWN)); + return; + } + + if (state->received < state->metadata.size) { + log::warning( + cat, + "Download incomplete: received {}/{} bytes.", + state->received, + state->metadata.size); + state->on_complete(static_cast(ERROR_UNKNOWN)); + return; + } + + log::info( + cat, "Download complete: {} ({} bytes).", state->file_id, state->received); + state->on_complete(state->metadata); + }; + + auto str = conn->open_stream(data_cb, close_cb); + + // Build and send the GET command + oxenc::bt_dict_producer cmd; + cmd.append("!", "GET"); + cmd.append("#", file_id); + + auto cmd_view = cmd.view(); + str->send(fmt::format("{}:{}", cmd_view.size(), cmd_view)); + str->send_fin(); + + _touch(); + log::debug(cat, "Download started for file {}.", file_id); + + } catch (const std::exception& e) { + log::error(cat, "Download failed: {}", e.what()); + on_complete(static_cast(ERROR_UNKNOWN)); + } + }); +} + +void streaming_file_upload( + std::shared_ptr loop, + attachment::Encryptor enc, + FileUploadRequest request, + std::function get_client) { + + struct upload_state { + std::mutex mutex; + std::condition_variable cv; + bool paused = false; + bool done = false; + QuicFileClient* client = nullptr; + std::shared_ptr stream; + std::string response_data; + std::optional> result; + // Tracked on the loop thread by the progress timer + int64_t preamble_size = 0; + int64_t last_acked = 0; // file-relative (preamble subtracted) + std::chrono::steady_clock::time_point last_ack_time; + std::chrono::steady_clock::time_point start_time; + }; + auto state = std::make_shared(); + state->last_ack_time = std::chrono::steady_clock::now(); + state->start_time = state->last_ack_time; + + auto fail = [&](int16_t err, bool timeout = false) { + if (request.on_complete) + loop->call([request, err, timeout] { request.on_complete(err, timeout); }); + }; + + loop->call([state, get_client = std::move(get_client)] { + auto* client = get_client(); + std::lock_guard lock{state->mutex}; + if (client) + state->client = client; + else + state->done = true; + state->cv.notify_one(); + }); + + auto key = enc.load_key_from_file(request.file, request.allow_large); + auto upload_size = attachment::encrypted_size(enc.data_size()); + auto enc_ptr = std::make_shared(std::move(enc)); + + { + std::unique_lock lock{state->mutex}; + state->cv.wait( + lock, [&] { return state->client || state->done || request.is_cancelled(); }); + if (request.is_cancelled()) + return fail(ERROR_REQUEST_CANCELLED); + if (state->done) + return fail(ERROR_FILE_SERVER_UNAVAILABLE); + } + + loop->call_get([&] { + auto conn = state->client->_ensure_connection(); + if (!conn) { + std::lock_guard lock{state->mutex}; + state->done = true; + return; + } + + auto str = conn->open_stream( + [state](quic::Stream&, std::span incoming) { + state->response_data += std::string_view{ + reinterpret_cast(incoming.data()), incoming.size()}; + }, + [state, upload_size](quic::Stream&, uint64_t error_code) { + std::lock_guard lock{state->mutex}; + if (error_code != 0) { + state->result = static_cast(error_code); + } else if (state->response_data.empty()) { + state->result = static_cast(ERROR_UNKNOWN); + } else { + try { + oxenc::bt_dict_consumer resp{state->response_data}; + file_metadata meta{}; + meta.id = resp.require("#"); + meta.size = upload_size; + meta.uploaded = from_epoch_s(resp.require("u")); + meta.expiry = from_epoch_s(resp.require("x")); + resp.finish(); + state->result = std::move(meta); + } catch (const std::exception& e) { + log::warning( + cat, "Failed to parse streaming upload response: {}", e.what()); + state->result = static_cast(ERROR_UNKNOWN); + } + } + state->done = true; + state->cv.notify_one(); + }); + + constexpr size_t WATERMARK_ALARM = 1024 * 1024; + constexpr size_t WATERMARK_CLEAR = 512 * 1024; + str->enable_watermarks( + WATERMARK_ALARM, + [state](quic::Stream&) { + std::lock_guard lock{state->mutex}; + state->paused = true; + }, + WATERMARK_CLEAR, + [state](quic::Stream&) { + { + std::lock_guard lock{state->mutex}; + state->paused = false; + } + state->cv.notify_one(); + }); + + oxenc::bt_dict_producer cmd; + cmd.append("!", "PUT"); + cmd.append("s", static_cast(upload_size)); + if (request.ttl) + cmd.append("t", static_cast(request.ttl->count())); + auto cmd_view = cmd.view(); + auto preamble = fmt::format("{}:{}", cmd_view.size(), cmd_view); + state->preamble_size = static_cast(preamble.size()); + str->send(std::move(preamble)); + + state->stream = std::move(str); + + // Disable the idle timer during the upload; stall detection replaces it. + state->client->_idle_timer.reset(); + }); + + { + std::lock_guard lock{state->mutex}; + if (state->done) + return fail(ERROR_FILE_SERVER_UNAVAILABLE); + } + + // Periodic timer for progress reporting and stall/overall timeout detection. + // Runs on the loop thread where get_stats() is a direct member access (no queuing). + std::shared_ptr progress_timer; + if (request.progress_interval > 0ms) { + progress_timer = + loop->call_every(request.progress_interval, [state, &request, upload_size] { + if (state->done || !state->stream) + return; + + auto now = std::chrono::steady_clock::now(); + auto [acked, unacked, unsent] = state->stream->get_stats(); + auto file_acked = std::max( + 0, static_cast(acked) - state->preamble_size); + + if (file_acked > state->last_acked) { + state->last_acked = file_acked; + state->last_ack_time = now; + + if (request.on_progress) + request.on_progress(file_acked, upload_size); + } + + // Stall detection: no ack progress for stall_timeout + if (request.stall_timeout > 0ms && + now - state->last_ack_time >= request.stall_timeout) { + log::warning( + cat, + "Streaming upload stalled: no ack progress for {}", + request.stall_timeout); + state->stream->close(QUIC_FILES_CLIENT_ABORT); + { + std::lock_guard lock{state->mutex}; + state->result = static_cast(ERROR_REQUEST_TIMEOUT); + state->done = true; + } + state->cv.notify_one(); + return; + } + + // Overall timeout + if (request.overall_timeout && + now - state->start_time >= *request.overall_timeout) { + log::warning(cat, "Streaming upload exceeded overall timeout"); + state->stream->close(QUIC_FILES_CLIENT_ABORT); + { + std::lock_guard lock{state->mutex}; + state->result = static_cast(ERROR_REQUEST_TIMEOUT); + state->done = true; + } + state->cv.notify_one(); + return; + } + }); + } + + auto check_cancelled = [&]() -> bool { + if (!request.is_cancelled()) + return false; + log::debug(cat, "Streaming file upload cancelled"); + loop->call([state, request] { + if (state->stream) + state->stream->close(QUIC_FILES_CLIENT_ABORT); + if (request.on_complete) + request.on_complete(ERROR_REQUEST_CANCELLED, false); + }); + return true; + }; + + // The `next()` call here involves file I/O and so can block: + for (auto chunk = enc_ptr->next(); !chunk.empty(); chunk = enc_ptr->next()) { + if (check_cancelled()) + return; + + { + std::unique_lock lock{state->mutex}; + state->cv.wait( + lock, [&] { return !state->paused || state->done || request.is_cancelled(); }); + if (check_cancelled()) + return; + if (state->done) + break; + } + + auto data = std::make_shared>(chunk.begin(), chunk.end()); + loop->call([state, data] { + if (state->stream) + state->stream->send(*data, data); + }); + } + + loop->call([state] { + if (state->stream) + state->stream->send_fin(); + }); + + { + std::unique_lock lock{state->mutex}; + state->cv.wait(lock, [&] { return state->done; }); + } + + // Stop the progress timer and restart the idle timer for connection reuse + progress_timer.reset(); + if (state->client) + loop->call([state] { state->client->_start_idle_timer(); }); + + if (request.on_complete && state->result) { + loop->call([state, request, result = std::move(*state->result), key, upload_size] { + if (auto* meta = std::get_if(&result)) { + if (request.on_progress && state->last_acked < upload_size) + request.on_progress(upload_size, upload_size); + request.on_complete(std::make_pair(std::move(*meta), key), false); + } else { + request.on_complete(std::get(result), false); + } + }); + } +} +} // namespace session::network diff --git a/src/network/backends/session_file_server.cpp b/src/network/backends/session_file_server.cpp index cbe9108c1..f2029068e 100644 --- a/src/network/backends/session_file_server.cpp +++ b/src/network/backends/session_file_server.cpp @@ -1,18 +1,22 @@ #include "session/network/backends/session_file_server.hpp" #include +#include #include +#include #include #include +#include #include "../session_network_internal.hpp" #include "session/blinding.hpp" +#include "session/clock.hpp" +#include "session/crypto/ed25519.hpp" #include "session/network/backends/backend_util.hpp" #include "session/network/backends/session_file_server.h" #include "session/network/key_types.hpp" #include "session/random.hpp" -#include "session/util.hpp" #if defined(__APPLE__) || !defined(__cpp_lib_chrono) || __cpp_lib_chrono < 201907L || \ (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 190000) @@ -32,14 +36,18 @@ const config::FileServer DEFAULT_CONFIG = { .scheme = "http", .host = "filev2.getsession.org", .port = 80, - // ED25519. `p=` in a download url carries the Ed form on every client, and the X25519 form - // for onion requests is derived from it. - // // NOT the file server's X25519-only key (`da21e1d886c6...ee59`), which cannot be used here: // it has no Ed private key and cannot be given one, since deriving Ed from X would mean // reversing a hash. A file server has to publish a real Ed keypair to be addressable this // way. - .pubkey_hex = "b8eef9821445ae16e2e97ef8aa6fe782fd11ad5253cd6723b281341dba22e371", + .pubkey_hex = oxenc::to_hex(QUIC_FS_ED_PUBKEY_MAINNET), + .max_file_size = 10'000'000}; + +const config::FileServer TESTNET_CONFIG = { + .scheme = "http", + .host = "superduperfiles.oxen.io", + .port = 80, + .pubkey_hex = oxenc::to_hex(QUIC_FS_ED_PUBKEY_TESTNET), .max_file_size = 10'000'000}; constexpr std::string_view HEADER_CONTENT_TYPE = "Content-Type"; @@ -57,11 +65,12 @@ constexpr std::string_view LEGACY_ENDPOINT_FILE_INDIVIDUAL = "files/{}"; std::optional parse_download_url(std::string_view url) { // Expected format: {scheme}://{host}/file/{file_id}(?:#p={customPubkey})(?:d) + // `p=` is the server's Ed25519 pubkey, present only for a server other than the built-in one. // Examples: // https://example.com/file/abc123 - // https://example.com/file/abc123#p=da21e1d886c6fbaea313f75298bd64aab03a97ce985b46bb2dad9f2089c8ee59 + // https://example.com/file/abc123#p=929e33ded05e653fec04b49645117f51851f102a947e04806791be416ed76602 // https://example.com/file/abc123#d - // https://example.com/file/abc123#p=abc123&d + // https://example.com/file/abc123#p=929e33ded05e653fec04b49645117f51851f102a947e04806791be416ed76602&d DownloadInfo info{}; auto match = backends::match_endpoint(ENDPOINT_FILE_INDIVIDUAL, url); @@ -112,25 +121,68 @@ std::optional parse_download_url(std::string_view url) { for (auto fragment : split(fragments, "&", true)) { if (fragment == backends::FRAGMENT_STREAM_ENCRYPTION) info.wants_stream_decryption = true; - else if ( - fragment.starts_with(fmt::format("{}=", backends::FRAGMENT_PUBKEY)) && - fragment.size() == 66 && // 'p=' + pubkey - oxenc::is_hex(fragment.substr(2)) && - fragment.substr(2) != file_server::DEFAULT_CONFIG.pubkey_hex) - info.custom_pubkey_hex = fragment.substr(2); + else if (fragment.starts_with("{}="_format(backends::FRAGMENT_PUBKEY))) { + // Unlike the other fragments a bad pubkey cannot just be skipped: dropping it leaves + // the url's host in place but falls back to our own file server's key, so the request + // would go out encrypted to a key the host it is addressed to does not hold. A url we + // cannot address is not a url we can use, so reject the whole thing. + auto pubkey_hex = fragment.substr(2); + if (pubkey_hex.size() != 64 || !oxenc::is_hex(pubkey_hex)) + return std::nullopt; + + ed25519_pubkey pubkey; + oxenc::from_hex(pubkey_hex.begin(), pubkey_hex.end(), pubkey.begin()); + + // Hex of the right length still leaves ~94% of values off the curve, and the ones that + // land on it but outside the prime-order subgroup are not keys either. + if (!ed25519::is_valid_pubkey(pubkey)) + return std::nullopt; + + if (pubkey_hex != file_server::DEFAULT_CONFIG.pubkey_hex) + info.custom_pubkey_hex = std::string{pubkey_hex}; + } else if (fragment.starts_with("{}="_format(backends::FRAGMENT_SROUTER))) { + // sr=address or sr=address:port (port defaults to QUIC_DEFAULT_PORT if omitted) + auto parts = split(fragment.substr(backends::FRAGMENT_SROUTER.size() + 1), ":"); + if (parts.size() <= 2 && !parts[0].empty()) { + uint16_t port = QUIC_DEFAULT_PORT; + if (parts.size() == 2 && (!quic::parse_int(parts[1], port) || port == 0)) + continue; // Invalid port, skip + info.srouter_target = SRouterTarget{std::string{parts[0]}, port}; + } + } // else ignore (unknown or invalid fragment) } return info; } +const std::string QUIC_FS_SESH_ADDRESS_MAINNET = "{:a}.sesh"_format(QUIC_FS_ED_PUBKEY_MAINNET); +const std::string QUIC_FS_SESH_ADDRESS_TESTNET = "{:a}.sesh"_format(QUIC_FS_ED_PUBKEY_TESTNET); + +std::optional default_quic_target( + const config::FileServer& http_config, opt::netid::Target netid) { + // Map known HTTP file server pubkeys to their QUIC file server .sesh addresses. + if (http_config.pubkey_hex == DEFAULT_CONFIG.pubkey_hex && netid == opt::netid::Target::mainnet) + return SRouterTarget{QUIC_FS_SESH_ADDRESS_MAINNET, QUIC_DEFAULT_PORT}; + + if (http_config.pubkey_hex == TESTNET_CONFIG.pubkey_hex && netid == opt::netid::Target::testnet) + return SRouterTarget{QUIC_FS_SESH_ADDRESS_TESTNET, QUIC_DEFAULT_PORT}; + + return std::nullopt; +} + // The port a url does not need to state, because the scheme already implies it. static uint16_t default_port_for_scheme(std::string_view scheme) { return (scheme == "https" ? 443 : 80); } -std::string generate_download_url(std::string_view file_id, const config::FileServer& config) { - const auto has_custom_pubkey = (config.pubkey_hex != file_server::DEFAULT_CONFIG.pubkey_hex); +std::string generate_download_url( + std::string_view file_id, const config::FileServer& config, bool stream_encrypted) { + // An empty pubkey means "no custom server", not "a custom server with no key": emitting `p=` + // for it produces a url that names a key it does not carry, which the parser cannot accept. + // Other clients read an empty custom pubkey the same way. + const auto has_custom_pubkey = !config.pubkey_hex.empty() && + config.pubkey_hex != file_server::DEFAULT_CONFIG.pubkey_hex; // Omitted when the scheme already implies it, so urls for the default file server are // byte-identical to those any other client produces for it. @@ -146,16 +198,29 @@ std::string generate_download_url(std::string_view file_id, const config::FileSe port_suffix, fmt::format(file_server::ENDPOINT_FILE_INDIVIDUAL, file_id)); - if (config.use_stream_encryption || has_custom_pubkey) { - buf += "#"; + // Fragments are appended straight onto the url; `sep` starts the list with '#' and joins the + // rest with '&'. + auto out = std::back_inserter(buf); + char sep = '#'; - if (has_custom_pubkey) - buf += fmt::format("{}={}", backends::FRAGMENT_PUBKEY, config.pubkey_hex); + if (has_custom_pubkey) { + fmt::format_to(out, "{}{}={}", sep, backends::FRAGMENT_PUBKEY, config.pubkey_hex); + sep = '&'; + } - if (config.use_stream_encryption) { - buf += (has_custom_pubkey ? "&" : ""); - buf += backends::FRAGMENT_STREAM_ENCRYPTION; - } + if (stream_encrypted) { + fmt::format_to(out, "{}{}", sep, backends::FRAGMENT_STREAM_ENCRYPTION); + sep = '&'; + } + + // Only a custom server needs to name its QUIC endpoint: the built-in ones are resolved from the + // network the recipient is on. The port is left off when it is the default, since whoever + // parses this fills in the same default. + if (config.srouter) { + fmt::format_to(out, "{}{}={}", sep, backends::FRAGMENT_SROUTER, config.srouter->address); + if (config.srouter->port != QUIC_DEFAULT_PORT) + fmt::format_to(out, ":{}", config.srouter->port); + sep = '&'; } return buf; @@ -175,7 +240,7 @@ Request to_request( const std::string& upload_id, const config::FileServer& config, UploadRequest upload_request) { - std::vector all_data; + std::vector all_data; while (true) { if (upload_request.is_cancelled()) @@ -215,8 +280,7 @@ Request to_request( ServerDestination{ config.scheme, config.host, - compute_x25519_pubkey( - to_span(oxenc::from_hex(config.pubkey_hex))), + compute_x25519_pubkey(ed25519_pubkey::from_hex(config.pubkey_hex)), config.port, std::move(headers), "POST"}, @@ -252,7 +316,7 @@ Request to_request( ServerDestination{ std::move(scheme), std::move(host), - compute_x25519_pubkey(to_span(oxenc::from_hex(pubkey_hex))), + compute_x25519_pubkey(ed25519_pubkey::from_hex(pubkey_hex)), port, std::nullopt, "GET"}, @@ -289,7 +353,7 @@ file_metadata parse_upload_response(const std::string& body, size_t upload_size) return metadata; } -std::pair> parse_download_response( +std::pair> parse_download_response( std::string_view download_url, const std::vector>& headers, const std::string& body) { @@ -312,7 +376,7 @@ std::pair> parse_download_response( } } - std::vector data(body.begin(), body.end()); + auto data = to_vector(body); if (metadata.size == 0) metadata.size = data.size(); @@ -334,8 +398,7 @@ Request extend_ttl( ServerDestination{ config.scheme, config.host, - compute_x25519_pubkey( - to_span(oxenc::from_hex(config.pubkey_hex))), + compute_x25519_pubkey(ed25519_pubkey::from_hex(config.pubkey_hex)), config.port, std::move(headers), "POST"}, @@ -360,23 +423,17 @@ Request get_client_version( } // Generate the auth signature - auto blinded_keys = blind_version_key_pair(to_span(seckey.view())); - auto timestamp = epoch_seconds(std::chrono::system_clock::now()); - auto signature = blind_version_sign(to_span(seckey.view()), platform, timestamp); - auto pubkey = compute_x25519_pubkey( - to_span(oxenc::from_hex(DEFAULT_CONFIG.pubkey_hex))); - std::string blinded_pk_hex; - blinded_pk_hex.reserve(66); - blinded_pk_hex += "07"; - oxenc::to_hex( - blinded_keys.first.begin(), - blinded_keys.first.end(), - std::back_inserter(blinded_pk_hex)); + auto sk = ed25519::PrivKeySpan::from(to_span(seckey.view())); + auto blinded_keys = blind_version_key_pair(sk); + auto timestamp = epoch_seconds(clock_now_s()); + auto signature = blind_version_sign(sk, platform, timestamp); + auto pubkey = compute_x25519_pubkey(ed25519_pubkey::from_hex(DEFAULT_CONFIG.pubkey_hex)); + auto blinded_pk_hex = "07{:x}"_format(blinded_keys.first); auto headers = std::vector>{}; headers.emplace_back(HEADER_PUBKEY, blinded_pk_hex); headers.emplace_back(HEADER_TIMESTAMP, "{}"_format(timestamp)); - headers.emplace_back(HEADER_SIGNATURE, oxenc::to_base64(signature.begin(), signature.end())); + headers.emplace_back(HEADER_SIGNATURE, oxenc::to_base64(signature)); return Request{ random::unique_id("GCV"), @@ -443,9 +500,8 @@ LIBSESSION_C_API bool session_file_server_generate_download_url( config.port = port; if (pubkey_hex) config.pubkey_hex = pubkey_hex; - config.use_stream_encryption = use_stream_encryption; - auto result = file_server::generate_download_url(file_id, config); + auto result = file_server::generate_download_url(file_id, config, use_stream_encryption); if (result.size() >= out_url_len) return false; @@ -461,7 +517,7 @@ LIBSESSION_C_API session_request_params* session_file_server_get_client_version( try { auto req = file_server::get_client_version( static_cast(platform), - network::ed25519_seckey::from_bytes({ed25519_secret, 64}), + network::ed25519_seckey::from_bytes(to_byte_span<64>(ed25519_secret)), std::chrono::milliseconds{request_timeout_ms}, (overall_timeout_ms > 0 ? std::optional{std::chrono::milliseconds{overall_timeout_ms}} diff --git a/src/network/ip_country/data.hpp b/src/network/ip_country/data.hpp new file mode 100644 index 000000000..878df68c6 --- /dev/null +++ b/src/network/ip_country/data.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +namespace session::ip_country::detail { + +/// The bundled database, as a tiling of the IPv4 space: `range_starts()` holds the first address of +/// each range in ascending order and `range_codes()` the country of each, so a range runs until the +/// next one starts and no end column is needed. Both are empty when built without +/// `WITH_IP_GEOLOCATION`, which is what makes every lookup a miss in that build without the lookup +/// code itself knowing anything about the option. +/// +/// Exactly one of `data.cpp` and `no_data.cpp` is compiled in, chosen by that option. `data.cpp` +/// is not in git: `utils/update-ip-country-db.py` downloads a DB-IP release and generates it, and +/// cmake refuses to configure with the option on until it has been run. + +/// First address of each range, ascending, starting at 0.0.0.0. This is the only array a lookup +/// binary searches; the table's size rests on `ipv4` being nothing but its uint32_t. +static_assert(sizeof(ipv4) == sizeof(uint32_t)); +std::span range_starts(); + +/// Country of the range at the same index in `range_starts()`, as an index into +/// `country_codes()`; index 0 means unassigned or reserved. +/// +/// The uint8_t element caps the code table at 256 entries (246 are in use). Widening it is a +/// change to this type, to the array in the generated data, and to the generator's own check. +std::span range_codes(); + +/// The country code table that `range_codes()` indexes: two-letter ISO 3166-1 alpha-2 codes, +/// sorted, with the empty "unknown" code at index 0. +std::span country_codes(); + +/// The attribution required by the database's licence, empty when no database is bundled. +std::string_view attribution(); + +/// The bundled release, e.g. "dbip-country-lite-2026-09", empty when no database is bundled. +std::string_view database_version(); + +} // namespace session::ip_country::detail diff --git a/src/network/ip_country/lookup.cpp b/src/network/ip_country/lookup.cpp new file mode 100644 index 000000000..600e52277 --- /dev/null +++ b/src/network/ip_country/lookup.cpp @@ -0,0 +1,35 @@ +#include +#include + +#include "data.hpp" + +namespace session::ip_country { + +bool available() { + return !detail::range_starts().empty(); +} + +std::optional lookup(oxen::quic::ipv4 ip) { + auto starts = detail::range_starts(); + auto next = std::ranges::upper_bound(starts, ip); + // The table tiles the whole address space from 0.0.0.0 up, so the only way not to land in a + // range is for there to be no ranges at all, i.e. a build without the bundled database. + if (next == starts.begin()) + return std::nullopt; + + auto code = detail::range_codes()[next - starts.begin() - 1]; + if (code == 0) + return std::nullopt; + + return detail::country_codes()[code]; +} + +std::string_view attribution() { + return detail::attribution(); +} + +std::string_view database_version() { + return detail::database_version(); +} + +} // namespace session::ip_country diff --git a/src/network/ip_country/no_data.cpp b/src/network/ip_country/no_data.cpp new file mode 100644 index 000000000..7732414aa --- /dev/null +++ b/src/network/ip_country/no_data.cpp @@ -0,0 +1,28 @@ +#include "data.hpp" + +// The database compiled in when WITH_IP_GEOLOCATION is off: an empty one, so that lookups miss +// rather than the API disappearing. See data.hpp. + +namespace session::ip_country::detail { + +std::span range_starts() { + return {}; +} + +std::span range_codes() { + return {}; +} + +std::span country_codes() { + return {}; +} + +std::string_view attribution() { + return {}; +} + +std::string_view database_version() { + return {}; +} + +} // namespace session::ip_country::detail diff --git a/src/network/key_types.cpp b/src/network/key_types.cpp index 1e65a0482..959c796af 100644 --- a/src/network/key_types.cpp +++ b/src/network/key_types.cpp @@ -3,9 +3,11 @@ #include #include #include -#include #include +#include +#include +#include #include namespace session::network { @@ -17,16 +19,15 @@ namespace detail { throw std::runtime_error{"Hex key data is invalid: data is not hex"}; if (hex.size() != 2 * length) throw std::runtime_error{ - "Hex key data is invalid: expected " + std::to_string(length) + - " hex digits, received " + std::to_string(hex.size())}; + "Hex key data is invalid: expected {} hex digits, received {}"_format( + length, hex.size())}; oxenc::from_hex(hex.begin(), hex.end(), reinterpret_cast(buffer)); } void load_from_bytes(void* buffer, size_t length, std::string_view bytes) { if (bytes.size() != length) - throw std::runtime_error{ - "Key data is invalid: expected " + std::to_string(length) + - " bytes, received " + std::to_string(bytes.size())}; + throw std::runtime_error{"Key data is invalid: expected {} bytes, received {}"_format( + length, bytes.size())}; std::memmove(buffer, bytes.data(), length); } @@ -40,17 +41,17 @@ std::string ed25519_pubkey::snode_address() const { legacy_pubkey legacy_seckey::pubkey() const { legacy_pubkey pk; - crypto_scalarmult_ed25519_base_noclamp(pk.data(), data()); + ed25519::scalarmult_base_noclamp(pk, *this); return pk; }; ed25519_pubkey ed25519_seckey::pubkey() const { ed25519_pubkey pk; - crypto_sign_ed25519_sk_to_pk(pk.data(), data()); + ed25519::sk_to_pk(pk, ed25519::PrivKeySpan::from(*this)); return pk; }; x25519_pubkey x25519_seckey::pubkey() const { x25519_pubkey pk; - crypto_scalarmult_curve25519_base(pk.data(), data()); + x25519::scalarmult_base(pk, *this); return pk; }; @@ -81,13 +82,8 @@ ed25519_pubkey parse_ed25519_pubkey(std::string_view pubkey_in) { x25519_pubkey parse_x25519_pubkey(std::string_view pubkey_in) { return parse_pubkey(pubkey_in); } -x25519_pubkey compute_x25519_pubkey(std::span ed25519_pk) { - std::array xpk; - if (0 != crypto_sign_ed25519_pk_to_curve25519(xpk.data(), ed25519_pk.data())) - throw std::runtime_error{ - "An error occured while attempting to convert Ed25519 pubkey to X25519; " - "is the pubkey valid?"}; - return x25519_pubkey::from_bytes({xpk.data(), 32}); +x25519_pubkey compute_x25519_pubkey(std::span ed25519_pk) { + return x25519_pubkey::from_bytes(ed25519::pk_to_x25519(ed25519_pk)); } } // namespace session::network diff --git a/src/network/network_config.cpp b/src/network/network_config.cpp index e97aca9ea..73b3b25b5 100644 --- a/src/network/network_config.cpp +++ b/src/network/network_config.cpp @@ -97,12 +97,31 @@ void Config::handle_config_opt(opt::file_server_max_file_size fsmfs) { cat, "Network config custom file server max file size set to {}", fsmfs.max_file_size); } -void Config::handle_config_opt(opt::file_server_use_stream_encryption fsuse) { - file_server_use_stream_encryption = fsuse.use_stream_encryption; +void Config::handle_config_opt(opt::file_server_srouter fssr) { + custom_file_server_srouter_address = fssr.address; + custom_file_server_srouter_port = fssr.port; log::debug( cat, - "Network config file use stream encryption set to {}", - fsuse.use_stream_encryption); + "Network config custom file server session router endpoint set to {}:{}", + fssr.address, + fssr.port ? "{}"_format(*fssr.port) : ""); +} + +// MARK: QUIC file server options + +void Config::handle_config_opt(opt::quic_file_server_ed_pubkey qfep) { + quic_file_server_ed_pubkey = std::move(qfep.pubkey_hex); + log::debug(cat, "Network config QUIC file server Ed25519 pubkey set"); +} + +void Config::handle_config_opt(opt::quic_file_server_address qfa) { + quic_file_server_address = std::move(qfa.address); + log::debug(cat, "Network config QUIC file server address set to {}", *quic_file_server_address); +} + +void Config::handle_config_opt(opt::quic_file_server_port qfp) { + quic_file_server_port = qfp.port; + log::debug(cat, "Network config QUIC file server port set to {}", qfp.port); } // MARK: General options @@ -235,9 +254,9 @@ void Config::handle_config_opt(opt::quic_keep_alive qka) { log::debug(cat, "Network config quic keep alive set to {}s", qka.duration.count()); } -void Config::handle_config_opt(opt::quic_disable_mtu_discovery) { - quic_disable_mtu_discovery = true; - log::debug(cat, "Network config disabled MTU discovery for Quic"); +void Config::handle_config_opt(opt::quic_max_udp_payload qmup) { + quic_max_udp_payload = qmup.size; + log::debug(cat, "Network config max QUIC UDP payload set to {} bytes", qmup.size); } // MARK: Onion Request Router Options diff --git a/src/network/request_queue.cpp b/src/network/request_queue.cpp index ee64dcfab..9164617d9 100644 --- a/src/network/request_queue.cpp +++ b/src/network/request_queue.cpp @@ -18,9 +18,18 @@ namespace { } RequestQueue::~RequestQueue() { - _timeout.reset(); + // Runs whatever adds are already queued before cancelling them below, so that a request that + // arrived just before this destructor still gets told it is not going to be sent. Dropping + // those jobs instead would drop their callbacks with them, and the caller would hear nothing. + _jq.call_get([] {}); + _jq.stop(); + + // The timeout event is the loop's to fire and reaches `this`, so it has to be taken away on the + // loop thread rather than from here. The cancellations go in the same job because that is + // where they have always been called from. + _loop.call_get([this] { + _timeout.reset(); - _loop->call_get([this] { for (auto& [id, request_pair] : _requests) { auto& [req, callback] = request_pair; @@ -38,14 +47,12 @@ RequestQueue::~RequestQueue() { } void RequestQueue::add(Request request, network_response_callback_t callback) { - _loop->call([self = shared_from_this(), - req = std::move(request), - cb = std::move(callback)]() mutable { + _jq.call([this, req = std::move(request), cb = std::move(callback)]() mutable { const auto req_id = req.request_id; const auto creation_time = req.creation_time; const auto timeout = req.overall_timeout; - self->_requests.emplace(req_id, std::make_pair(std::move(req), std::move(cb))); - self->_queue.emplace_back(req_id); + _requests.emplace(req_id, std::make_pair(std::move(req), std::move(cb))); + _queue.emplace_back(req_id); if (timeout) { auto expiry = creation_time + *timeout; @@ -53,24 +60,24 @@ void RequestQueue::add(Request request, network_response_callback_t callback) { // We hint at the end because it is an extremely common pattern that you use the same // timeout for all (or most) requests in which case each new request timeout *does* land // at the end. - self->_req_expiries.emplace_hint(self->_req_expiries.end(), expiry, req_id); + _req_expiries.emplace_hint(_req_expiries.end(), expiry, req_id); // If the expiry entry landed at the beginning of the map -- either because it was // empty, or because this has a shorter timeout than what's already in there -- then we // need to (re)schedule the event to this request's timeout. - if (self->_req_expiries.begin()->second == req_id) - self->update_timeout(); + if (_req_expiries.begin()->second == req_id) + update_timeout(); } }); } void RequestQueue::add_front(std::pair req_pair) { - _loop->call([self = shared_from_this(), pair = std::move(req_pair)] { + _jq.call([this, pair = std::move(req_pair)] { const auto req_id = pair.first.request_id; const auto creation_time = pair.first.creation_time; const auto timeout = pair.first.overall_timeout; - self->_requests.emplace(req_id, std::move(pair)); - self->_queue.emplace_front(req_id); + _requests.emplace(req_id, std::move(pair)); + _queue.emplace_front(req_id); if (timeout) { auto expiry = creation_time + *timeout; @@ -78,34 +85,34 @@ void RequestQueue::add_front(std::pair req // We hint at the end because it is an extremely common pattern that you use the same // timeout for all (or most) requests in which case each new request timeout *does* land // at the end. - self->_req_expiries.emplace_hint(self->_req_expiries.end(), expiry, req_id); + _req_expiries.emplace_hint(_req_expiries.end(), expiry, req_id); // If the expiry entry landed at the beginning of the map -- either because it was // empty, or because this has a shorter timeout than what's already in there -- then we // need to (re)schedule the event to this request's timeout. - if (self->_req_expiries.begin()->second == req_id) - self->update_timeout(); + if (_req_expiries.begin()->second == req_id) + update_timeout(); } }); } std::deque> RequestQueue::pop_all() { - return _loop->call_get([self = shared_from_this()] { + return _jq.call_get([this] { std::deque> popped_items; - for (const auto& id : self->_queue) { - auto it = self->_requests.find(id); + for (const auto& id : _queue) { + auto it = _requests.find(id); - if (it != self->_requests.end()) { + if (it != _requests.end()) { popped_items.push_back(std::move(it->second)); - self->_requests.erase(it); + _requests.erase(it); } } - self->_queue.clear(); - self->_requests.clear(); - self->_req_expiries.clear(); - self->update_timeout(); + _queue.clear(); + _requests.clear(); + _req_expiries.clear(); + update_timeout(); return popped_items; }); @@ -148,7 +155,7 @@ void RequestQueue::update_timeout() { if (!_timeout) { // If this is the first request timeout then set up the timeout event timer: _timeout.reset(event_new( - _loop->get_event_base(), + _loop.get_event_base(), -1, // Not attached to an actual socket EV_TIMEOUT, // Stays active (i.e. repeats) once fired [](evutil_socket_t, short, void* self) { diff --git a/src/network/routing/direct_router.cpp b/src/network/routing/direct_router.cpp index 6cc09f768..1af3071f2 100644 --- a/src/network/routing/direct_router.cpp +++ b/src/network/routing/direct_router.cpp @@ -92,6 +92,49 @@ void DirectRouter::upload(UploadRequest request) { }); } +void DirectRouter::upload_file(FileUploadRequest request, std::span seed) { + if (!_config.quic_file_server_address || !_config.quic_file_server_ed_pubkey) { + if (request.on_complete) + request.on_complete(ERROR_FILE_SERVER_UNAVAILABLE, false); + return; + } + + attachment::Encryptor enc{seed, request.domain}; + auto address = *_config.quic_file_server_address; + auto pubkey_hex = *_config.quic_file_server_ed_pubkey; + auto port = _config.quic_file_server_port; + const auto upload_id = random::unique_id("UPL"); + + auto& upload_thread = + _active_uploads.emplace(upload_id, std::make_pair(UploadRequest{}, std::thread{})) + .first->second.second; + + upload_thread = std::thread([weak_self = weak_from_this(), + this, + enc = std::move(enc), + request = std::move(request), + address, + pubkey_hex, + port, + upload_id]() mutable { + streaming_file_upload( + _loop, + std::move(enc), + std::move(request), + [weak_self, this, address, pubkey_hex, port]() -> QuicFileClient* { + auto self = weak_self.lock(); + if (!self) + return nullptr; + return &_get_file_client(ed25519_pubkey::from_hex(pubkey_hex), address, port); + }); + + _loop->call([weak_self = weak_from_this(), this, upload_id] { + if (auto self = weak_self.lock()) + _cleanup_upload(upload_id); + }); + }); +} + void DirectRouter::download(DownloadRequest request) { _loop->call([weak_self = weak_from_this(), req = std::move(request)] { if (auto self = weak_self.lock()) @@ -165,21 +208,148 @@ void DirectRouter::_send_request_internal(Request request, network_response_call }); } +void DirectRouter::_cleanup_upload(const std::string& upload_id) { + auto node = _active_uploads.extract(upload_id); + if (!node.empty()) { + auto& thread = node.mapped().second; + if (thread.joinable()) + thread.join(); + } +} + +QuicFileClient& DirectRouter::_get_file_client( + const ed25519_pubkey& pubkey, std::string_view address, uint16_t port) { + auto [it, inserted] = _file_clients.try_emplace(pubkey, nullptr); + if (inserted) + it->second = std::make_unique(_loop, pubkey, std::string{address}, port); + else + it->second->set_target(pubkey, std::string{address}, port); + return *it->second; +} + void DirectRouter::_upload_internal(UploadRequest request) { const std::string upload_id = random::unique_id("UP"); log::info(cat, "[Upload {}]: Starting upload.", upload_id); - // Make the callback atomic so we don't need to worry about it being called multiple times (eg. - // network shutdown cancelling the request and the transport shutdown automatically triggering - // callbacks) request.on_complete = make_callback_atomic(std::move(request.on_complete)); - auto& [_, upload_thread] = - _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) - .first->second; - // Accumulate data on a background thread as we don't know whether `next_data` is doing file I/O - // or just reading from memory (it's a bit of a waste if it's in-memory data but loading from - // disk should be prioritised) + // Use the QUIC file server path if configured, otherwise fall back to the legacy HTTP path + if (!_config.quic_file_server_address || !_config.quic_file_server_ed_pubkey) { + _upload_internal_legacy(std::move(request), std::move(upload_id)); + return; + } + + auto& upload_thread = _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) + .first->second.second; + + auto address = *_config.quic_file_server_address; + auto pubkey_hex = *_config.quic_file_server_ed_pubkey; + auto port = _config.quic_file_server_port; + + upload_thread = std::thread([weak_self = weak_from_this(), + this, + upload_request = request, + upload_id, + address, + pubkey_hex, + port] { + auto self = weak_self.lock(); + if (!self) + return; + + try { + std::vector all_data; + while (true) { + if (upload_request.is_cancelled()) + throw cancellation_exception{"Cancelled during data accumulation."}; + auto chunk = upload_request.next_data(); + if (chunk.empty()) + break; + auto* p = reinterpret_cast(chunk.data()); + all_data.insert(all_data.end(), p, p + chunk.size()); + } + + if (all_data.empty()) + throw std::runtime_error{"No data to upload"}; + + log::debug( + cat, + "[Upload {}]: Accumulated {} bytes, uploading to {}:{}.", + upload_id, + all_data.size(), + address, + port); + + _loop->call([weak_self, + this, + upload_request, + upload_id, + address, + pubkey_hex, + port, + data = std::move(all_data)]() mutable { + auto self = weak_self.lock(); + if (!self) + return; + + if (upload_request.is_cancelled()) { + upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); + _cleanup_upload(upload_id); + return; + } + + auto pubkey = ed25519_pubkey::from_hex(pubkey_hex); + auto& client = _get_file_client(pubkey, address, port); + + client.upload( + std::move(data), + upload_request.ttl, + [weak_self, this, upload_request, upload_id]( + std::variant result) { + auto self = weak_self.lock(); + if (!self) + return; + + if (auto* meta = std::get_if(&result)) + log::info( + cat, + "[Upload {}]: Success, file ID: {}", + upload_id, + meta->id); + else + log::error( + cat, + "[Upload {}]: Failed with error {}", + upload_id, + std::get(result)); + + upload_request.on_complete(std::move(result), false); + _cleanup_upload(upload_id); + }); + }); + } catch (const cancellation_exception&) { + _loop->call([weak_self = weak_from_this(), this, upload_request, upload_id] { + if (auto self = weak_self.lock()) { + upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); + _cleanup_upload(upload_id); + } + }); + } catch (const std::exception& e) { + log::error(cat, "[Upload {}]: Exception: {}", upload_id, e.what()); + _loop->call([weak_self = weak_from_this(), this, upload_request, upload_id] { + if (auto self = weak_self.lock()) { + upload_request.on_complete(ERROR_UNKNOWN, false); + _cleanup_upload(upload_id); + } + }); + } + }); +} + +void DirectRouter::_upload_internal_legacy(UploadRequest request, std::string upload_id) { + auto& upload_thread = _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) + .first->second.second; + upload_thread = std::thread([weak_self = weak_from_this(), this, upload_request = request, @@ -189,8 +359,6 @@ void DirectRouter::_upload_internal(UploadRequest request) { if (!self) return; - // Onion requests don't support streaming data so we need to load all the data from the - // streaming source into memory try { Request request = file_server::to_request(upload_id, file_server_config, upload_request); @@ -203,11 +371,7 @@ void DirectRouter::_upload_internal(UploadRequest request) { if (upload_request.is_cancelled() || !req.body) { log::debug(cat, "[Upload {}]: Cancelled before sending request.", upload_id); upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); - - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); + _cleanup_upload(upload_id); return; } @@ -230,11 +394,7 @@ void DirectRouter::_upload_internal(UploadRequest request) { if (!self) return; - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); + _cleanup_upload(upload_id); try { if (upload_request.is_cancelled()) @@ -290,12 +450,7 @@ void DirectRouter::_upload_internal(UploadRequest request) { auto self = weak_self.lock(); if (!self) return; - - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); - + _cleanup_upload(upload_id); upload_request.on_complete(ERROR_UNKNOWN, false); }); } @@ -306,10 +461,63 @@ void DirectRouter::_download_internal(DownloadRequest request) { const std::string download_id = random::unique_id("DL"); log::info(cat, "[Download {}]: Starting download.", download_id); - // Make the callback atomic so we don't need to worry about it being called multiple times (eg. - // network shutdown cancelling the request and the transport shutdown automatically triggering - // callbacks) request.on_complete = make_callback_atomic(std::move(request.on_complete)); + + if (!_config.quic_file_server_address || !_config.quic_file_server_ed_pubkey) { + _download_internal_legacy(std::move(request), std::move(download_id)); + return; + } + + // QUIC download: parse file_id from URL, connect directly to configured file server + auto download_info = file_server::parse_download_url(request.download_url); + if (!download_info) { + log::error( + cat, "[Download {}]: Invalid download URL: {}", download_id, request.download_url); + request.on_complete(ERROR_INVALID_DOWNLOAD_URL, false); + return; + } + + _active_downloads[download_id] = request; + auto file_id = download_info->file_id; + auto address = *_config.quic_file_server_address; + auto pubkey = ed25519_pubkey::from_hex(*_config.quic_file_server_ed_pubkey); + auto port = _config.quic_file_server_port; + + auto& client = _get_file_client(pubkey, address, port); + + log::debug( + cat, "[Download {}]: Downloading {} from {}:{}.", download_id, file_id, address, port); + + client.download( + std::move(file_id), + request.on_data, + [weak_self = weak_from_this(), this, request, download_id]( + std::variant result) { + auto self = weak_self.lock(); + if (!self) + return; + + _active_downloads.erase(download_id); + + if (auto* meta = std::get_if(&result)) + log::info( + cat, + "[Download {}]: Success, file ID: {} ({} bytes)", + download_id, + meta->id, + meta->size); + else + log::error( + cat, + "[Download {}]: Failed with error {}", + download_id, + std::get(result)); + + request.on_complete(std::move(result), false); + }); +} + +void DirectRouter::_download_internal_legacy(DownloadRequest request, std::string download_id) { _active_downloads[download_id] = request; try { @@ -355,7 +563,7 @@ void DirectRouter::_download_internal(DownloadRequest request) { metadata.id); if (request.on_data) - request.on_data(metadata, std::move(data)); + request.on_data(metadata, to_span(data)); request.on_complete(std::move(metadata), false); } catch (const cancellation_exception&) { diff --git a/src/network/routing/onion_request_router.cpp b/src/network/routing/onion_request_router.cpp index 99f6c4549..5b1b4e02a 100644 --- a/src/network/routing/onion_request_router.cpp +++ b/src/network/routing/onion_request_router.cpp @@ -150,7 +150,7 @@ namespace { std::optional>> parse_error_response( uint16_t status_code, const std::optional& error_body, - std::optional> destination_pubkey) { + std::optional> destination_pubkey) { for (const auto& pattern : error_patterns) { if (pattern.code != status_code) continue; @@ -288,8 +288,8 @@ cached_edge_node cached_edge_node::from_disk(std::string_view str) { OnionRequestRouter::OnionRequestRouter( config::OnionRequestRouter config, - std::shared_ptr loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, std::weak_ptr snode_pool, std::weak_ptr transport) : _config{std::move(config)}, @@ -299,8 +299,8 @@ OnionRequestRouter::OnionRequestRouter( _transport{transport} { log::trace(cat, "Initializing."); - _request_queues[PathCategory::standard] = detail::RequestQueue::make(_loop); - _request_queues[PathCategory::file] = detail::RequestQueue::make(_loop); + _request_queues[PathCategory::standard] = std::make_shared(_loop); + _request_queues[PathCategory::file] = std::make_shared(_loop); if (_config.cache_directory) { std::string cache_file_name; @@ -314,8 +314,9 @@ OnionRequestRouter::OnionRequestRouter( for (const auto& node : _config.seed_nodes) node.to_disk(std::back_inserter(seed_node_data)); - auto hash_bytes = session::hash::hash(32, session::to_span(seed_node_data)); - cache_file_name = "edge_nodes_devnet_" + oxenc::to_hex(hash_bytes); + cache_file_name = + "edge_nodes_devnet_" + + oxenc::to_hex(session::hash::blake2b<32>(session::to_span(seed_node_data))); break; } @@ -323,7 +324,7 @@ OnionRequestRouter::OnionRequestRouter( _load_from_disk(); } - _loop->call_soon([this] { + _jq.call_soon([this] { auto snode_pool = _snode_pool.lock(); if (!snode_pool) { log::critical(cat, "SnodePool was destroyed, cannot setup router."); @@ -331,12 +332,10 @@ OnionRequestRouter::OnionRequestRouter( } if (snode_pool->size() == 0) - snode_pool->refresh_if_needed({}, [weak_self = weak_from_this()] { + // The pool outlives us, so this one keeps a weak guard rather than a bare `this`. + snode_pool->refresh_if_needed({}, [weak_self = weak_from_this(), this] { if (auto self = weak_self.lock()) - self->_loop->call([weak_self] { - if (auto self = weak_self.lock()) - self->_finish_setup(); - }); + _jq.call([this] { _finish_setup(); }); }); else _finish_setup(); @@ -344,9 +343,15 @@ OnionRequestRouter::OnionRequestRouter( } OnionRequestRouter::~OnionRequestRouter() { - // Use 'call_get' to force this to be synchronous - if (_loop) - _loop->call_get([this] { _close_connections(); }); + // Both halves go in one job on the loop's own queue, in this order and for this reason: the + // upload threads post their completion onto _jq, and posting to a stopped queue throws -- + // which, on a thread of ours, is a terminate. _close_connections joins those threads, so once + // it returns nothing can post any more and the queue can be stopped. Being a single job also + // leaves no window in which a job of ours could run against a half-torn-down router. + _loop.call_get([this] { + _close_connections(); + _jq.stop(); + }); log::debug(cat, "Destroyed."); } @@ -478,22 +483,20 @@ void OnionRequestRouter::_perform_edge_node_write( void OnionRequestRouter::suspend() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _suspended = true; // Write the edge nodes to disk before suspension completes - if (_disk_loop) { - std::vector edge_nodes; + std::vector edge_nodes; - for (const auto& path_list : std::views::values(_paths)) - for (const auto& path : path_list) - if (!path.nodes.empty()) - edge_nodes.emplace_back(path.nodes[0], path.edge_first_connected_at); + for (const auto& path_list : std::views::values(_paths)) + for (const auto& path : path_list) + if (!path.nodes.empty()) + edge_nodes.emplace_back(path.nodes[0], path.edge_first_connected_at); - _disk_loop->call([path = _edge_node_cache_file_path, nodes = std::move(edge_nodes)] { - OnionRequestRouter::_perform_edge_node_write(path, nodes); - }); - } + _disk_loop.call([path = _edge_node_cache_file_path, nodes = std::move(edge_nodes)] { + OnionRequestRouter::_perform_edge_node_write(path, nodes); + }); _close_connections(); log::info(cat, "Suspended."); @@ -502,7 +505,7 @@ void OnionRequestRouter::suspend() { void OnionRequestRouter::resume(bool automatically_reconnect) { // Use 'call_get' to force this to be synchronous - _loop->call_get([this, automatically_reconnect] { + _jq.call_get([this, automatically_reconnect] { if (!_suspended) return; @@ -517,22 +520,22 @@ void OnionRequestRouter::resume(bool automatically_reconnect) { void OnionRequestRouter::close_connections() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { _close_connections(); }); + _jq.call_get([this] { _close_connections(); }); } void OnionRequestRouter::clear_cache() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _cached_edge_nodes = {}; - _disk_loop->call([path = _edge_node_cache_file_path] { + _disk_loop.call([path = _edge_node_cache_file_path] { OnionRequestRouter::_clear_disk_cache(path); }); }); } std::vector OnionRequestRouter::get_active_paths() { - return _loop->call_get([this] { + return _jq.call_get([this] { std::vector result; result.reserve(_paths.size()); @@ -545,28 +548,88 @@ std::vector OnionRequestRouter::get_active_paths() { } std::vector OnionRequestRouter::get_all_used_nodes() { - return _loop->call_get([this] { return extract_nodes(_paths, _pending_paths); }); + return _jq.call_get([this] { return extract_nodes(_paths, _pending_paths); }); } void OnionRequestRouter::send_request(Request request, network_response_callback_t callback) { - _loop->call([weak_self = weak_from_this(), req = std::move(request), cb = std::move(callback)] { - if (auto self = weak_self.lock()) - self->_send_request_internal(std::move(req), std::move(cb)); + _jq.call([this, req = std::move(request), cb = std::move(callback)]() mutable { + _send_request_internal(std::move(req), std::move(cb)); }); } void OnionRequestRouter::upload(UploadRequest request) { - _loop->call([weak_self = weak_from_this(), req = std::move(request)] { - if (auto self = weak_self.lock()) - self->_upload_internal(std::move(req)); + _jq.call([this, req = std::move(request)]() mutable { _upload_internal(std::move(req)); }); +} + +void OnionRequestRouter::upload_file(FileUploadRequest request, std::span seed) { + attachment::Encryptor enc{seed, request.domain}; + const auto upload_id = random::unique_id("UPL"); + + auto& upload_thread = + _active_uploads.emplace(upload_id, std::make_pair(UploadRequest{}, std::thread{})) + .first->second.second; + + // This thread is ours: _close_connections joins it before the queue is stopped, so it outlives + // neither us nor the queue it posts to. + upload_thread = std::thread([this, + enc = std::move(enc), + request = std::move(request), + upload_id, + file_server_config = _config.file_server_config]() mutable { + try { + auto key = enc.load_key_from_file(request.file, request.allow_large); + auto enc_size = attachment::encrypted_size(enc.data_size()); + + // Accumulate all encrypted output into a buffer (onion requests require the + // full payload upfront). + std::vector all_data; + all_data.reserve(enc_size); + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) + all_data.insert(all_data.end(), chunk.begin(), chunk.end()); + + // Build the one-shot Request via to_request (needs an UploadRequest with next_data) + UploadRequest legacy_req; + legacy_req.request_timeout = request.request_timeout; + legacy_req.overall_timeout = request.overall_timeout; + legacy_req.stall_timeout = request.stall_timeout; + legacy_req.ttl = request.ttl; + auto data_ptr = std::make_shared>(std::move(all_data)); + bool consumed = false; + legacy_req.next_data = [data_ptr, consumed]() mutable -> std::vector { + if (consumed) + return {}; + consumed = true; + return std::move(*data_ptr); + }; + + auto req = file_server::to_request(upload_id, file_server_config, legacy_req); + + // Wrap the FileUploadRequest callback to inject the key on success + _dispatch_upload( + upload_id, + std::move(req), + [request] { return request.is_cancelled(); }, + [request, key](std::variant result, bool timeout) { + if (!request.on_complete) + return; + if (auto* meta = std::get_if(&result)) + request.on_complete(std::make_pair(std::move(*meta), key), timeout); + else + request.on_complete(std::get(result), timeout); + }); + } catch (const std::exception& e) { + log::error(cat, "[Upload {}]: File upload failed: {}", upload_id, e.what()); + _jq.call([this, request, upload_id] { + _cleanup_upload(upload_id); + if (request.on_complete) + request.on_complete(ERROR_UNKNOWN, false); + }); + } }); } void OnionRequestRouter::download(DownloadRequest request) { - _loop->call([weak_self = weak_from_this(), req = std::move(request)] { - if (auto self = weak_self.lock()) - self->_download_internal(std::move(req)); - }); + _jq.call([this, req = std::move(request)]() mutable { _download_internal(std::move(req)); }); } // MARK: Internal Logic @@ -850,6 +913,95 @@ void OnionRequestRouter::_send_request_internal( } } +void OnionRequestRouter::_cleanup_upload(const std::string& upload_id) { + auto node = _active_uploads.extract(upload_id); + if (!node.empty()) { + auto& thread = node.mapped().second; + if (thread.joinable()) + thread.join(); + } +} + +void OnionRequestRouter::_dispatch_upload( + std::string upload_id, + Request req, + std::function is_cancelled, + std::function, bool)> on_result) { + _jq.call([this, + upload_id, + req = std::move(req), + is_cancelled = std::move(is_cancelled), + on_result = std::move(on_result)]() mutable { + if (is_cancelled() || !req.body) { + log::debug(cat, "[Upload {}]: Cancelled before sending request.", upload_id); + on_result(ERROR_REQUEST_CANCELLED, false); + _cleanup_upload(upload_id); + return; + } + + const auto upload_size = req.body->size(); + log::debug( + cat, "[Upload {}]: Accumulated {} bytes, sending request.", upload_id, upload_size); + + _send_request_internal( + std::move(req), + // Ends up held by the transport, which outlives us, so this one keeps a weak guard. + [weak_self = weak_from_this(), + this, + upload_id, + is_cancelled, + on_result, + upload_size]( + bool success, + bool timeout, + int16_t status_code, + std::vector> headers, + std::optional body) { + auto self = weak_self.lock(); + if (!self) + return; + + _cleanup_upload(upload_id); + + try { + if (is_cancelled()) + throw cancellation_exception{"Cancelled during request."}; + + if (!success || timeout) + throw status_code_exception{ + status_code, + headers, + fmt::format( + "Request failed with status {}, timeout={}.", + status_code, + timeout)}; + + if (!body) + throw std::runtime_error{"No response body."}; + + auto metadata = file_server::parse_upload_response(*body, upload_size); + log::info( + cat, + "[Upload {}]: Successfully uploaded {} bytes as file ID: {}", + upload_id, + metadata.size, + metadata.id); + + on_result(std::move(metadata), false); + } catch (const cancellation_exception&) { + log::error(cat, "[Upload {}]: Cancelled", upload_id); + on_result(ERROR_REQUEST_CANCELLED, false); + } catch (const status_code_exception& e) { + log::error(cat, "[Upload {}]: Failure with error: {}", upload_id, e.what()); + on_result(e.status_code, false); + } catch (const std::exception& e) { + log::error(cat, "[Upload {}]: Failure with error: {}", upload_id, e.what()); + on_result(ERROR_UNKNOWN, false); + } + }); + }); +} + void OnionRequestRouter::_upload_internal(UploadRequest request) { const std::string upload_id = random::unique_id("UP"); log::info(cat, "[Upload {}]: Starting upload.", upload_id); @@ -865,122 +1017,24 @@ void OnionRequestRouter::_upload_internal(UploadRequest request) { // Accumulate data on a background thread as we don't know whether `next_data` is doing file I/O // or just reading from memory (it's a bit of a waste if it's in-memory data but loading from // disk should be prioritised) - upload_thread = std::thread([weak_self = weak_from_this(), - this, + // Ours, and joined by _close_connections before the queue is stopped -- see upload_file. + upload_thread = std::thread([this, upload_request = request, upload_id, file_server_config = _config.file_server_config] { - auto self = weak_self.lock(); - if (!self) - return; - - // Onion requests don't support streaming data so we need to load all the data from the - // streaming source into memory try { - Request request = - file_server::to_request(upload_id, file_server_config, upload_request); + auto req = file_server::to_request(upload_id, file_server_config, upload_request); - _loop->call([weak_self, this, upload_request, req = std::move(request), upload_id] { - auto self = weak_self.lock(); - if (!self) - return; - - if (upload_request.is_cancelled() || !req.body) { - log::debug(cat, "[Upload {}]: Cancelled before sending request.", upload_id); - upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); - - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); - return; - } - - const auto upload_size = req.body->size(); - log::debug( - cat, - "[Upload {}]: Accumulated {} bytes, building request.", - upload_id, - upload_size); - - _send_request_internal( - std::move(req), - [weak_self, this, upload_id, upload_request, upload_size]( - bool success, - bool timeout, - int16_t status_code, - std::vector> headers, - std::optional body) { - auto self = weak_self.lock(); - if (!self) - return; - - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); - - try { - if (upload_request.is_cancelled()) - throw cancellation_exception{"Cancelled during request."}; - - if (!success || timeout) - throw status_code_exception{ - status_code, - headers, - fmt::format( - "Request failed with status {}, timeout={}.", - status_code, - timeout)}; - - if (!body) - throw std::runtime_error{"No response body."}; - - auto metadata = - file_server::parse_upload_response(*body, upload_size); - log::info( - cat, - "[Upload {}]: Successfully uploaded {} bytes as file ID: " - "{}", - upload_id, - metadata.size, - metadata.id); - - upload_request.on_complete(std::move(metadata), false); - } catch (const cancellation_exception&) { - log::error(cat, "[Upload {}]: Cancelled", upload_id); - upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); - } catch (const status_code_exception& e) { - log::error( - cat, - "[Upload {}]: Failure with error: {}", - upload_id, - e.what()); - upload_request.on_complete(e.status_code, false); - } catch (const std::exception& e) { - log::error( - cat, - "[Upload {}]: Failure with error: {}", - upload_id, - e.what()); - upload_request.on_complete(ERROR_UNKNOWN, false); - } - }); - }); + _dispatch_upload( + upload_id, + std::move(req), + [upload_request] { return upload_request.is_cancelled(); }, + upload_request.on_complete); } catch (const std::exception& e) { log::error(cat, "[Upload {}]: Exception during upload: {}", upload_id, e.what()); - _loop->call([weak_self, this, upload_request, upload_id] { - auto self = weak_self.lock(); - if (!self) - return; - - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); - + _jq.call([this, upload_request, upload_id] { + _cleanup_upload(upload_id); upload_request.on_complete(ERROR_UNKNOWN, false); }); } @@ -1040,7 +1094,7 @@ void OnionRequestRouter::_download_internal(DownloadRequest request) { metadata.id); if (request.on_data) - request.on_data(metadata, std::move(data)); + request.on_data(metadata, to_span(data)); request.on_complete(std::move(metadata), false); } catch (const cancellation_exception&) { @@ -1300,12 +1354,9 @@ void OnionRequestRouter::_on_edge_connectivity_response( _config.path_build_retry_limit); _update_status(); - _loop->call_later( - delay, - [weak_self = weak_from_this(), path_id, category, initiating_req_id, edge_node] { - if (auto self = weak_self.lock()) - self->_build_path(category, initiating_req_id, {edge_node}, path_id); - }); + _jq.call_later(delay, [this, path_id, category, initiating_req_id, edge_node] { + _build_path(category, initiating_req_id, {edge_node}, path_id); + }); return; } @@ -1522,7 +1573,7 @@ void OnionRequestRouter::_send_on_path( OnionPath& path, Request request, network_response_callback_t callback) { log::trace(cat, "[Request {}]: Sending on path {}", request.request_id, path.id); - std::vector encrypted_blob; + std::vector encrypted_blob; std::shared_ptr parser; try { @@ -2070,7 +2121,7 @@ void OnionRequestRouter::_update_rotation_timer() { if (!_path_rotation_timer) { // If this is the first request timeout then set up the timeout event timer: _path_rotation_timer.reset(event_new( - _loop->get_event_base(), + _loop.get_event_base(), -1, // Not attached to an actual socket EV_TIMEOUT, // Stays active (i.e. repeats) once fired [](evutil_socket_t, short, void* self) { diff --git a/src/network/routing/session_router_router.cpp b/src/network/routing/session_router_router.cpp index cf1606d05..683f2c2f8 100644 --- a/src/network/routing/session_router_router.cpp +++ b/src/network/routing/session_router_router.cpp @@ -9,6 +9,7 @@ #include #include +#include "session/crypto/ed25519.hpp" #include "session/network/network_opt.hpp" #include "session/onionreq/builder.hpp" #include "session/onionreq/response_parser.hpp" @@ -22,6 +23,38 @@ using namespace oxen::log::literals; namespace session::network { +// Holding the claim is what keeps the tunnel mapped; dropping it releases our hold, and Session +// Router tears the mapping down once nobody else is holding it either. +struct ActiveTunnel { + session::router::udp_tunnel tunnel; + + // Set once the session behind the mapping is up. Until then requests wait in + // `_pending_requests` rather than being written into a mapping with nothing to carry them. + bool established = false; +}; + +static std::optional pubkey_from_srouter_address(std::string_view address); + +// The inner QUIC connection's UDP payload size, fixed rather than derived from the tunnel's +// suggestion. +// +// The suggestion cannot be made to mean what it needs to. It is computed from the outer +// *endpoint's* configured max_udp_payload, which is a policy knob and not a measurement -- it is +// unset in the default configuration, so the calculation does not even run there. The two values +// that are real are no better suited: a connection's max_datagram_size is deliberately the size +// reachable *by splitting a packet in two*, so sizing the inner from it guarantees every inner +// packet splits, and the per-piece size that would actually avoid splitting belongs to a connection +// this layer does not hold. Nor would knowing it once be enough: it moves when the first hop +// changes, and nothing here would hear about that. +// +// So the inner is pinned to the one size QUIC guarantees every path carries. libquic splits +// datagrams that do not fit, so a larger value would buy throughput when the outer path is roomy +// and cost a split packet per datagram when it is not -- and we cannot tell which we have. The +// outer connection still discovers its own path MTU; that is where the gain is, and an application +// that needs to pin it (iOS, where discovery has misbehaved) still can, through +// opt::quic_max_udp_payload. +static constexpr size_t TUNNELED_QUIC_MAX_UDP_PAYLOAD = 1200; + namespace { auto cat = oxen::log::Cat("session-router"); @@ -49,22 +82,23 @@ namespace { return *key; } - std::pair, uint16_t> remote_info_for_destination( + std::pair, uint16_t> remote_info_for_destination( const network_destination& dest, const std::string& request_id) { - std::optional, uint16_t>> result; + std::optional, uint16_t>> result; std::visit( [&result, &request_id](const T& arg) { if constexpr (std::is_same_v) { log::trace( cat, "[Request {}]: Using pre-resolved RemoteAddress.", request_id); - result.emplace(arg.view_remote_key(), arg.port()); + result.emplace( + as_span(arg.view_remote_key()).template first<32>(), arg.port()); } else if constexpr (std::is_same_v) { log::trace( cat, "[Request {}]: Resolving service_node to RemoteAddress.", request_id); - result.emplace(arg.remote_pubkey, arg.omq_port); + result.emplace(arg.view_remote_key(), arg.omq_port); } }, dest); @@ -72,9 +106,6 @@ namespace { if (!result) throw std::runtime_error{"Invalid destination"}; - if (result->first.size() != 32) - throw std::runtime_error{"Invalid remote key"}; - return *result; } @@ -111,9 +142,6 @@ void SessionRouter::_init() { data-dir={} [bind] listen=:0 - [logging] - type=none - level=*=debug,quic=info )"_format(opt::netid::to_string(_config.netid), _config.cache_directory); try { @@ -131,15 +159,10 @@ void SessionRouter::_init() { return; if (snode_pool->size() == 0) + // The pool outlives us, so this one keeps its weak guard. snode_pool->refresh_if_needed({}, [weak_self, this] { - auto self = weak_self.lock(); - if (!self) - return; - - _loop->call([weak_self] { - if (auto self = weak_self.lock()) - self->_finish_setup(); - }); + if (auto self = weak_self.lock()) + _jq.call([this] { _finish_setup(); }); }); else _finish_setup(); @@ -157,21 +180,26 @@ SessionRouter::~SessionRouter() { std::vector threads_to_join; // Use 'call_get' to force this to be synchronous - if (_loop) - _loop->call_get([this, &threads_to_join] { - // Harvest upload thread handles *before* _close_connections clears the map - for (auto& [_, upload] : _active_uploads) - if (upload.second.joinable()) - threads_to_join.push_back(std::move(upload.second)); + _loop->call_get([this, &threads_to_join] { + // Harvest upload thread handles *before* _close_connections clears the map + for (auto& [_, upload] : _active_uploads) + if (upload.second.joinable()) + threads_to_join.push_back(std::move(upload.second)); - _close_connections(); - }); + _close_connections(); + }); - // Block until upload threads have finished + // Block until upload threads have finished. These wait on loop jobs of their own, so this has + // to happen out here rather than inside the job above -- and it is why this object must be + // destroyed off the loop thread: joining from the loop would be waiting on ourselves. for (auto& t : threads_to_join) if (t.joinable()) t.join(); + // Only once nothing can post any more: a post to a stopped queue throws, and a throw on one of + // those upload threads would be a terminate. + _loop->call_get([this] { _jq.stop(); }); + log::debug(cat, "Destroyed."); } @@ -179,7 +207,7 @@ SessionRouter::~SessionRouter() { void SessionRouter::suspend() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _suspended = true; _close_connections(); log::info(cat, "Suspended."); @@ -188,7 +216,7 @@ void SessionRouter::suspend() { void SessionRouter::resume(bool /*automatically_reconnect*/) { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { if (!_suspended) return; @@ -199,7 +227,7 @@ void SessionRouter::resume(bool /*automatically_reconnect*/) { void SessionRouter::close_connections() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { _close_connections(); }); + _jq.call_get([this] { _close_connections(); }); } void SessionRouter::clear_cache() { @@ -212,26 +240,139 @@ std::vector SessionRouter::get_active_paths() { } void SessionRouter::send_request(Request request, network_response_callback_t callback) { - _loop->call([weak_self = weak_from_this(), req = std::move(request), cb = std::move(callback)] { - if (auto self = weak_self.lock()) - self->_send_request_internal(std::move(req), std::move(cb)); + _jq.call([this, req = std::move(request), cb = std::move(callback)]() mutable { + _send_request_internal(std::move(req), std::move(cb)); }); } void SessionRouter::upload(UploadRequest request) { - _loop->call([weak_self = weak_from_this(), req = std::move(request)] { - if (auto self = weak_self.lock()) - self->_upload_internal(std::move(req)); + _jq.call([this, req = std::move(request)]() mutable { _upload_internal(std::move(req)); }); +} + +void SessionRouter::upload_file(FileUploadRequest request, std::span seed) { + auto quic_target = file_server::default_quic_target(_config.file_server_config, _config.netid); + if (!quic_target) { + // TODO: legacy file upload fallback + if (request.on_complete) + request.on_complete(ERROR_FILE_SERVER_UNAVAILABLE, false); + return; + } + + // Construct the Encryptor now (on the caller's thread), consuming the seed. + auto enc = std::make_shared(seed, request.domain); + auto target = std::move(*quic_target); + + // Dispatch to the loop thread so we wait for _ready before spawning the upload thread. + _jq.call([this, + enc = std::move(enc), + request = std::move(request), + target = std::move(target)]() mutable { + if (!_ready) { + log::debug(cat, "Router not ready, queueing upload_file."); + // _pending_operations is ours and is run from our own jobs, so `this` is safe here too. + _pending_operations.emplace_back([this, + enc = std::move(enc), + request = std::move(request), + target = std::move(target)]() mutable { + _start_file_upload(std::move(enc), std::move(request), std::move(target)); + }); + return; + } + + _start_file_upload(std::move(enc), std::move(request), std::move(target)); }); } -void SessionRouter::download(DownloadRequest request) { - _loop->call([weak_self = weak_from_this(), req = std::move(request)] { - if (auto self = weak_self.lock()) - self->_download_internal(std::move(req)); +void SessionRouter::_start_file_upload( + std::shared_ptr enc, + FileUploadRequest request, + file_server::SRouterTarget target) { + const std::string upload_id = random::unique_id("UPL"); + auto& upload_thread = + _active_uploads.emplace(upload_id, std::make_pair(UploadRequest{}, std::thread{})) + .first->second.second; + + upload_thread = std::thread([weak_self = weak_from_this(), + this, + enc = std::move(enc), + request = std::move(request), + target = std::move(target), + upload_id]() mutable { + // The get_client callback runs on the loop thread: it establishes the tunnel + // and returns the QuicFileClient. This blocks (via promise/future) until the + // tunnel is established. + auto client_promise = std::make_shared>(); + auto client_future = client_promise->get_future(); + + streaming_file_upload( + _loop, + std::move(*enc), + std::move(request), + [weak_self, this, target, client_promise]() -> QuicFileClient* { + auto self = weak_self.lock(); + if (!self) + return nullptr; + + // The claim is kept alongside the other tunnels rather than in a local: the + // file client goes on using this port long after this function has returned. + // + // establish_udp does not block -- the port mapping is local, so local_port is + // usable on return -- but the session behind it may not be up yet. Sending + // into a mapping with nothing behind it means the QUIC handshake's packets are + // dropped and retried on QUIC's own timer, which is where a cold upload spends + // seconds doing nothing. So the callbacks are taken: `established` is what + // says the mapping now carries traffic, and a failure is reported rather than + // waited out -- an unreachable relay is known immediately and there is no + // point spending the request timeout discovering it again. + auto& held = _tunnel(target.address); + if (!held.tunnel) { + auto address = target.address; + held.tunnel = srouter->establish_udp( + target.address, + target.port, + [weak_self, this, address](router::tunnel_info) { + if (auto self = weak_self.lock()) + _tunnel(address).established = true; + }, + [weak_self, this, address](router::tunnel_failure failure) { + auto self = weak_self.lock(); + if (!self) + return; + log::error( + cat, + "File server {} is {}.", + address, + failure == router::tunnel_failure::unreachable + ? "unreachable" + : "not responding"); + // Drops the claim, so the next attempt builds a fresh mapping + // rather than reusing one known to carry nothing. + _fail_tunnel( + address, + failure == router::tunnel_failure::unreachable); + }); + } + if (!held.tunnel) { + log::error(cat, "File server {} is unreachable.", target.address); + return nullptr; + } + + auto pubkey = pubkey_from_srouter_address(held.tunnel->remote); + if (!pubkey) + return nullptr; + + return &_get_file_client( + *pubkey, "::1", held.tunnel->local_port, TUNNELED_QUIC_MAX_UDP_PAYLOAD); + }); + + _jq.call([this, upload_id] { _cleanup_upload(upload_id); }); }); } +void SessionRouter::download(DownloadRequest request) { + _jq.call([this, req = std::move(request)]() mutable { _download_internal(std::move(req)); }); +} + // MARK: Internal Logic void SessionRouter::_finish_setup() { @@ -240,14 +381,16 @@ void SessionRouter::_finish_setup() { log::debug(cat, "Finishing setup, router is now ready."); auto requests_to_process = std::move(_pending_requests); - if (requests_to_process.empty()) + auto ops_to_process = std::move(_pending_operations); + + size_t pending_count = ops_to_process.size(); + for (auto& [_, reqs] : requests_to_process) + pending_count += reqs.size(); + + if (pending_count == 0) return; - // Process any requests that were queued before we were ready - log::debug( - cat, - "Processing {} requests queued during initialization.", - requests_to_process.size()); + log::debug(cat, "Processing {} operations queued during initialization.", pending_count); for (auto& [address, requests] : requests_to_process) { if (!requests.empty()) { @@ -258,11 +401,12 @@ void SessionRouter::_finish_setup() { _send_request_internal(std::move(req), std::move(cb)); } } + + for (auto& op : ops_to_process) + op(); } void SessionRouter::_close_connections() { - // TODO: Need to close any active connections on the session router instance. - // Cancel any uploads and downloads for (auto& [id, request_and_thread] : _active_uploads) { request_and_thread.first.cancel(); @@ -292,7 +436,7 @@ void SessionRouter::_close_connections() { "Network is suspended."); // Clear all storage of requests, paths and connections so that we are in a fresh state on - // relaunch + // relaunch; dropping our claims releases the tunnels behind them. _active_tunnels.clear(); _pending_requests.clear(); _update_status(ConnectionStatus::disconnected); @@ -394,9 +538,14 @@ void SessionRouter::_send_direct_request(Request request, network_response_callb remote_info_for_destination(request.destination, request.request_id); const auto remote_pubkey_hex = oxenc::to_hex(remote_pubkey); - if (auto it = _active_tunnels.find(remote_pubkey_hex); it != _active_tunnels.end()) { + if (auto it = _active_tunnels.find(remote_pubkey_hex); + it != _active_tunnels.end() && it->second->established) { log::trace(cat, "[Request {}] Found active tunnel.", request.request_id); - _send_via_tunnel(it->second, std::move(request), std::move(callback)); + _send_via_tunnel( + it->second->tunnel->remote, + it->second->tunnel->local_port, + std::move(request), + std::move(callback)); return; } @@ -490,7 +639,7 @@ void SessionRouter::_send_proxy_request(Request request, network_response_callba } service_node proxy_node = proxy_nodes[0]; - std::vector encrypted_blob; + std::vector encrypted_blob; std::shared_ptr parser; log::debug( cat, "[Request {}]: Selected {} as proxy.", request.request_id, proxy_node.to_string()); @@ -554,50 +703,281 @@ void SessionRouter::_send_proxy_request(Request request, network_response_callba _send_direct_request(std::move(proxy_request), std::move(proxy_callback)); } +// Extracts the Ed25519 pubkey from a resolved session-router address like "b32zpubkey.sesh" +// or "b32zpubkey.snode". Returns nullopt if the address is not a valid pubkey-based address. +static std::optional pubkey_from_srouter_address(std::string_view address) { + auto dot = address.find('.'); + if (dot == std::string_view::npos || dot == 0) + return std::nullopt; + + auto b32z = address.substr(0, dot); + if (!oxenc::is_base32z(b32z) || oxenc::from_base32z_size(b32z.size()) != 32) + return std::nullopt; + + std::optional result{std::in_place}; + oxenc::from_base32z(b32z.begin(), b32z.end(), result->begin()); + + // The length and alphabet checks above only establish that the label decodes to 32 bytes; the + // address names a router by its Ed25519 pubkey, so 32 bytes that are not one do not name + // anything. The caller hands this straight to a file client as the remote's identity key. + if (!ed25519::is_valid_pubkey(*result)) + return std::nullopt; + + return result; +} + +void SessionRouter::_cleanup_upload(const std::string& upload_id) { + auto node = _active_uploads.extract(upload_id); + if (!node.empty()) { + auto& thread = node.mapped().second; + if (thread.joinable()) + thread.join(); + } +} + +QuicFileClient& SessionRouter::_get_file_client( + const ed25519_pubkey& pubkey, + std::string_view address, + uint16_t port, + std::optional max_udp_payload) { + auto [it, inserted] = _file_clients.try_emplace(pubkey, nullptr); + if (inserted) + it->second = std::make_unique( + _loop, pubkey, std::string{address}, port, max_udp_payload); + else + it->second->set_target(pubkey, std::string{address}, port); + return *it->second; +} + +void SessionRouter::_quic_upload_via_tunnel( + UploadRequest upload_request, + std::string upload_id, + std::vector data, + router::tunnel_info info) { + auto pubkey = pubkey_from_srouter_address(info.remote); + if (!pubkey) { + log::error( + cat, + "[Upload {}]: Could not extract pubkey from resolved address {}", + upload_id, + info.remote); + upload_request.on_complete(ERROR_UNKNOWN, false); + _cleanup_upload(upload_id); + return; + } + + _get_file_client(*pubkey, "::1", info.local_port, TUNNELED_QUIC_MAX_UDP_PAYLOAD) + .upload(std::move(data), + upload_request.ttl, + [weak_self = weak_from_this(), this, upload_request, upload_id]( + std::variant result) { + auto self = weak_self.lock(); + if (!self) + return; + + if (auto* meta = std::get_if(&result)) + log::info( + cat, "[Upload {}]: Success, file ID: {}", upload_id, meta->id); + else + log::error( + cat, + "[Upload {}]: Failed with error {}", + upload_id, + std::get(result)); + + upload_request.on_complete(std::move(result), false); + _cleanup_upload(upload_id); + }); +} + +void SessionRouter::_quic_download_via_tunnel( + DownloadRequest request, + std::string download_id, + std::string file_id, + router::tunnel_info info) { + auto pubkey = pubkey_from_srouter_address(info.remote); + if (!pubkey) { + log::error( + cat, + "[Download {}]: Could not extract pubkey from resolved address {}", + download_id, + info.remote); + _active_downloads.erase(download_id); + request.on_complete(ERROR_UNKNOWN, false); + return; + } + + _get_file_client(*pubkey, "::1", info.local_port, TUNNELED_QUIC_MAX_UDP_PAYLOAD) + .download( + std::move(file_id), + request.on_data, + [weak_self = weak_from_this(), this, request, download_id]( + std::variant result) { + auto self = weak_self.lock(); + if (!self) + return; + + _active_downloads.erase(download_id); + + if (auto* meta = std::get_if(&result)) + log::info( + cat, + "[Download {}]: Success, file ID: {} ({} bytes)", + download_id, + meta->id, + meta->size); + else + log::error( + cat, + "[Download {}]: Failed with error {}", + download_id, + std::get(result)); + + request.on_complete(std::move(result), false); + }); +} + void SessionRouter::_upload_internal(UploadRequest request) { - // TODO: Update this to use streaming approach + if (!_ready) { + log::debug(cat, "Router not ready, queueing upload."); + _pending_operations.emplace_back( + [weak_self = weak_from_this(), req = std::move(request)]() mutable { + if (auto self = weak_self.lock()) + self->_upload_internal(std::move(req)); + }); + return; + } + const std::string upload_id = random::unique_id("UP"); log::info(cat, "[Upload {}]: Starting upload.", upload_id); - // Make the callback atomic so we don't need to worry about it being called multiple times (eg. - // network shutdown cancelling the request and the transport shutdown automatically triggering - // callbacks) request.on_complete = make_callback_atomic(std::move(request.on_complete)); - auto& [_, upload_thread] = - _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) - .first->second; - // Accumulate data on a background thread as we don't know whether `next_data` is doing file I/O - // or just reading from memory (it's a bit of a waste if it's in-memory data but loading from - // disk should be prioritised) + auto quic_target = file_server::default_quic_target(_config.file_server_config, _config.netid); + if (!quic_target) { + _upload_internal_legacy(std::move(request), std::move(upload_id)); + return; + } + + // QUIC upload: accumulate data on background thread, then tunnel and upload + auto& upload_thread = _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) + .first->second.second; + upload_thread = std::thread([weak_self = weak_from_this(), this, upload_request = request, upload_id, - file_server_config = _config.file_server_config] { + target = std::move(*quic_target)] { auto self = weak_self.lock(); if (!self) return; - // Onion requests don't support streaming data so we need to load all the data from the - // streaming source into memory try { - Request request = - file_server::to_request(upload_id, file_server_config, upload_request); + std::vector all_data; + while (true) { + if (upload_request.is_cancelled()) + throw cancellation_exception{"Cancelled during data accumulation."}; + auto chunk = upload_request.next_data(); + if (chunk.empty()) + break; + auto* p = reinterpret_cast(chunk.data()); + all_data.insert(all_data.end(), p, p + chunk.size()); + } + + if (all_data.empty()) + throw std::runtime_error{"No data to upload"}; - _loop->call([weak_self, this, upload_request, req = std::move(request), upload_id] { - auto self = weak_self.lock(); - if (!self) + log::debug( + cat, + "[Upload {}]: Accumulated {} bytes, establishing tunnel to {}.", + upload_id, + all_data.size(), + target.address); + + _jq.call([this, + upload_request, + upload_id, + target, + data = std::move(all_data)]() mutable { + if (upload_request.is_cancelled()) { + upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); + _cleanup_upload(upload_id); return; + } + auto& held = _tunnel(target.address); + held.tunnel = srouter->establish_udp( + target.address, + target.port, + [weak_self = weak_from_this(), + this, + upload_request, + upload_id, + data = std::move(data)](router::tunnel_info info) mutable { + if (auto self = weak_self.lock()) + _quic_upload_via_tunnel( + upload_request, + upload_id, + std::move(data), + std::move(info)); + }, + [weak_self = weak_from_this(), this, upload_request, upload_id]( + router::tunnel_failure failure) { + if (auto self = weak_self.lock()) { + bool timeout = failure == router::tunnel_failure::timeout; + log::error( + cat, + "[Upload {}]: Tunnel establishment failed: {}.", + upload_id, + timeout ? "timed out" : "remote is unreachable"); + upload_request.on_complete( + timeout ? ERROR_BUILD_TIMEOUT : ERROR_INVALID_DESTINATION, + timeout); + _cleanup_upload(upload_id); + } + }); + if (!held.tunnel) { + // Neither callback fires when the remote is unreachable, so this is the only + // chance to report it. + log::error(cat, "[Upload {}]: {} is unreachable.", upload_id, target.address); + upload_request.on_complete(ERROR_INVALID_DESTINATION, false); + _cleanup_upload(upload_id); + } + }); + } catch (const cancellation_exception&) { + _jq.call([this, upload_request, upload_id] { + upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); + _cleanup_upload(upload_id); + }); + } catch (const std::exception& e) { + log::error(cat, "[Upload {}]: Exception: {}", upload_id, e.what()); + _jq.call([this, upload_request, upload_id] { + upload_request.on_complete(ERROR_UNKNOWN, false); + _cleanup_upload(upload_id); + }); + } + }); +} + +// Legacy HTTP-based upload path, used when no QUIC file server target is available. +void SessionRouter::_upload_internal_legacy(UploadRequest request, std::string upload_id) { + auto& upload_thread = _active_uploads.emplace(upload_id, std::make_pair(request, std::thread{})) + .first->second.second; + + // Ours, and joined before the queue is stopped -- see ~SessionRouter. + upload_thread = std::thread([this, + upload_request = request, + upload_id, + file_server_config = _config.file_server_config] { + try { + Request request = + file_server::to_request(upload_id, file_server_config, upload_request); + + _jq.call([this, upload_request, req = std::move(request), upload_id]() mutable { if (upload_request.is_cancelled() || !req.body) { log::debug(cat, "[Upload {}]: Cancelled before sending request.", upload_id); upload_request.on_complete(ERROR_REQUEST_CANCELLED, false); - - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); + _cleanup_upload(upload_id); return; } @@ -610,7 +990,12 @@ void SessionRouter::_upload_internal(UploadRequest request) { _send_request_internal( std::move(req), - [weak_self, this, upload_id, upload_request, upload_size]( + // Ends up held by the transport, which outlives us: weak guard. + [weak_self = weak_from_this(), + this, + upload_id, + upload_request, + upload_size]( bool success, bool timeout, int16_t status_code, @@ -620,11 +1005,7 @@ void SessionRouter::_upload_internal(UploadRequest request) { if (!self) return; - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && - active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); + _cleanup_upload(upload_id); try { if (upload_request.is_cancelled()) @@ -676,16 +1057,8 @@ void SessionRouter::_upload_internal(UploadRequest request) { } catch (const std::exception& e) { log::error(cat, "[Upload {}]: Exception during upload: {}", upload_id, e.what()); - _loop->call([weak_self, this, upload_request, upload_id] { - auto self = weak_self.lock(); - if (!self) - return; - - // Join the thread to keep it alive during callback handling - auto active_upload_node = _active_uploads.extract(upload_id); - if (!active_upload_node.empty() && active_upload_node.mapped().second.joinable()) - active_upload_node.mapped().second.join(); - + _jq.call([this, upload_request, upload_id] { + _cleanup_upload(upload_id); upload_request.on_complete(ERROR_UNKNOWN, false); }); } @@ -693,13 +1066,73 @@ void SessionRouter::_upload_internal(UploadRequest request) { } void SessionRouter::_download_internal(DownloadRequest request) { + if (!_ready) { + log::debug(cat, "Router not ready, queueing download."); + _pending_operations.emplace_back( + [weak_self = weak_from_this(), req = std::move(request)]() mutable { + if (auto self = weak_self.lock()) + self->_download_internal(std::move(req)); + }); + return; + } + const std::string download_id = random::unique_id("DL"); log::info(cat, "[Download {}]: Starting download.", download_id); - // Make the callback atomic so we don't need to worry about it being called multiple times (eg. - // network shutdown cancelling the request and the transport shutdown automatically triggering - // callbacks) request.on_complete = make_callback_atomic(std::move(request.on_complete)); + + // Check for a QUIC target: first from the URL's sr= fragment, then from default mapping + std::optional quic_target; + auto download_info = file_server::parse_download_url(request.download_url); + if (download_info && download_info->srouter_target) + quic_target = std::move(download_info->srouter_target); + else + quic_target = file_server::default_quic_target(_config.file_server_config, _config.netid); + + if (!quic_target || !download_info) { + _download_internal_legacy(std::move(request), std::move(download_id)); + return; + } + + // QUIC download path + _active_downloads[download_id] = request; + auto file_id = download_info->file_id; + + auto& held = _tunnel(quic_target->address); + held.tunnel = srouter->establish_udp( + quic_target->address, + quic_target->port, + [weak_self = weak_from_this(), this, request, download_id, file_id]( + router::tunnel_info info) mutable { + if (auto self = weak_self.lock()) + _quic_download_via_tunnel( + request, download_id, std::move(file_id), std::move(info)); + }, + [weak_self = weak_from_this(), this, request, download_id]( + router::tunnel_failure failure) { + if (auto self = weak_self.lock()) { + bool timeout = failure == router::tunnel_failure::timeout; + log::error( + cat, + "[Download {}]: Tunnel establishment failed: {}.", + download_id, + timeout ? "timed out" : "remote is unreachable"); + _active_downloads.erase(download_id); + request.on_complete( + timeout ? ERROR_BUILD_TIMEOUT : ERROR_INVALID_DESTINATION, timeout); + } + }); + if (!held.tunnel) { + // Neither callback fires when the remote is unreachable, so this is the only chance to + // report it. + log::error(cat, "[Download {}]: {} is unreachable.", download_id, quic_target->address); + _active_downloads.erase(download_id); + request.on_complete(ERROR_INVALID_DESTINATION, false); + } +} + +// Legacy HTTP-based download path, used when no QUIC file server target is available. +void SessionRouter::_download_internal_legacy(DownloadRequest request, std::string download_id) { _active_downloads[download_id] = request; try { @@ -745,7 +1178,7 @@ void SessionRouter::_download_internal(DownloadRequest request) { metadata.id); if (request.on_data) - request.on_data(metadata, std::move(data)); + request.on_data(metadata, to_span(data)); request.on_complete(std::move(metadata), false); } catch (const cancellation_exception&) { @@ -779,7 +1212,7 @@ void SessionRouter::_download_internal(DownloadRequest request) { } void SessionRouter::_establish_tunnel( - std::span& remote_pubkey, + std::span remote_pubkey, const uint16_t remote_port, const std::string& initiating_req_id) { auto address_pubkey_hex = oxenc::to_hex(remote_pubkey); @@ -789,22 +1222,7 @@ void SessionRouter::_establish_tunnel( cat, "Destination had an invalid remote key, request {} is being dropped.", initiating_req_id); - // Fail all the pending requests for this connection - if (auto it = _pending_requests.find(address_pubkey_hex); it != _pending_requests.end()) { - auto to_fail = std::move(it->second); - _pending_requests.erase(it); - log::error( - cat, - "Failing {} pending request(s) due to connection failure.", - to_fail.size()); - - for (auto& [req, cb] : to_fail) - cb(false, - false, - ERROR_INVALID_DESTINATION, - {content_type_plain_text}, - "Failed to establish tunnel to remote."); - } + _fail_tunnel(address_pubkey_hex, false); return; } @@ -819,9 +1237,9 @@ void SessionRouter::_establish_tunnel( // } std::string srouter_address; - srouter_address.reserve(oxenc::to_base32z_size(32UL) + ".snode"sv.size()); + srouter_address.reserve(oxenc::to_base32z_size(remote_pubkey.size()) + ".snode"sv.size()); oxenc::to_base32z( - remote_pubkey.begin(), remote_pubkey.begin() + 32, std::back_inserter(srouter_address)); + remote_pubkey.begin(), remote_pubkey.end(), std::back_inserter(srouter_address)); srouter_address += ".snode"sv; // srouter::RouterID router_id{remote_pubkey.first<32>()}; @@ -836,7 +1254,7 @@ void SessionRouter::_establish_tunnel( "[Request {}] Establishing new tunnel to {}.", initiating_req_id, address_pubkey_hex); - srouter->establish_udp( + auto tunnel = srouter->establish_udp( srouter_address, test_port, [weak_self = weak_from_this(), this, address_pubkey_hex, initiating_req_id]( @@ -851,9 +1269,12 @@ void SessionRouter::_establish_tunnel( initiating_req_id, address_pubkey_hex); + // This can fire before `establish_udp` returns, when a session to the remote is + // already up, so the lease may not have been recorded yet. + _tunnel(address_pubkey_hex).established = true; + auto requests_to_process = std::move(_pending_requests[address_pubkey_hex]); _pending_requests.erase(address_pubkey_hex); - _active_tunnels.insert_or_assign(address_pubkey_hex, info); // We had a successful connection so update the status to connected _update_status(ConnectionStatus::connected); @@ -866,51 +1287,89 @@ void SessionRouter::_establish_tunnel( info.remote); for (auto&& [req, cb] : std::move(requests_to_process)) - _send_via_tunnel(info, std::move(req), std::move(cb)); + _send_via_tunnel( + info.remote, info.local_port, std::move(req), std::move(cb)); } }, - [weak_self = weak_from_this(), this, address_pubkey_hex, initiating_req_id]() mutable { + [weak_self = weak_from_this(), this, address_pubkey_hex, initiating_req_id]( + router::tunnel_failure failure) mutable { auto self = weak_self.lock(); if (!self) return; + // A relay contact we didn't have when the tunnel was requested, and which the + // lookup then found doesn't exist, arrives here rather than as a nullopt return. + bool unreachable = failure == router::tunnel_failure::unreachable; + log::info( cat, - "[Request {}] Unable to establish session router UDP connection to {}.", + "[Request {}] Session router connection to {} failed: {}.", initiating_req_id, - address_pubkey_hex); + address_pubkey_hex, + unreachable ? "node is not reachable" : "timed out"); - _active_tunnels.erase(address_pubkey_hex); + _fail_tunnel(address_pubkey_hex, unreachable); + }); - // Fail all the pending requests for this connection - if (auto it = _pending_requests.find(address_pubkey_hex); - it != _pending_requests.end()) { - auto to_fail = std::move(it->second); - _pending_requests.erase(it); + // No tunnel at all means Session Router holds no relay contact for this node, i.e. it isn't + // participating in the network rather than merely being slow to answer. Neither callback + // fires in that case, so the failure is ours to report. + if (!tunnel) { + log::info( + cat, + "[Request {}] {} is not reachable via session router.", + initiating_req_id, + address_pubkey_hex); - log::error( - cat, - "Failing {} pending requests due to UDP connection failure.", - to_fail.size()); - - for (auto& [req, cb] : to_fail) - cb(false, - false, - ERROR_REQUEST_TIMEOUT, - {content_type_plain_text}, - "Timeout"); - } + _fail_tunnel(address_pubkey_hex, true); + return; + } - // If we have no longer have any active connections then we are disconnected - if (_active_tunnels.empty()) - _update_status(ConnectionStatus::disconnected); - }); + // The established callback may already have run, so keep whatever it recorded and only fill + // in the claim. + _tunnel(address_pubkey_hex).tunnel = std::move(tunnel); } -void SessionRouter::_send_via_tunnel( - router::tunnel_info tunnel, Request request, network_response_callback_t callback) { - // TODO: Is there a way to check that the 'tunnel_info' still active?. +ActiveTunnel& SessionRouter::_tunnel(const std::string& pubkey_hex) { + auto& entry = _active_tunnels[pubkey_hex]; + if (!entry) + entry = std::make_unique(); + return *entry; +} +void SessionRouter::_fail_tunnel(const std::string& pubkey_hex, bool unreachable) { + // Dropping our claim releases the mapping; a later attempt builds a fresh one, which is also + // what gives Session Router the chance to notice that an unreachable node has come back. + _active_tunnels.erase(pubkey_hex); + + if (auto snode_pool = _snode_pool.lock()) + if (auto key = ed25519_pubkey::maybe_from_hex(pubkey_hex)) + snode_pool->record_node_failure(*key, unreachable); + + if (auto it = _pending_requests.find(pubkey_hex); it != _pending_requests.end()) { + auto to_fail = std::move(it->second); + _pending_requests.erase(it); + + log::error(cat, "Failing {} pending request(s) to {}.", to_fail.size(), pubkey_hex); + + for (auto& [req, cb] : to_fail) + cb(false, + !unreachable, + unreachable ? ERROR_INVALID_DESTINATION : ERROR_REQUEST_TIMEOUT, + {content_type_plain_text}, + unreachable ? "Node is not reachable via session router" : "Timeout"); + } + + // If we no longer have any active connections then we are disconnected + if (_active_tunnels.empty()) + _update_status(ConnectionStatus::disconnected); +} + +void SessionRouter::_send_via_tunnel( + std::string tunnel_remote, + uint16_t tunnel_local_port, + Request request, + network_response_callback_t callback) { // If the request has already timedout at this point then just fail it immediately auto timeout = request.time_remaining(); if (timeout <= 0s) @@ -928,7 +1387,7 @@ void SessionRouter::_send_via_tunnel( } // We have a valid connection and stream so we can send the request - log::debug(cat, "[Request {}] Sending to {}.", request.request_id, tunnel.remote); + log::debug(cat, "[Request {}] Sending to {}.", request.request_id, tunnel_remote); auto [remote_pubkey, _] = remote_info_for_destination(request.destination, request.request_id); const auto remote_pubkey_hex = oxenc::to_hex(remote_pubkey); @@ -936,7 +1395,8 @@ void SessionRouter::_send_via_tunnel( // auto test_key = // oxenc::from_base64("1n+DAM9hKyJhtXSPR5L/HdemIKPiHs8dZsPn2kEQuMs="); auto test_key // = oxenc::from_base32z("55fxd8stjrt9g6rsbftx7eesy47pj4751xjghinr3k9ffxh4ieyo"); - auto router_target = oxen::quic::RemoteAddress{test_key, "::1", tunnel.local_port}; + auto router_target = + oxen::quic::RemoteAddress{as_span(test_key), "::1", tunnel_local_port}; // Construct the actual request to send std::optional remaining_overall_timeout = diff --git a/src/network/service_node.cpp b/src/network/service_node.cpp index affa303a2..7660a182a 100644 --- a/src/network/service_node.cpp +++ b/src/network/service_node.cpp @@ -54,7 +54,7 @@ service_node service_node::from_json(nlohmann::json json) { throw std::invalid_argument{ "Invalid service node json: pubkey_ed25519 is not a valid, hex pubkey"}; - std::vector pubkey; + std::vector pubkey; pubkey.reserve(32); oxenc::from_hex(pk_ed.begin(), pk_ed.end(), std::back_inserter(pubkey)); @@ -205,10 +205,8 @@ std::pair, int> service_node::process_snode_cache_bin( try { // Pubkey - std::vector pubkey; - pubkey.assign( - reinterpret_cast(current_ptr), - reinterpret_cast(current_ptr) + PK_SIZE); + std::vector pubkey; + pubkey.assign(current_ptr, current_ptr + PK_SIZE); note_ptr += PK_SIZE; // Swarm ID diff --git a/src/network/session_network.cpp b/src/network/session_network.cpp index 7d160fc41..cce074796 100644 --- a/src/network/session_network.cpp +++ b/src/network/session_network.cpp @@ -10,6 +10,7 @@ #include #include +#include "../internal-util.hpp" #include "session/blinding.hpp" #include "session/network/backends/session_file_server.hpp" #include "session/network/network_config.hpp" @@ -37,9 +38,19 @@ namespace { constexpr auto clock_out_of_sync_error = "Clock out of sync"; + // Checks a precondition and, if true, fires the request's on_complete with the given error. + // Returns true if the condition was met (i.e. the caller should return). + template + bool fail_if(Req& req, bool cond, int16_t err) { + if (cond && req.on_complete) + req.on_complete(err, false); + return cond; + } + config::FileServer build_file_server_config(const config::Config& main_config) { - config::FileServer file_server_config = file_server::DEFAULT_CONFIG; - file_server_config.use_stream_encryption = main_config.file_server_use_stream_encryption; + config::FileServer file_server_config = main_config.netid == opt::netid::Target::testnet + ? file_server::TESTNET_CONFIG + : file_server::DEFAULT_CONFIG; if (main_config.custom_file_server_scheme) file_server_config.scheme = *main_config.custom_file_server_scheme; @@ -56,6 +67,12 @@ namespace { if (main_config.custom_file_server_max_file_size) file_server_config.max_file_size = *main_config.custom_file_server_max_file_size; + if (main_config.custom_file_server_srouter_address) + file_server_config.srouter = file_server::SRouterTarget{ + *main_config.custom_file_server_srouter_address, + main_config.custom_file_server_srouter_port.value_or( + file_server::QUIC_DEFAULT_PORT)}; + return file_server_config; } @@ -78,12 +95,16 @@ namespace { config::QuicTransport build_quic_transport_config(const config::Config& main_config) { return {main_config.quic_handshake_timeout, main_config.quic_keep_alive, - main_config.quic_disable_mtu_discovery}; + main_config.quic_max_udp_payload}; } config::DirectRouter build_direct_router_config( - const config::Config& /*main_config*/, const config::FileServer& file_server_config) { - return {file_server_config}; + const config::Config& main_config, const config::FileServer& file_server_config) { + return {file_server_config, + main_config.netid, + main_config.quic_file_server_address, + main_config.quic_file_server_ed_pubkey, + main_config.quic_file_server_port.value_or(file_server::QUIC_DEFAULT_PORT)}; } config::SessionRouter build_session_router_config( @@ -119,10 +140,6 @@ namespace { main_config.onionreq_min_path_counts}; } -} // namespace - -namespace detail { - std::vector convert_service_nodes( std::vector nodes) { std::vector converted_nodes; @@ -135,7 +152,7 @@ namespace detail { return converted_nodes; } -} // namespace detail +} // namespace Network::Network(config::Config _conf) : config{std::move(_conf)}, file_server_config{std::move(build_file_server_config(config))} { @@ -160,12 +177,16 @@ Network::Network(config::Config _conf) : // Now we can properly do any setup needed _loop = std::make_shared(); _disk_loop = std::make_shared(); + _jq.emplace(*_loop); // Setup the transport layer switch (config.transport) { case opt::transport::Type::quic: - _transport = std::make_shared( - std::move(build_quic_transport_config(config)), _loop); + // Created through the loop's deleter, so that dropping our reference to it in + // ~Network destroys it *on the loop* and blocks until that has happened: no libquic + // callback can then be in flight into it, which is what lets it hold a bare `this`. + _transport = _loop->make_shared( + std::move(build_quic_transport_config(config)), *_loop); break; } @@ -181,7 +202,7 @@ Network::Network(config::Config _conf) : "Transport provided to the SnodePool bootstrap fetcher has been destroyed."); }; _snode_pool = std::make_shared( - std::move(build_snode_pool_config(config)), _loop, _disk_loop, bootstrap_fetcher); + std::move(build_snode_pool_config(config)), *_loop, *_disk_loop, bootstrap_fetcher); // Additional transport configuration _transport->set_node_failure_reporter( @@ -193,20 +214,24 @@ Network::Network(config::Config _conf) : // Setup the router switch (config.router) { case opt::router::Type::onion_requests: - _router = std::make_unique( + _router = _loop->make_shared( std::move(build_onion_request_router_config(config, file_server_config)), - _loop, - _disk_loop, + *_loop, + *_disk_loop, _snode_pool, _transport); break; case opt::router::Type::session_router: +#ifdef ENABLE_NETWORKING_SROUTER _router = SessionRouter::make( std::move(build_session_router_config(config, file_server_config)), _loop, _snode_pool, _transport); +#else + throw std::runtime_error{"Session Router support is not enabled in this build!"}; +#endif break; case opt::router::Type::direct: @@ -243,10 +268,21 @@ Network::Network(config::Config _conf) : _transport->on_status_changed = [this] { _recalculate_status(); }; // Perform a clock resync - _loop->call_soon([this] { _resync_clock(std::nullopt, nullptr); }); + _jq->call_soon([this] { _resync_clock(std::nullopt, nullptr); }); } Network::~Network() { + // A Network is singly owned, so this runs on whichever thread its owner dropped it from -- and + // it must not be the loop thread, because the loops are joined at the bottom of this function + // and a thread cannot join itself. The only way to get here on the loop is for an owner to + // destroy its Network from inside a callback we handed it, so say so rather than leaving the + // std::system_error from the join to explain it. + if (_loop->inside()) + log::critical( + cat, + "Network is being destroyed from its own loop thread -- most likely by dropping it " + "from inside one of its own callbacks. This is about to abort."); + // Use 'call_get' to force this to be synchronous _loop->call_get([this] { // Need to ensure the destruction of the router and transport objects don't trigger @@ -262,10 +298,19 @@ Network::~Network() { // Explicitly destroy in dependency order while _loop is still alive. Their destructors post // final cleanup via call_get so the loop must be running when they destruct. + // + // This has to happen before the queue below is stopped, for two reasons: these destructors are + // what guarantee that no callback of ours is still held anywhere (which is what lets those + // callbacks capture `this` bare), and a component finishing up may still post onto our queue -- + // which throws once the queue has been stopped. _router.reset(); _snode_pool.reset(); _transport.reset(); + // Nothing can reach us any more, so cancel whatever is left queued here rather than letting it + // run against members that are about to go. + _loop->call_get([this] { _jq->stop(); }); + // Now shut down the loops (these destructors join their threads) _disk_loop.reset(); _loop.reset(); @@ -275,7 +320,7 @@ Network::~Network() { void Network::clear_cache() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq->call_get([this] { if (_snode_pool) _snode_pool->clear_cache(); if (_router) @@ -289,7 +334,7 @@ void Network::suspend() { // Use 'call_get' to force this to be synchronous. Some of these suspend() calls queue things // on the disk loop, but they don't have to worry about synchronizing because we flush queued // disk loop jobs before we finish. - _loop->call_get([this] { + _jq->call_get([this] { _suspended = true; if (_snode_pool) @@ -323,7 +368,7 @@ void Network::suspend() { void Network::resume(bool automatically_reconnect) { // Use 'call_get' to force this to be synchronous - _loop->call_get([this, automatically_reconnect] { + _jq->call_get([this, automatically_reconnect] { if (!_suspended) return; @@ -340,7 +385,7 @@ void Network::resume(bool automatically_reconnect) { std::chrono::steady_clock::now() - _last_successful_clock_resync; if (time_since_last_resync >= config.min_resume_clock_resync_interval) { - _loop->call_soon([this] { + _jq->call_soon([this] { log::info( cat, "Performing clock resync as enough time has passed since the last resync."); @@ -355,7 +400,7 @@ void Network::resume(bool automatically_reconnect) { void Network::close_connections() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { _close_connections(); }); + _jq->call_get([this] { _close_connections(); }); } // MARK: Interface @@ -375,10 +420,10 @@ void Network::get_swarm( session::network::x25519_pubkey swarm_pubkey, bool ignore_strike_count, std::function swarm)> callback) { - _loop->call([this, - pubkey = std::move(swarm_pubkey), - ignore_strike_count, - cb = std::move(callback)] { + _jq->call([this, + pubkey = std::move(swarm_pubkey), + ignore_strike_count, + cb = std::move(callback)] { if (!_snode_pool) { log::warning( cat, @@ -393,7 +438,7 @@ void Network::get_swarm( void Network::get_random_nodes( uint16_t count, std::function nodes)> callback) { - _loop->call([this, count, cb = std::move(callback)] { + _jq->call([this, count, cb = std::move(callback)] { if (!_snode_pool) { log::warning( cat, @@ -443,16 +488,11 @@ void Network::send_request(Request request, network_response_callback_t callback try { auto processed_request = _preprocess_request(std::move(request)); + // Bare `this`: the router is destroyed by ~Network before our queue is stopped, so it + // cannot still be holding this callback by the time any of our state goes. auto router_callback = - [weak_self = weak_from_this(), - this, - original_req = processed_request, - cb = std::move(callback)]( + [this, original_req = processed_request, cb = std::move(callback)]( bool success, bool timeout, int16_t status_code, auto headers, auto body) { - auto self = weak_self.lock(); - if (!self) - return; - const auto dest_is_snode = std::holds_alternative(original_req.destination); @@ -486,9 +526,26 @@ void Network::send_request(Request request, network_response_callback_t callback return; } + // The node itself could not be reached -- no relay contact for it, so session + // router cannot carry anything there. The swarm is not in question, so the + // request moves to the next member rather than being failed. Without this the + // first send to a node that does not participate dies, and the node is only + // struck out *afterwards*, so the cost is one dead request per such node. + if (final_status_code == ERROR_INVALID_DESTINATION && dest_is_snode && + original_req.swarm_pubkey) { + _retry_next_swarm_node( + std::move(original_req), + timeout, + status_code, + std::move(headers), + std::move(body), + std::move(cb)); + return; + } + // For debugging purposes we want to add a log if this was a successful request // after we did an automatic retry - if (original_req.retry_count > 0) + if (original_req.retry_421_count > 0) log::info( cat, "[Request {}] Received valid response after 421 retry.", @@ -506,29 +563,16 @@ void Network::send_request(Request request, network_response_callback_t callback } void Network::upload(UploadRequest request) { - if (_suspended) { - if (request.on_complete) - request.on_complete(ERROR_NETWORK_SUSPENDED, false); + if (fail_if(request, _suspended, ERROR_NETWORK_SUSPENDED)) return; - } - if (!_transport) { - if (request.on_complete) - request.on_complete(ERROR_NO_TRANSPORT_LAYER, false); + if (fail_if(request, !_transport, ERROR_NO_TRANSPORT_LAYER)) return; - } - if (!_router) { - if (request.on_complete) - request.on_complete(ERROR_NO_ROUTING_LAYER, false); + if (fail_if(request, !_router, ERROR_NO_ROUTING_LAYER)) return; - } auto user_callback = request.on_complete; - request.on_complete = [weak_self = weak_from_this(), this, user_callback]( + request.on_complete = [this, user_callback]( std::variant result, bool timeout) { - auto self = weak_self.lock(); - if (!self) - return; - if (auto* status_code = std::get_if(&result)) { // Handle 425 (clock out of sync) // If we got a 425 (no need to handle a 406 as we only ever upload to a server), @@ -551,33 +595,54 @@ void Network::upload(UploadRequest request) { user_callback(std::move(result), timeout); }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" _router->upload(std::move(request)); +#pragma GCC diagnostic pop +} + +void Network::upload_file(FileUploadRequest request, std::span seed) { + if (fail_if(request, _suspended, ERROR_NETWORK_SUSPENDED)) + return; + if (fail_if(request, !_transport, ERROR_NO_TRANSPORT_LAYER)) + return; + if (fail_if(request, !_router, ERROR_NO_ROUTING_LAYER)) + return; + + auto user_callback = request.on_complete; + request.on_complete = + [this, user_callback]( + std::variant, int16_t> result, + bool timeout) { + if (auto* status_code = std::get_if(&result)) { + if (*status_code == ERROR_TOO_EARLY) { + log::info(cat, "File upload received 425, triggering clock resync."); + _resync_clock(std::nullopt, nullptr); + + if (user_callback) + user_callback(*status_code, timeout); + return; + } + } + + if (user_callback) + user_callback(std::move(result), timeout); + }; + + _router->upload_file(std::move(request), seed); } void Network::download(DownloadRequest request) { - if (_suspended) { - if (request.on_complete) - request.on_complete(ERROR_NETWORK_SUSPENDED, false); + if (fail_if(request, _suspended, ERROR_NETWORK_SUSPENDED)) return; - } - if (!_transport) { - if (request.on_complete) - request.on_complete(ERROR_NO_TRANSPORT_LAYER, false); + if (fail_if(request, !_transport, ERROR_NO_TRANSPORT_LAYER)) return; - } - if (!_router) { - if (request.on_complete) - request.on_complete(ERROR_NO_ROUTING_LAYER, false); + if (fail_if(request, !_router, ERROR_NO_ROUTING_LAYER)) return; - } auto user_callback = request.on_complete; - request.on_complete = [weak_self = weak_from_this(), this, user_callback, req = request]( + request.on_complete = [this, user_callback, req = request]( std::variant result, bool timeout) { - auto self = weak_self.lock(); - if (!self) - return; - if (auto* status_code = std::get_if(&result)) { // If we got a 425 (no need to handle a 406 as we only ever download from a server), // then the device clock is out of sync so we need to kick off a clock resync request @@ -617,7 +682,7 @@ void Network::_close_connections() { } void Network::_recalculate_status() { - _loop->call([this] { + _jq->call([this] { if (!_transport || !_router) return _update_status(ConnectionStatus::disconnected); @@ -770,7 +835,10 @@ void Network::_update_network_state(const std::string& body) { if (new_versions != old_versions) on_network_info_changed( - _network_time_offset.load(), new_versions.hardfork, new_versions.softfork); + std::chrono::duration_cast( + AdjustedClock::get_offset()), + new_versions.hardfork, + new_versions.softfork); } } catch (const std::exception& e) { log::warning(cat, "Failed to parse network state from response: {}", e.what()); @@ -779,9 +847,113 @@ void Network::_update_network_state(const std::string& body) { // MARK: Specific Error Handling +// The least time worth starting another attempt with. A request given a second or two cannot +// realistically resolve a node, connect and get an answer, so spending the remainder of the budget +// on it only delays telling the caller what we already know. +static constexpr auto MIN_RETRY_BUDGET = 2s; + +void Network::_retry_next_swarm_node( + Request original_request, + bool timeout, + int16_t status_code, + std::vector> headers, + std::optional body, + network_response_callback_t final_callback) { + + auto* failed_node = std::get_if(&original_request.destination); + if (!failed_node || !original_request.swarm_pubkey) + return final_callback(false, timeout, status_code, std::move(headers), std::move(body)); + + original_request.failed_nodes.push_back(*failed_node); + auto swarm_pubkey = *original_request.swarm_pubkey; + + // Deliberately not refreshing the snode cache first, which is what the 421 path does: nothing + // here suggests our swarm information is stale, only that one member of it is unreachable. The + // swarm comes back from the cache, so this costs nothing and returns the same members in the + // same order. + // + // The failure that got us here is carried into the callback rather than referenced from out + // here: this returns before get_swarm answers, so anything left behind would be gone by then. + _snode_pool->get_swarm( + swarm_pubkey, + false, + [this, + req = std::move(original_request), + cb = std::move(final_callback), + timeout, + status_code, + headers = std::move(headers), + body = std::move(body)]( + swarm::swarm_id_t, std::vector swarm_nodes) mutable { + // Reports the failure that got us here rather than one of our own invention: the + // caller wants to know why the request did not go through, and "no members left" + // says less than the reason each of them was unusable. + auto give_up = [&] { + cb(false, timeout, status_code, std::move(headers), std::move(body)); + }; + + // The first member that has not already failed. get_swarm shuffles and then + // partitions by strike count, so this is not a fixed order -- what it gives is the + // least-struck members first, in a random order among equals. That is the right + // preference anyway; what matters here is only that a member already spent is + // never chosen again, which is what ends the walk. + auto next = std::ranges::find_if(swarm_nodes, [&](const service_node& node) { + return std::ranges::find(req.failed_nodes, node) == req.failed_nodes.end(); + }); + + if (next == swarm_nodes.end()) { + log::warning( + cat, + "[Request {}] No swarm member left to try: all {} were unreachable.", + req.request_id, + req.failed_nodes.size()); + return give_up(); + } + + auto chosen = next->to_string(); + auto retry = std::move(req); + retry.destination = std::move(*next); + + // Each attempt gets the per-request timeout or whatever is left of the operation's + // overall budget, whichever is shorter -- so walking the swarm cannot outlive what + // the caller asked for, however many members turn out to be unusable. The budget + // runs from the *original* request's creation, which a re-send carries with it, so + // time spent on earlier members counts against later ones. + if (retry.overall_timeout) { + auto spent = std::chrono::duration_cast( + std::chrono::steady_clock::now() - retry.creation_time); + auto left = *retry.overall_timeout - spent; + + if (left < MIN_RETRY_BUDGET) { + log::warning( + cat, + "[Request {}] Out of time to try another swarm member ({}ms left " + "of {}ms).", + retry.request_id, + left.count(), + retry.overall_timeout->count()); + return give_up(); + } + + retry.request_timeout = std::min(retry.request_timeout, left); + } + + log::info( + cat, + "[Request {}] Node unreachable, retrying on {} with {}ms ({} already " + "tried).", + retry.request_id, + chosen, + retry.request_timeout.count(), + retry.failed_nodes.size()); + + send_request(std::move(retry), std::move(cb)); + }); +} + void Network::_handle_421_retry( Request original_request, network_response_callback_t final_callback) { - if (original_request.retry_count >= config.redirect_retry_count) { + if (original_request.retry_421_count >= config.redirect_retry_count) { log::error( cat, "Request {} received 421 but exceeded max retry count.", @@ -873,7 +1045,7 @@ void Network::_handle_421_retry( req_to_retry.request_id, swarm_nodes[new_target].to_string()); auto final_request = req_to_retry; - final_request.retry_count++; + final_request.retry_421_count++; final_request.destination = std::move(swarm_nodes[new_target]); this->send_request(std::move(final_request), std::move(cb)); }); @@ -911,7 +1083,7 @@ void Network::_resync_clock( if (original_request && request_callback) { // If we don't have a resync request queue then create one if (!_clock_resync_request_queue) - _clock_resync_request_queue = detail::RequestQueue::make(_loop); + _clock_resync_request_queue = std::make_shared(*_loop); _clock_resync_request_queue->add(std::move(*original_request), std::move(request_callback)); } @@ -981,7 +1153,7 @@ void Network::_launch_next_clock_out_of_sync_request( std::vector> /*headers*/, std::optional response) { auto end_steady = std::chrono::steady_clock::now(); - auto end_system = sysclock_now_ms(); + auto end_system = clock_now_ms(); // If the resync was cancelled or completed while we were in-flight, do nothing if (!_current_clock_resync_id || *_current_clock_resync_id != request_id) { @@ -1079,7 +1251,7 @@ void Network::_on_clock_resync_complete(const uint8_t /*total_requests*/) { median_offset = (middle_values_sum / 2); } - _network_time_offset = median_offset; + AdjustedClock::set_offset(median_offset); _last_successful_clock_resync = std::chrono::steady_clock::now(); log::info( cat, "[Request {}] Network offset set to: {}ms", refresh_id, median_offset.count()); @@ -1136,20 +1308,9 @@ struct session_response_handle_cpp_t { namespace { -inline std::shared_ptr unbox(network_object* network_) { +inline session::network::Network* unbox(network_object* network_) { assert(network_ && network_->internals); - return *static_cast*>(network_->internals); -} - -inline bool set_error(char* error, const std::exception& e) { - if (!error) - return false; - - std::string msg = e.what(); - if (msg.size() > 255) - msg.resize(255); - std::memcpy(error, msg.c_str(), msg.size() + 1); - return false; + return static_cast(network_->internals); } } // namespace @@ -1194,8 +1355,6 @@ LIBSESSION_C_API session_network_config session_network_config_default() { default: config.transport = SESSION_NETWORK_TRANSPORT_QUIC; } - config.file_server_use_stream_encryption = cpp_defaults.file_server_use_stream_encryption; - config.increase_no_file_limit = cpp_defaults.increase_no_file_limit; config.path_length = cpp_defaults.path_length; config.enforce_subnet_diversity = cpp_defaults.enforce_subnet_diversity; @@ -1244,15 +1403,18 @@ LIBSESSION_C_API session_network_config session_network_config_default() { .count(); config.quic_keep_alive_seconds = std::chrono::duration_cast(cpp_defaults.quic_keep_alive).count(); - config.quic_disable_mtu_discovery = cpp_defaults.quic_disable_mtu_discovery; + config.quic_disable_mtu_discovery = cpp_defaults.quic_max_udp_payload.has_value(); + config.quic_max_udp_payload = cpp_defaults.quic_max_udp_payload.value_or(0); return config; } LIBSESSION_C_API bool session_network_init( network_object** network, const session_network_config* config, char* error) { - if (!network || !config) - return set_error(error, std::invalid_argument{"network or config were null."}); + if (!network || !config) { + session::copy_c_str(error, 256, "network or config were null."); + return false; + } try { // Build the configuration options (ordered this way for the debug logs to make the most @@ -1314,9 +1476,6 @@ LIBSESSION_C_API bool session_network_init( cpp_opts.emplace_back( opt::file_server_max_file_size(config->custom_file_server_max_file_size)); - cpp_opts.emplace_back( - opt::file_server_use_stream_encryption(config->file_server_use_stream_encryption)); - // General if (config->increase_no_file_limit) cpp_opts.emplace_back(opt::increase_no_file_limit{}); @@ -1431,26 +1590,29 @@ LIBSESSION_C_API bool session_network_init( cpp_opts.emplace_back(opt::quic_keep_alive{ std::chrono::seconds{config->quic_keep_alive_seconds}}); - if (config->quic_disable_mtu_discovery) - cpp_opts.emplace_back(opt::quic_disable_mtu_discovery{}); + if (config->quic_max_udp_payload > 0) + cpp_opts.emplace_back(opt::quic_max_udp_payload{config->quic_max_udp_payload}); + else if (config->quic_disable_mtu_discovery) + cpp_opts.emplace_back(opt::quic_max_udp_payload{1200}); break; } // Construct the Network instance Config final_config(cpp_opts); - auto n = std::make_shared(std::move(final_config)); + auto n = std::make_unique(std::move(final_config)); auto n_object = std::make_unique(); - n_object->internals = new std::shared_ptr(n); + n_object->internals = n.release(); *network = n_object.release(); return true; } catch (const std::exception& e) { - return set_error(error, e); + session::copy_c_str(error, 256, e.what()); + return false; } } LIBSESSION_C_API void session_network_free(network_object* network) { - delete static_cast*>(network->internals); + delete static_cast(network->internals); delete network; } @@ -1579,7 +1741,7 @@ LIBSESSION_C_API void session_network_get_active_paths( size_t total_metadata_size = 0; for (const auto& p : cpp_paths) { std::visit( - [&](const T& /*md*/) { + [&](const T&) { if constexpr (std::is_same_v) total_metadata_size += sizeof(session_onion_path_metadata); else { @@ -1671,7 +1833,7 @@ LIBSESSION_C_API void session_network_get_swarm( x25519_pubkey::from_hex({swarm_pubkey_hex, 64}), ignore_strike_count, [cb = std::move(callback), ctx](swarm_id_t, std::vector nodes) { - auto c_nodes = network::detail::convert_service_nodes(nodes); + auto c_nodes = convert_service_nodes(nodes); cb(c_nodes.data(), c_nodes.size(), ctx); }); } @@ -1684,7 +1846,7 @@ LIBSESSION_C_API void session_network_get_random_nodes( assert(callback); unbox(network)->get_random_nodes( count, [cb = std::move(callback), ctx](std::vector nodes) { - auto c_nodes = network::detail::convert_service_nodes(nodes); + auto c_nodes = convert_service_nodes(nodes); cb(c_nodes.data(), c_nodes.size(), ctx); }); } @@ -1745,9 +1907,9 @@ LIBSESSION_C_API void session_network_send_request( throw std::invalid_argument( "Invalid request: Must have either 'snode_dest' or 'server_dest' set."); - std::optional> body; + std::optional> body; if (params->body && params->body_size > 0) - body.emplace(params->body, params->body + params->body_size); + body = to_vector(to_byte_span(params->body, params->body_size)); std::optional request_id; if (params->request_id) @@ -1840,9 +2002,9 @@ LIBSESSION_C_API session_upload_handle_t* session_network_upload( const auto on_complete_fn = callbacks->on_complete; const auto ctx = callbacks->ctx; - cpp_request.next_data = [next_data_fn, ctx]() -> std::vector { - std::vector buffer(64 * 1024); // 64KB chunks - size_t bytes = next_data_fn(buffer.data(), buffer.size(), ctx); + cpp_request.next_data = [next_data_fn, ctx]() -> std::vector { + std::vector buffer(64 * 1024); // 64KB chunks + size_t bytes = next_data_fn(to_unsigned(buffer.data()), buffer.size(), ctx); if (bytes == 0 || bytes == static_cast(-1)) return {}; @@ -1873,8 +2035,11 @@ LIBSESSION_C_API session_upload_handle_t* session_network_upload( result); }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" handle->cancelled = cpp_request.cancelled; unbox(network)->upload(std::move(cpp_request)); +#pragma GCC diagnostic pop return handle.release(); } catch (...) { @@ -1889,7 +2054,6 @@ LIBSESSION_C_API session_download_handle_t* session_network_download( int64_t stall_timeout_ms, int64_t request_timeout_ms, int64_t overall_timeout_ms, - int64_t /*partial_min_interval_ms*/, int8_t desired_path_index) { if (!network || !download_url || !callbacks) @@ -1917,7 +2081,7 @@ LIBSESSION_C_API session_download_handle_t* session_network_download( if (on_data_fn) cpp_request.on_data = [on_data_fn, ctx]( const file_metadata& metadata, - std::vector data) { + std::span data) { session_file_metadata c_meta{}; std::strncpy(c_meta.file_id, metadata.id.c_str(), sizeof(c_meta.file_id) - 1); c_meta.file_id[sizeof(c_meta.file_id) - 1] = '\0'; @@ -1925,7 +2089,7 @@ LIBSESSION_C_API session_download_handle_t* session_network_download( c_meta.uploaded_timestamp = epoch_seconds(metadata.uploaded); c_meta.expiry_timestamp = epoch_seconds(metadata.expiry); - on_data_fn(&c_meta, data.data(), data.size(), ctx); + on_data_fn(&c_meta, to_unsigned(data.data()), data.size(), ctx); }; cpp_request.on_complete = [on_complete_fn, diff --git a/src/network/session_network_types.cpp b/src/network/session_network_types.cpp index 4b40e1325..b49c8deec 100644 --- a/src/network/session_network_types.cpp +++ b/src/network/session_network_types.cpp @@ -14,7 +14,7 @@ Request::Request( std::string request_id, network_destination destination, std::string endpoint, - std::optional> body, + std::optional> body, RequestCategory category, std::chrono::milliseconds request_timeout, std::optional overall_timeout, @@ -33,7 +33,7 @@ Request::Request( Request::Request( network_destination destination, std::string endpoint, - std::optional> body, + std::optional> body, RequestCategory category, std::chrono::milliseconds request_timeout, std::optional overall_timeout, diff --git a/src/network/snode_pool.cpp b/src/network/snode_pool.cpp index d4f61ad9a..a38ee6937 100644 --- a/src/network/snode_pool.cpp +++ b/src/network/snode_pool.cpp @@ -14,6 +14,7 @@ #include #include +#include "session/clock.hpp" #include "session/file.hpp" #include "session/hash.hpp" #include "session/random.hpp" @@ -46,17 +47,14 @@ namespace { SnodePool::SnodePool( config::SnodePool config, - std::shared_ptr loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, network_fetcher_t direct_fetcher) : _config{std::move(config)}, _loop{loop}, _disk_loop{disk_loop}, _direct_fetcher{std::move(direct_fetcher)} { - if (!_loop || !_disk_loop) - throw std::invalid_argument{"Cannot construct a SnodePool with an empty loop/disk_loop"}; - if (_config.cache_directory) { std::string cache_file_name; @@ -69,8 +67,9 @@ SnodePool::SnodePool( for (const auto& node : _config.seed_nodes) node.to_disk(std::back_inserter(seed_node_data)); - auto hash_bytes = session::hash::hash(32, session::to_span(seed_node_data)); - cache_file_name = "snode_pool_devnet_" + oxenc::to_hex(hash_bytes); + cache_file_name = + "snode_pool_devnet_" + + oxenc::to_hex(session::hash::blake2b<32>(session::to_span(seed_node_data))); break; } @@ -80,6 +79,12 @@ SnodePool::SnodePool( } } +SnodePool::~SnodePool() { + // Cancels the queue's jobs -- including the delayed refresh and strike-flush ones -- and waits + // out whatever is running, so that nothing is part-way through us when the members below go. + _jq.stop(); +} + // MARK: Disk I/O Functions // Consume a raw integer (in little-endian format) from the beginning of `b`, throwing if @@ -175,7 +180,7 @@ void SnodePool::_load_from_disk() { throw empty_file_exception{}; // We want to filter on load so we don't start the app with expired strikes - auto threshold = sysclock_now_s() - STRIKE_EXPIRY; + auto threshold = clock_now_s() - STRIKE_EXPIRY; std::string_view buf{ reinterpret_cast(loaded_strikes_data.data()), @@ -336,7 +341,7 @@ void SnodePool::_perform_strikes_write( // MARK: Refresh Functions void SnodePool::_refresh_snode_cache(std::optional request_id_opt) { - _loop->call([this, request_id_opt] { + _jq.call([this, request_id_opt] { if (_suspended) { log::info(cat, "Ignoring refresh as pool is suspended."); return; @@ -432,12 +437,12 @@ void SnodePool::_launch_next_refresh_request( const bool refreshing_from_seed_nodes, const bool use_direct_fetcher, const uint8_t total_requests) { - _loop->call([this, - request_id, - index, - refreshing_from_seed_nodes, - use_direct_fetcher, - total_requests] { + _jq.call([this, + request_id, + index, + refreshing_from_seed_nodes, + use_direct_fetcher, + total_requests] { if (!_current_snode_cache_refresh_id) return; @@ -544,14 +549,10 @@ void SnodePool::_launch_next_refresh_request( "trying again in {}ms.", target_request_id, delay.count()); - _loop->call_later(delay, [weak_self = weak_from_this(), this] { + _jq.call_later(delay, [this] { // We need to wait until after the `call_later` to reset the `refresh_id` (and clear // previous results) as if we don't then additional refreshes could be triggered // during the delay - auto self = weak_self.lock(); - if (!self) - return; - _current_snode_cache_refresh_id.reset(); _snode_refresh_results.clear(); _refresh_snode_cache(); @@ -592,7 +593,10 @@ void SnodePool::_launch_next_refresh_request( fetcher_to_use( request, - [this, + // The fetcher hands this to the transport, which outlives us, so unlike our own + // jobs this one cannot rely on the queue being stopped and keeps a weak guard. + [weak_self = weak_from_this(), + this, request_id, index, target_request_id, @@ -604,6 +608,10 @@ void SnodePool::_launch_next_refresh_request( int16_t status_code, std::vector> /*headers*/, std::optional response) { + auto self = weak_self.lock(); + if (!self) + return; + // If the refresh was cancelled or completed while we were in-flight, do nothing if (!_current_snode_cache_refresh_id || *_current_snode_cache_refresh_id != request_id) { @@ -642,21 +650,20 @@ void SnodePool::_launch_next_refresh_request( "{}ms.", e.what(), delay.count()); - _loop->call_later( + _jq.call_later( delay, - [weak_self = weak_from_this(), + [this, request_id, index, refreshing_from_seed_nodes, use_direct_fetcher, total_requests] { - if (auto self = weak_self.lock()) - self->_retry_refresh_request( - request_id, - index, - refreshing_from_seed_nodes, - use_direct_fetcher, - total_requests); + _retry_refresh_request( + request_id, + index, + refreshing_from_seed_nodes, + use_direct_fetcher, + total_requests); }); return; } @@ -707,27 +714,21 @@ void SnodePool::_on_refresh_complete( refresh_id, raw_results.size()); - _loop->call([this, - refresh_id, - raw_results, - refreshing_from_seed_nodes, - use_direct_fetcher, - total_requests] { + _jq.call([this, + refresh_id, + raw_results, + refreshing_from_seed_nodes, + use_direct_fetcher, + total_requests] { // Throw away everything we received and start the requests again after a backoff - auto discard_and_retry = [weak_self = weak_from_this(), + auto discard_and_retry = [this, refresh_id, refreshing_from_seed_nodes, use_direct_fetcher, total_requests](std::string_view reason) { - auto self = weak_self.lock(); - - if (!self) - return; - - self->_snode_refresh_results.clear(); - self->_snode_cache_refresh_failure_count++; - auto delay = - self->_config.retry_delay.exponential(self->_snode_cache_refresh_failure_count); + _snode_refresh_results.clear(); + _snode_cache_refresh_failure_count++; + auto delay = _config.retry_delay.exponential(_snode_cache_refresh_failure_count); log::error( cat, @@ -735,21 +736,20 @@ void SnodePool::_on_refresh_complete( refresh_id, delay.count(), reason); - self->_loop->call_later( + _jq.call_later( delay, - [weak_self, + [this, refresh_id, refreshing_from_seed_nodes, use_direct_fetcher, total_requests] { - if (auto self = weak_self.lock()) - for (uint8_t i = 0; i < total_requests; ++i) - self->_launch_next_refresh_request( - refresh_id, - i, - refreshing_from_seed_nodes, - use_direct_fetcher, - total_requests); + for (uint8_t i = 0; i < total_requests; ++i) + _launch_next_refresh_request( + refresh_id, + i, + refreshing_from_seed_nodes, + use_direct_fetcher, + total_requests); }); }; @@ -863,7 +863,7 @@ void SnodePool::_on_refresh_complete( void SnodePool::_update_cache(std::string refresh_id, std::vector nodes) { // Use 'call_get' to force this to be synchronous; a plain 'call' would queue when we aren't // already on the loop thread, and the captured `nodes` reference would dangle - _loop->call_get([this, refresh_id, &nodes] { + _jq.call_get([this, refresh_id, &nodes] { // Shuffle the nodes so we don't have a specific order std::ranges::shuffle(nodes, csrng); log::info( @@ -885,7 +885,7 @@ void SnodePool::_update_cache(std::string refresh_id, std::vector _refresh_candidate_nodes.clear(); _snode_cache_refresh_failure_count = 0; - _disk_loop->call([path = _snode_cache_file_path, cache = _snode_cache] { + _disk_loop.call([path = _snode_cache_file_path, cache = _snode_cache] { SnodePool::_perform_cache_write(path, cache); }); @@ -917,12 +917,12 @@ void SnodePool::_update_cache(std::string refresh_id, std::vector void SnodePool::suspend() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _suspended = true; // Force a strike write immediately if we had one scheduled if (_strikes_flush_scheduled) - _disk_loop->call([path = _strikes_file_path, strikes = _snode_strikes] { + _disk_loop.call([path = _strikes_file_path, strikes = _snode_strikes] { SnodePool::_perform_strikes_write(path, strikes); }); log::info(cat, "Suspended."); @@ -931,7 +931,7 @@ void SnodePool::suspend() { void SnodePool::resume() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { if (!_suspended) return; @@ -942,24 +942,24 @@ void SnodePool::resume() { void SnodePool::set_routed_fetcher( network_fetcher_t routed_fetcher, fetcher_connectivity_check_t connectivity_check) { - _loop->call([this, rf = std::move(routed_fetcher), cc = std::move(connectivity_check)] { + _jq.call([this, rf = std::move(routed_fetcher), cc = std::move(connectivity_check)] { _routed_fetcher = std::move(rf); _routed_fetcher_connectivity_check = std::move(cc); }); } size_t SnodePool::size() { - return _loop->call_get([this] { return _snode_cache.size(); }); + return _jq.call_get([this] { return _snode_cache.size(); }); } void SnodePool::clear_cache() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _snode_cache = {}; _all_swarms = {}; _swarm_cache = {}; - _disk_loop->call([path = _snode_cache_file_path] { SnodePool::_clear_disk_cache(path); }); + _disk_loop.call([path = _snode_cache_file_path] { SnodePool::_clear_disk_cache(path); }); }); } @@ -968,8 +968,8 @@ void SnodePool::record_node_failure(const service_node& node, bool permanent) { } void SnodePool::record_node_failure(const ed25519_pubkey& key, bool permanent) { - _loop->call([this, key, permanent] { - auto now = sysclock_now_s(); + _jq.call([this, key, permanent] { + auto now = clock_now_s(); if (permanent) for (int i = 0; i < _config.cache_node_strike_threshold; ++i) @@ -987,18 +987,15 @@ void SnodePool::record_node_failure(const ed25519_pubkey& key, bool permanent) { if (!_strikes_flush_scheduled && !_suspended) { _strikes_flush_scheduled = true; - _loop->call_later(SAVE_THROTTLE, [weak_self = weak_from_this()] { - if (auto self = weak_self.lock()) { - self->_strikes_flush_scheduled = false; + _jq.call_later(SAVE_THROTTLE, [this] { + _strikes_flush_scheduled = false; - if (self->_suspended) - return; + if (_suspended) + return; - self->_disk_loop->call( - [path = self->_strikes_file_path, strikes = self->_snode_strikes] { - SnodePool::_perform_strikes_write(path, strikes); - }); - } + _disk_loop.call([path = _strikes_file_path, strikes = _snode_strikes] { + SnodePool::_perform_strikes_write(path, strikes); + }); }); } }); @@ -1009,14 +1006,14 @@ uint16_t SnodePool::node_strike_count(const service_node& node) { } uint16_t SnodePool::node_strike_count(const ed25519_pubkey& key) { - return _loop->call_get([this, &key] { + return _jq.call_get([this, &key] { auto it = _snode_strikes.find(key); if (it == _snode_strikes.end()) return uint16_t{0}; const auto& stamps = it->second; - const auto threshold = sysclock_now_s() - STRIKE_EXPIRY; + const auto threshold = clock_now_s() - STRIKE_EXPIRY; uint16_t count = 0; for (auto t : stamps) @@ -1029,19 +1026,19 @@ uint16_t SnodePool::node_strike_count(const ed25519_pubkey& key) { void SnodePool::clear_node_strikes() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { _snode_strikes.clear(); _strikes_flush_scheduled = false; // Immediately write to disk after clearing the snode strikes - _disk_loop->call( + _disk_loop.call( [path = _strikes_file_path] { SnodePool::_perform_strikes_write(path, {}); }); }); } void SnodePool::refresh_if_needed( const std::vector& in_use_nodes, std::function on_refresh_complete) { - _loop->call([this, in_use_nodes, cb = std::move(on_refresh_complete)] { + _jq.call([this, in_use_nodes, cb = std::move(on_refresh_complete)] { if (_suspended) { log::info(cat, "Ignoring refresh as pool is suspended."); return; @@ -1102,10 +1099,7 @@ void SnodePool::refresh_if_needed( // on_refresh_complete callback immediately) if (needs_to_start_refresh) if (delay) { - _loop->call_later(*delay, [weak_self = weak_from_this()] { - if (auto self = weak_self.lock()) - self->_refresh_snode_cache(); - }); + _jq.call_later(*delay, [this] { _refresh_snode_cache(); }); } else _refresh_snode_cache(); else if (!already_running && cb) @@ -1118,12 +1112,9 @@ std::vector SnodePool::get_unused_nodes( // Kick of a cache refresh in the background if needed (call_soon to ensure it is scheduled // after whatever called `get_unused_nodes` which may be something trying to make it's own // request that we would want to run first) - _loop->call_soon([weak_self = weak_from_this(), exclude_nodes] { - if (auto self = weak_self.lock()) - self->refresh_if_needed(exclude_nodes); - }); + _jq.call_soon([this, exclude_nodes] { refresh_if_needed(exclude_nodes); }); - return _loop->call_get([this, count, exclude_nodes] { + return _jq.call_get([this, count, exclude_nodes] { if (_snode_cache.empty()) { log::warning(cat, "Cannot get unused nodes: snode cache is empty."); return std::vector{}; @@ -1189,7 +1180,7 @@ void SnodePool::get_swarm( std::function swarm)> callback) { log::trace(cat, "{} called for {}.", __PRETTY_FUNCTION__, swarm_pubkey.hex()); - _loop->call([this, swarm_pubkey, ignore_strike_count, cb = std::move(callback)] { + _jq.call([this, swarm_pubkey, ignore_strike_count, cb = std::move(callback)] { auto filter_by_strikes = [this](std::vector nodes) -> std::vector { // Shuffle everything to start with @@ -1252,10 +1243,7 @@ void SnodePool::get_swarm( } // Trigger a non-blocking background refresh if the data is stale - _loop->call_soon([weak_self = weak_from_this()] { - if (auto self = weak_self.lock()) - self->refresh_if_needed({}); - }); + _jq.call_soon([this] { refresh_if_needed({}); }); // Perform the swarm calculation using our local copy of the data auto swarm = swarm::get_swarm(swarm_pubkey, _all_swarms); diff --git a/src/network/transport/quic_transport.cpp b/src/network/transport/quic_transport.cpp index f90f72007..6f27681d6 100644 --- a/src/network/transport/quic_transport.cpp +++ b/src/network/transport/quic_transport.cpp @@ -4,7 +4,7 @@ #include #include -#include "session/ed25519.hpp" +#include "session/crypto/ed25519.hpp" #include "session/network/session_network_types.hpp" using namespace oxen; @@ -33,16 +33,22 @@ namespace { constexpr auto ALPN = "oxenstorage"sv; -QuicTransport::QuicTransport(config::QuicTransport config, std::shared_ptr loop) : +QuicTransport::QuicTransport(config::QuicTransport config, oxen::quic::Loop& loop) : _config{std::move(config)}, _loop{loop} { log::trace(cat, "Initializing."); _recreate_endpoint(); } QuicTransport::~QuicTransport() { - // Use 'call_get' to force this to be synchronous - if (_loop) - _loop->call_get([this] { _close_connections(); }); + // Nothing queued here runs after this, and whatever was running has finished by the time it + // returns -- so no job of ours can be part-way through when the members below go. + _jq.stop(); + + // Our own queue is stopped, so this last piece of teardown goes on the loop's. + // _close_connections resets the endpoint, and the endpoint's deleter destroys it *on the loop*, + // so once this returns libquic can no longer call into the connection callbacks below -- which + // is what lets those callbacks hold a bare `this`. + _loop.call_get([this] { _close_connections(); }); log::debug(cat, "Destroyed."); } @@ -50,7 +56,7 @@ QuicTransport::~QuicTransport() { void QuicTransport::suspend() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { if (!_suspended) return; @@ -62,7 +68,7 @@ void QuicTransport::suspend() { void QuicTransport::resume(bool /*automatically_reconnect*/) { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { + _jq.call_get([this] { // Recreate the endpoint before updating the `_suspended` flag to avoid the chance that // something will try to use it before we are ready _recreate_endpoint(); @@ -73,13 +79,12 @@ void QuicTransport::resume(bool /*automatically_reconnect*/) { void QuicTransport::close_connections() { // Use 'call_get' to force this to be synchronous - _loop->call_get([this] { _close_connections(); }); + _jq.call_get([this] { _close_connections(); }); } void QuicTransport::set_node_failure_reporter(node_failure_reporter_t reporter) { - _loop->call([weak_self = weak_from_this(), r = std::move(reporter)] { - if (auto self = weak_self.lock()) - self->_report_node_failure.emplace(std::move(r)); + _jq.call([this, r = std::move(reporter)]() mutable { + _report_node_failure.emplace(std::move(r)); }); } @@ -91,16 +96,7 @@ void QuicTransport::verify_connectivity( std::function error_code)> callback) { // For Quic, a successful connection IS a successful ping so we can just check for an existing // connection and, if one doesn't exist, try to establish one - _loop->call([weak_self = weak_from_this(), - this, - node = std::move(node), - cb = std::move(callback), - request_id, - category]() { - auto self = weak_self.lock(); - if (!self) - return; - + _jq.call([this, node = std::move(node), cb = std::move(callback), request_id, category]() { const auto pubkey_hex = node.remote_pubkey.hex(); // If we already have a connection we can stop here @@ -113,46 +109,37 @@ void QuicTransport::verify_connectivity( if (_pending_requests.count(pubkey_hex) == 0 && _pending_verification_callbacks.at(pubkey_hex).size() == 1) _establish_connection( - {node.remote_pubkey, node.host(), node.omq_port}, request_id, category); + {node.remote_pubkey.view(), node.host(), node.omq_port}, request_id, category); }); } void QuicTransport::add_failure_listener( const ed25519_pubkey& pubkey, std::function listener) { - _loop->call([weak_self = weak_from_this(), - pk_hex = pubkey.hex(), - l = std::move(listener)]() mutable { - if (auto self = weak_self.lock()) - self->_failure_listeners[pk_hex].push_back(std::move(l)); + _jq.call([this, pk_hex = pubkey.hex(), l = std::move(listener)]() mutable { + _failure_listeners[pk_hex].push_back(std::move(l)); }); } void QuicTransport::remove_failure_listeners(const ed25519_pubkey& pubkey) { - _loop->call([weak_self = weak_from_this(), pk_hex = pubkey.hex()] { - if (auto self = weak_self.lock()) - self->_failure_listeners.erase(pk_hex); - }); + _jq.call([this, pk_hex = pubkey.hex()] { _failure_listeners.erase(pk_hex); }); } void QuicTransport::send_request(Request request, network_response_callback_t callback) { log::trace(cat, "Dispatching request {} to loop.", request.request_id); - _loop->call([weak_self = weak_from_this(), req = std::move(request), cb = std::move(callback)] { - if (auto self = weak_self.lock()) - self->_send_request_internal(std::move(req), std::move(cb)); + _jq.call([this, req = std::move(request), cb = std::move(callback)]() mutable { + _send_request_internal(std::move(req), std::move(cb)); }); } // MARK: Internal Logic void QuicTransport::_recreate_endpoint() { - // The optional must CONTAIN the option to have any effect: libquic's optional-taking - // handle_ep_opt overload does nothing when the optional is empty, so a default-constructed - // std::optional{} silently leaves discovery enabled. _endpoint = quic::Endpoint::endpoint( - *_loop, + _loop, quic::Address{}, - (_config.disable_mtu_discovery ? std::make_optional() - : std::nullopt)); + (_config.max_udp_payload + ? std::make_optional(*_config.max_udp_payload) + : std::nullopt)); } void QuicTransport::_close_connections() { @@ -237,7 +224,7 @@ void QuicTransport::_send_request_internal(Request request, network_response_cal cat, "[Request {}]: Resolving service_node to RemoteAddress.", request_id); - remote.emplace(arg.remote_pubkey, arg.host(), arg.omq_port); + remote.emplace(arg.remote_pubkey.view(), arg.host(), arg.omq_port); } }, request.destination); @@ -297,8 +284,8 @@ void QuicTransport::_establish_connection( if (!_endpoint) throw std::runtime_error{"Network is invalid"}; - auto conn_key_pair = ed25519::ed25519_key_pair(); - auto creds = quic::GNUTLSCreds::make_from_ed_seckey(to_string_view(conn_key_pair.second)); + auto [conn_pk, conn_sk] = ed25519::keypair(); + auto creds = quic::GNUTLSCreds::make_from_ed_seckey(to_string_view(conn_sk)); // If we are starting a connection attempt then transition to the "connecting" state if (_status.load() == ConnectionStatus::unknown || @@ -317,12 +304,10 @@ void QuicTransport::_establish_connection( oxen::quic::opt::outbound_alpn(ALPN), oxen::quic::opt::handshake_timeout{_config.handshake_timeout}, oxen::quic::opt::keep_alive{_config.keep_alive}, - [weak_self = weak_from_this(), this, address_pubkey_hex, initiating_req_id]( - oxen::quic::Connection& conn) { - auto self = weak_self.lock(); - if (!self) - return; - + // libquic hands these a live Connection, so they run inline on the loop rather than + // as jobs of ours. ~QuicTransport destroys the endpoint on the loop before the + // members below are touched, so there is no window in which this can fire late. + [this, address_pubkey_hex, initiating_req_id](oxen::quic::Connection& conn) { log::info( cat, "[Request {}] Successfully established connection to {}.", @@ -362,11 +347,10 @@ void QuicTransport::_establish_connection( conn_id, address_pubkey_hex, std::move(req), std::move(cb)); } }, - [weak_self = weak_from_this(), address_pubkey_hex, initiating_req_id]( + [this, address_pubkey_hex, initiating_req_id]( oxen::quic::Connection&, uint64_t error_code) { - if (auto self = weak_self.lock()) - self->_fail_connection( - address_pubkey_hex, initiating_req_id, error_code, std::nullopt); + _fail_connection( + address_pubkey_hex, initiating_req_id, error_code, std::nullopt); }); } catch (const std::exception& e) { _fail_connection(address_pubkey_hex, initiating_req_id, std::nullopt, e.what()); @@ -486,17 +470,12 @@ void QuicTransport::_send_on_connection( request.endpoint, payload, timeout, - [weak_self = weak_from_this(), - this, + [this, cb = std::move(callback), conn_id, remote_pubkey_hex, stream_id = target_stream->stream_id(), req_id = request.request_id](quic::message resp) { - auto self = weak_self.lock(); - if (!self) - return; - log::trace(cat, "[Request {}] Received response.", req_id); // Since the request completed it's round-trip if it isn't the "reserverd" stream @@ -536,7 +515,7 @@ void QuicTransport::_send_on_connection( final_timeout = result->second; } - log::debug(cat, "[Request {}] Failed with QUIC error: {}.", req_id, err_body); + log::warning(cat, "[Request {}] Failed with QUIC error: {}.", req_id, err_body); return cb( false, final_timeout, diff --git a/src/onionreq/builder.cpp b/src/onionreq/builder.cpp index dcb8b2c06..dbb07c13c 100644 --- a/src/onionreq/builder.cpp +++ b/src/onionreq/builder.cpp @@ -50,8 +50,8 @@ namespace detail { namespace { - std::vector encode_size(uint32_t s) { - std::vector result; + std::vector encode_size(uint32_t s) { + std::vector result; result.resize(4); oxenc::write_host_as_little(s, result.data()); return result; @@ -63,7 +63,7 @@ EncryptType parse_enc_type(std::string_view enc_type) { return EncryptType::xchacha20; if (enc_type == "aes-gcm" || enc_type == "gcm") return EncryptType::aes_gcm; - throw std::runtime_error{"Invalid encryption type " + std::string{enc_type}}; + throw std::runtime_error{"Invalid encryption type {}"_format(enc_type)}; } Builder Builder::make( @@ -88,7 +88,7 @@ Builder::Builder( add_hop(n.remote_pubkey); } -void Builder::add_hop(std::span remote_key) { +void Builder::add_hop(std::span remote_key) { hops_.push_back( {network::ed25519_pubkey::from_bytes(remote_key), network::compute_x25519_pubkey(remote_key)}); @@ -121,13 +121,13 @@ void Builder::set_destination(network_destination destination) { throw std::invalid_argument{"Invalid destination type."}; } -std::vector Builder::generate_onion_blob( - const std::optional>& plaintext_body) { +std::vector Builder::generate_onion_blob( + const std::optional>& plaintext_body) { return build(_generate_payload(plaintext_body)); } -std::vector Builder::_generate_payload( - std::optional> body) const { +std::vector Builder::_generate_payload( + std::optional> body) const { // If we don't have the data required for a server request, then assume it's targeting a // service node which has a different structure (`method` is the endpoint and the body is // `params`) @@ -135,14 +135,15 @@ std::vector Builder::_generate_payload( nlohmann::json params_json; if (body && !body->empty()) - params_json = nlohmann::json::parse(*body); + // Parse as a char view: libc++'s std::char_traits has no std::byte specialization. + params_json = nlohmann::json::parse(to_string_view(*body)); else params_json = nlohmann::json::object(); nlohmann::json wrapped_payload = {{"method", endpoint_}, {"params", params_json}}; std::string payload_str = wrapped_payload.dump(); - return {payload_str.begin(), payload_str.end()}; + return to_vector(payload_str); } // Otherwise generate the payload for a server request @@ -174,11 +175,11 @@ std::vector Builder::_generate_payload( payload.emplace_back(session::to_string(*body)); auto result = oxenc::bt_serialize(payload); - return to_vector(result); + return to_vector(result); } -std::vector Builder::build(std::vector payload) { - std::vector blob; +std::vector Builder::build(std::vector payload) { + std::vector blob; // First hop: // @@ -223,7 +224,7 @@ std::vector Builder::build(std::vector payload) { nlohmann::json final_route; { - crypto_box_keypair(A.data(), a.data()); + crypto_box_keypair(to_unsigned(A.data()), to_unsigned(a.data())); HopEncryption e{a, A, false}; // The data we send to the destination differs depending on whether the destination is a @@ -251,7 +252,7 @@ std::vector Builder::build(std::vector payload) { }; auto control_dump = control.dump(); - auto control_span = to_span(control_dump); + auto control_span = to_span(control_dump); auto data = encode_size(payload.size()); data.insert(data.end(), payload.begin(), payload.end()); data.insert(data.end(), control_span.begin(), control_span.end()); @@ -293,7 +294,7 @@ std::vector Builder::build(std::vector payload) { data.insert(data.end(), routing_span.begin(), routing_span.end()); // Generate eph key for *this* request and encrypt it: - crypto_box_keypair(A.data(), a.data()); + crypto_box_keypair(to_unsigned(A.data()), to_unsigned(a.data())); HopEncryption e{a, A, false}; blob = e.encrypt(enc_type, data, it->second); } @@ -357,12 +358,8 @@ LIBSESSION_C_API void onion_request_builder_set_snode_destination( const char* ed25519_pubkey) { assert(builder && ip && ed25519_pubkey); - std::vector pubkey; - pubkey.reserve(32); - oxenc::from_hex(ed25519_pubkey, ed25519_pubkey + 64, std::back_inserter(pubkey)); - unbox(builder).set_destination(session::network::service_node{ - session::network::ed25519_pubkey::from_bytes(pubkey), + session::network::ed25519_pubkey::from_hex({ed25519_pubkey, 64}), oxen::quic::ipv4{std::span(ip, 4)}, 0, quic_port, @@ -411,7 +408,8 @@ LIBSESSION_C_API bool onion_request_builder_build( try { auto& unboxed_builder = unbox(builder); - auto payload = unboxed_builder.build({payload_in, payload_in + payload_in_len}); + auto payload = + unboxed_builder.build(session::to_vector(std::span{payload_in, payload_in_len})); if (unboxed_builder.final_hop_x25519_keypair) { auto key_pair = unboxed_builder.final_hop_x25519_keypair.value(); diff --git a/src/onionreq/hop_encryption.cpp b/src/onionreq/hop_encryption.cpp index 69a491c6c..1f8ad8b87 100644 --- a/src/onionreq/hop_encryption.cpp +++ b/src/onionreq/hop_encryption.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include "session/export.h" +#include "session/hash.hpp" #include "session/network/key_types.hpp" #include "session/onionreq/builder.hpp" #include "session/util.hpp" @@ -27,21 +27,22 @@ namespace session::onionreq { namespace { // Derive shared secret from our (ephemeral) `seckey` and the other party's `pubkey` - std::array calculate_shared_secret( + std::array calculate_shared_secret( const network::x25519_seckey& seckey, const network::x25519_pubkey& pubkey) { - std::array secret; - if (crypto_scalarmult(secret.data(), seckey.data(), pubkey.data()) != 0) + std::array secret; + if (crypto_scalarmult( + secret.data(), to_unsigned(seckey.data()), to_unsigned(pubkey.data())) != 0) throw std::runtime_error("Shared key derivation failed (crypto_scalarmult)"); return secret; } constexpr std::string_view salt{"LOKI"}; - std::array derive_symmetric_key( + std::array derive_symmetric_key( const network::x25519_seckey& seckey, const network::x25519_pubkey& pubkey) { auto key = calculate_shared_secret(seckey, pubkey); - auto usalt = to_span(salt); + auto usalt = to_span(salt); crypto_auth_hmacsha256_state state; @@ -64,17 +65,14 @@ namespace { static_assert(crypto_aead_xchacha20poly1305_ietf_KEYBYTES >= crypto_scalarmult_BYTES); if (0 != crypto_scalarmult( key.data(), - local_sec.data(), - remote_pub.data())) // Use key as tmp storage for aB + to_unsigned(local_sec.data()), + to_unsigned(remote_pub.data()))) // Use key as tmp storage for aB throw std::runtime_error{"Failed to compute shared key for xchacha20"}; - crypto_generichash_state h; - crypto_generichash_init(&h, nullptr, 0, key.size()); - crypto_generichash_update(&h, key.data(), crypto_scalarmult_BYTES); - crypto_generichash_update( - &h, (local_first ? local_pub : remote_pub).data(), local_pub.size()); - crypto_generichash_update( - &h, (local_first ? remote_pub : local_pub).data(), local_pub.size()); - crypto_generichash_final(&h, key.data(), key.size()); + hash::blake2b( + key, + key, + local_first ? local_pub : remote_pub, + local_first ? remote_pub : local_pub); return key; } @@ -89,9 +87,9 @@ bool HopEncryption::response_long_enough(EncryptType type, size_t response_size) return false; } -std::vector HopEncryption::encrypt( +std::vector HopEncryption::encrypt( EncryptType type, - std::vector plaintext, + std::vector plaintext, const network::x25519_pubkey& pubkey) const { switch (type) { case EncryptType::xchacha20: return encrypt_xchacha20(plaintext, pubkey); @@ -100,9 +98,9 @@ std::vector HopEncryption::encrypt( throw std::runtime_error{"Invalid encryption type"}; } -std::vector HopEncryption::decrypt( +std::vector HopEncryption::decrypt( EncryptType type, - std::vector ciphertext, + std::vector ciphertext, const network::x25519_pubkey& pubkey) const { switch (type) { case EncryptType::xchacha20: return decrypt_xchacha20(ciphertext, pubkey); @@ -111,8 +109,8 @@ std::vector HopEncryption::decrypt( throw std::runtime_error{"Invalid decryption type"}; } -std::vector HopEncryption::encrypt_aesgcm( - std::vector plaintext, const network::x25519_pubkey& pubKey) const { +std::vector HopEncryption::encrypt_aesgcm( + std::vector plaintext, const network::x25519_pubkey& pubKey) const { auto key = derive_symmetric_key(private_key_, pubKey); // Initialise cipher context with the key @@ -120,21 +118,21 @@ std::vector HopEncryption::encrypt_aesgcm( static_assert(key.size() == AES256_KEY_SIZE); gcm_aes256_set_key(&ctx, key.data()); - std::vector output; + std::vector output; output.resize(GCM_IV_SIZE + plaintext.size() + GCM_DIGEST_SIZE); // Start the output with the random IV, and load it into ctx auto* o = output.data(); randombytes_buf(o, GCM_IV_SIZE); - gcm_aes256_set_iv(&ctx, GCM_IV_SIZE, o); + gcm_aes256_set_iv(&ctx, GCM_IV_SIZE, to_unsigned(o)); o += GCM_IV_SIZE; // Append encrypted data - gcm_aes256_encrypt(&ctx, plaintext.size(), o, plaintext.data()); + gcm_aes256_encrypt(&ctx, plaintext.size(), to_unsigned(o), to_unsigned(plaintext.data())); o += plaintext.size(); // Append digest - gcm_aes256_digest(&ctx, GCM_DIGEST_SIZE, o); + gcm_aes256_digest(&ctx, GCM_DIGEST_SIZE, to_unsigned(o)); o += GCM_DIGEST_SIZE; assert(o == output.data() + output.size()); @@ -142,13 +140,12 @@ std::vector HopEncryption::encrypt_aesgcm( return output; } -std::vector HopEncryption::decrypt_aesgcm( - std::vector ciphertext_, const network::x25519_pubkey& pubKey) const { - std::span ciphertext = to_span(ciphertext_); +std::vector HopEncryption::decrypt_aesgcm( + std::span ciphertext, const network::x25519_pubkey& pubKey) const { - if (!response_long_enough(EncryptType::aes_gcm, ciphertext_.size())) + if (!response_long_enough(EncryptType::aes_gcm, ciphertext.size())) throw std::invalid_argument{ - "Ciphertext data is too short: " + session::to_string(ciphertext_)}; + fmt::format("Ciphertext data is too short: {}", ciphertext.size())}; auto key = derive_symmetric_key(private_key_, pubKey); @@ -157,18 +154,19 @@ std::vector HopEncryption::decrypt_aesgcm( static_assert(key.size() == AES256_KEY_SIZE); gcm_aes256_set_key(&ctx, key.data()); - gcm_aes256_set_iv(&ctx, GCM_IV_SIZE, ciphertext.data()); + gcm_aes256_set_iv(&ctx, GCM_IV_SIZE, to_unsigned(ciphertext.data())); ciphertext = ciphertext.subspan(GCM_IV_SIZE); auto digest_in = ciphertext.subspan(ciphertext.size() - GCM_DIGEST_SIZE); ciphertext = ciphertext.subspan(0, ciphertext.size() - GCM_DIGEST_SIZE); - std::vector plaintext; + std::vector plaintext; plaintext.resize(ciphertext.size()); - gcm_aes256_decrypt(&ctx, ciphertext.size(), plaintext.data(), ciphertext.data()); + gcm_aes256_decrypt( + &ctx, ciphertext.size(), to_unsigned(plaintext.data()), to_unsigned(ciphertext.data())); - std::array digest_out; + std::array digest_out; gcm_aes256_digest(&ctx, digest_out.size(), digest_out.data()); if (sodium_memcmp(digest_out.data(), digest_in.data(), GCM_DIGEST_SIZE) != 0) @@ -177,10 +175,10 @@ std::vector HopEncryption::decrypt_aesgcm( return plaintext; } -std::vector HopEncryption::encrypt_xchacha20( - std::vector plaintext, const network::x25519_pubkey& pubKey) const { +std::vector HopEncryption::encrypt_xchacha20( + std::vector plaintext, const network::x25519_pubkey& pubKey) const { - std::vector ciphertext; + std::vector ciphertext; ciphertext.resize( crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + plaintext.size() + crypto_aead_xchacha20poly1305_ietf_ABYTES); @@ -197,7 +195,7 @@ std::vector HopEncryption::encrypt_xchacha20( crypto_aead_xchacha20poly1305_ietf_encrypt( c, &clen, - plaintext.data(), + to_unsigned(plaintext.data()), plaintext.size(), nullptr, 0, // additional data @@ -209,22 +207,22 @@ std::vector HopEncryption::encrypt_xchacha20( return ciphertext; } -std::vector HopEncryption::decrypt_xchacha20( - std::vector ciphertext_, const network::x25519_pubkey& pubKey) const { - std::span ciphertext = to_span(ciphertext_); +std::vector HopEncryption::decrypt_xchacha20( + std::span ciphertext_, const network::x25519_pubkey& pubKey) const { + auto ciphertext = ciphertext_; // Extract nonce from the beginning of the ciphertext: auto nonce = ciphertext.subspan(0, crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); ciphertext = ciphertext.subspan(nonce.size()); - if (!response_long_enough(EncryptType::xchacha20, ciphertext_.size())) + if (!response_long_enough(EncryptType::xchacha20, ciphertext.size())) throw std::invalid_argument{ "Ciphertext data is too short: " + std::string(reinterpret_cast(ciphertext_.data()))}; const auto key = xchacha20_shared_key(public_key_, private_key_, pubKey, !server_); - std::vector plaintext; + std::vector plaintext; plaintext.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES); auto* m = reinterpret_cast(plaintext.data()); unsigned long long mlen; @@ -232,11 +230,11 @@ std::vector HopEncryption::decrypt_xchacha20( m, &mlen, nullptr, // nsec (always unused) - ciphertext.data(), + to_unsigned(ciphertext.data()), ciphertext.size(), nullptr, 0, // additional data - nonce.data(), + to_unsigned(nonce.data()), key.data())) throw std::runtime_error{"Could not decrypt (XChaCha20-Poly1305)"}; assert(mlen <= plaintext.size()); diff --git a/src/onionreq/parser.cpp b/src/onionreq/parser.cpp index e83640d6a..a510427d3 100644 --- a/src/onionreq/parser.cpp +++ b/src/onionreq/parser.cpp @@ -6,12 +6,14 @@ #include #include +#include "session/util.hpp" + namespace session::onionreq { OnionReqParser::OnionReqParser( - std::span x25519_pk, - std::span x25519_sk, - std::span req, + std::span x25519_pk, + std::span x25519_sk, + std::span req, size_t max_size) : keys{network::x25519_pubkey::from_bytes(x25519_pk), network::x25519_seckey::from_bytes(x25519_sk)}, @@ -29,7 +31,9 @@ OnionReqParser::OnionReqParser( throw std::invalid_argument{"encrypted onion request data segment too small"}; auto ciphertext = req.subspan(0, size); req = req.subspan(size); - auto metadata = nlohmann::json::parse(req); + // Parse as a char view: nlohmann instantiates char_traits, and libc++ only + // specializes std::char_traits for the standard character types (not std::byte). + auto metadata = nlohmann::json::parse(to_string_view(req)); if (auto encit = metadata.find("enc_type"); encit != metadata.end()) enc_type = parse_enc_type(encit->get()); @@ -40,12 +44,11 @@ OnionReqParser::OnionReqParser( else throw std::invalid_argument{"metadata does not have 'ephemeral_key' entry"}; - payload_ = enc.decrypt(enc_type, to_vector(ciphertext), remote_pk); + payload_ = enc.decrypt(enc_type, to_vector(ciphertext), remote_pk); } -std::vector OnionReqParser::encrypt_reply( - std::span reply) const { - return enc.encrypt(enc_type, to_vector(reply), remote_pk); +std::vector OnionReqParser::encrypt_reply(std::span reply) const { + return enc.encrypt(enc_type, to_vector(reply), remote_pk); } } // namespace session::onionreq diff --git a/src/onionreq/response_parser.cpp b/src/onionreq/response_parser.cpp index 89230aee4..f4a9b533c 100644 --- a/src/onionreq/response_parser.cpp +++ b/src/onionreq/response_parser.cpp @@ -4,13 +4,15 @@ #include #include +#include #include #include "session/export.h" -#include "session/network/service_node.hpp" #include "session/onionreq/builder.h" #include "session/onionreq/builder.hpp" #include "session/onionreq/hop_encryption.hpp" +#include "session/onionreq/response_parser.h" +#include "session/util.hpp" using namespace session; @@ -34,7 +36,7 @@ bool ResponseParser::response_long_enough(EncryptType enc_type, size_t response_ return HopEncryption::response_long_enough(enc_type, response_size); } -std::vector ResponseParser::decrypt(std::vector ciphertext) const { +std::vector ResponseParser::decrypt(std::vector ciphertext) const { HopEncryption d{x25519_keypair_.second, x25519_keypair_.first, false}; // FIXME: The legacy PN server doesn't support 'xchacha20' onion requests so would return an @@ -85,13 +87,14 @@ DecryptedResponse ResponseParser::_decrypt_v3_response(const std::string& respon if (!oxenc::is_base64(base64_iv_and_ciphertext)) throw std::runtime_error{"Invalid base64 encoded IV and ciphertext."}; - std::vector iv_and_ciphertext; + std::vector iv_and_ciphertext; oxenc::from_base64( base64_iv_and_ciphertext.begin(), base64_iv_and_ciphertext.end(), std::back_inserter(iv_and_ciphertext)); auto result = decrypt(iv_and_ciphertext); - auto result_json = nlohmann::json::parse(result); + // Parse as a char view: libc++'s std::char_traits has no std::byte specialization. + auto result_json = nlohmann::json::parse(to_string_view(result)); int16_t status_code; std::vector> headers; std::string body; @@ -180,18 +183,17 @@ LIBSESSION_C_API bool onion_request_decrypt( break; default: - throw std::runtime_error{"Invalid decryption type " + std::to_string(enc_type_)}; + throw std::runtime_error{ + "Invalid decryption type {}"_format(static_cast(enc_type_))}; } session::onionreq::HopEncryption d{ - session::network::x25519_seckey::from_bytes({final_x25519_seckey, 32}), - session::network::x25519_pubkey::from_bytes({final_x25519_pubkey, 32}), + session::network::x25519_seckey::from_bytes(to_byte_span<32>(final_x25519_seckey)), + session::network::x25519_pubkey::from_bytes(to_byte_span<32>(final_x25519_pubkey)), false}; - std::vector result; - std::vector ciphertext; - ciphertext.reserve(ciphertext_len); - ciphertext.assign(ciphertext_, ciphertext_ + ciphertext_len); + std::vector result; + std::vector ciphertext{to_vector(to_byte_span(ciphertext_, ciphertext_len))}; // FIXME: The legacy PN server doesn't support 'xchacha20' onion requests so would return an // error encrypted with 'aes_gcm' so try to decrypt in case that is what happened - this @@ -200,14 +202,15 @@ LIBSESSION_C_API bool onion_request_decrypt( result = d.decrypt( enc_type, ciphertext, - session::network::x25519_pubkey::from_bytes({destination_x25519_pubkey, 32})); + session::network::x25519_pubkey::from_bytes( + to_byte_span<32>(destination_x25519_pubkey))); } catch (...) { if (enc_type == session::onionreq::EncryptType::xchacha20) result = d.decrypt( session::onionreq::EncryptType::aes_gcm, ciphertext, session::network::x25519_pubkey::from_bytes( - {destination_x25519_pubkey, 32})); + to_byte_span<32>(destination_x25519_pubkey))); else return false; } diff --git a/src/pro_backend.cpp b/src/pro_backend.cpp index 9ee501303..3623b3bce 100644 --- a/src/pro_backend.cpp +++ b/src/pro_backend.cpp @@ -4,24 +4,25 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include +#include "internal-util.hpp" #include "json_parser.hpp" #include "pro_message.hpp" namespace { -using namespace session::detail; - // Fractional UNIX seconds (double) -> millisecond-precision system time, preserving the provider's // sub-second precision (rounded to the nearest millisecond). session::sys_ms sys_ms_from_seconds(double seconds) { @@ -53,28 +54,6 @@ namespace { // content type travels alongside the payload so clients never hardcode a format. constexpr char application_json[] = "application/json"; - // Normalise an Ed25519 private key to libsodium's 64-byte form (seed(32) || pubkey(32)): a - // 32-byte seed is expanded, a 64-byte key copied, anything else throws. The public key is then - // available at bytes [32, 64) so callers never need a second derivation. - cleared_uc64 normalize_privkey(std::span privkey, const char* name) { - cleared_uc64 out; - if (privkey.size() == crypto_sign_ed25519_SEEDBYTES) { - array_uc32 pubkey; - crypto_sign_ed25519_seed_keypair(pubkey.data(), out.data(), privkey.data()); - } else if (privkey.size() == crypto_sign_ed25519_SECRETKEYBYTES) { - std::memcpy(out.data(), privkey.data(), crypto_sign_ed25519_SECRETKEYBYTES); - } else { - throw std::invalid_argument{fmt::format("Invalid {}: expected 32 or 64 bytes", name)}; - } - return out; - } - - // Public-key view (bytes [32, 64)) of a normalised 64-byte private key. - std::span pubkey_of(const cleared_uc64& priv64) { - return std::span( - priv64.data() + crypto_sign_ed25519_SEEDBYTES, crypto_sign_ed25519_PUBLICKEYBYTES); - } - } // namespace // C endpoint symbols: each points at the single master endpoint string defined above, so the C API @@ -91,9 +70,10 @@ LIBSESSION_EXPORT extern const char* const SESSION_PRO_BACKEND_GET_PRO_REVOCATIO // Backend base URL + Ed25519 pubkey: C symbols pointing at the single C++ definitions above. LIBSESSION_EXPORT extern const char* const SESSION_PRO_BACKEND_URL = URL.data(); -LIBSESSION_EXPORT extern const unsigned char* const SESSION_PRO_BACKEND_PUBKEY = PUBKEY.data(); +LIBSESSION_EXPORT extern const unsigned char* const SESSION_PRO_BACKEND_PUBKEY = + reinterpret_cast(PUBKEY.data()); LIBSESSION_EXPORT extern const unsigned char* const SESSION_PRO_BACKEND_PUBKEY_X25519 = - PUBKEY_X25519.data(); + reinterpret_cast(PUBKEY_X25519.data()); // Payment-provider code strings: C symbols pointing at the single C++ definitions above. LIBSESSION_EXPORT extern const char* const SESSION_PRO_BACKEND_PAYMENT_PROVIDER_CODE_GOOGLE_PLAY = @@ -136,24 +116,6 @@ const ProviderURLs* provider_urls(std::string_view provider_code) { return nullptr; } -LIBSESSION_C_API session_pro_backend_provider_urls -session_pro_backend_get_provider_urls(const char* provider_code) { - // Each present field is a static, null-terminated literal; an absent one is NULL. `found` - // distinguishes an unknown provider ({} -> found == false) from a recognised provider that - // simply has some/all URLs absent. - auto c = [](const std::optional& u) -> const char* { - return u ? u->data() : nullptr; - }; - if (auto u = provider_urls(provider_code)) - return {.found = true, - .refund_platform_url = c(u->refund_platform_url), - .refund_support_url = c(u->refund_support_url), - .refund_status_url = c(u->refund_status_url), - .update_subscription_url = c(u->update_subscription_url), - .cancel_subscription_url = c(u->cancel_subscription_url)}; - return {}; -} - std::span visible_platforms() { static const std::array platforms = { PAYMENT_PROVIDER_GOOGLE_PLAY, PAYMENT_PROVIDER_APP_STORE}; @@ -190,20 +152,6 @@ std::optional parse_plan_period(std::string_view code) { return ProPlanPeriod{count, unit}; } -LIBSESSION_C_API const char* const* session_pro_backend_visible_platforms(size_t* count) { - // Derived from the C++ list (single source); each slug's data() is a static, null-terminated - // literal, so the pointer array is safe to hand out as static storage. - static const std::vector codes = [] { - std::vector v; - for (auto slug : visible_platforms()) - v.push_back(slug.data()); - return v; - }(); - if (count) - *count = codes.size(); - return codes.data(); -} - namespace { // libsession-side slug when the backend's reply can't be parsed at all (malformed envelope, // missing/unrecognized status). Distinct from any backend error_code slug. @@ -215,17 +163,17 @@ namespace { // + `error`. Returns the `result` object to read the payload from when status is "ok" (an empty // object otherwise; the caller returns early on `!result`). Throws parse_error on a malformed // envelope. - nlohmann::json::object_t read_envelope(std::string_view json, ResponseBase& result) { - nlohmann::json j = json_parse(json); - auto status = json_require(j, "status"); + nlohmann::json::object_t read_envelope(std::string_view json_in, ResponseBase& result) { + nlohmann::json j = json::parse(json_in); + auto status = json::require(j, "status"); if (status == "ok") { result.status = ResponseStatus::Ok; - return json_require(j, "result"); + return json::require(j, "result"); } if (status == "fail" || status == "error") { result.status = status == "fail" ? ResponseStatus::Fail : ResponseStatus::Error; - result.error_code = json_require(j, "error_code"); - result.error = json_require(j, "error"); + result.error_code = json::require(j, "error_code"); + result.error = json::require(j, "error"); return {}; } throw parse_error{fmt::format("Unrecognized response status: '{}'", status)}; @@ -237,32 +185,32 @@ namespace { // No `version` to read: the format is fixed by the endpoint we asked, and is bound into the // proof's signature by its domain prefix rather than carried as a field. A future format is // a new endpoint returning its own proof type, not a version bump on this response. - result.proof.expiry_at = json_require(result_obj, "expiry_ts"); - json_require_hex(result_obj, "revocation_tag", result.proof.revocation_tag); - json_require_hex(result_obj, "rotating_pkey", result.proof.rotating_pubkey); - json_require_hex(result_obj, "sig", result.proof.sig); + result.proof.expiry_at = json::require(result_obj, "expiry_ts"); + json::require_binary(result_obj, "revocation_tag", result.proof.revocation_tag); + json::require_binary(result_obj, "rotating_pkey", result.proof.rotating_pubkey); + json::require_binary(result_obj, "sig", result.proof.sig); // Advisory and unsigned (pro-wire-protocol.md §2.2) -- never fed into signature // verification -- but required: a proof response without it can't refresh the cached access // expiry, which breaks renewal, so treat a missing value as a malformed response. result.account_expiry = - json_require(result_obj, "account_expiry_ts"); + json::require(result_obj, "account_expiry_ts"); // The two values that qualify `account_expiry_ts`. Absent -> keep the {0}/{false} default: // the backend sends `0`/`false` whenever grace/renewal don't apply and omits them on a // backend that predates them, so a missing field means "not applicable", never "keep the // stale value". A present value must still be well-formed (a wrong type throws). if (result_obj.contains("account_grace_period_duration")) - result.account_grace_period = - json_require(result_obj, "account_grace_period_duration"); + result.account_grace_period = json::require( + result_obj, "account_grace_period_duration"); if (result_obj.contains("account_auto_renewing")) - result.account_auto_renewing = json_require(result_obj, "account_auto_renewing"); + result.account_auto_renewing = json::require(result_obj, "account_auto_renewing"); } } // namespace -GenerateProProofResponse parse_pro_proof(std::string_view json) { +GenerateProProofResponse parse_pro_proof(std::string_view json_in) { GenerateProProofResponse result = {}; - auto result_obj = read_envelope(json, result); + auto result_obj = read_envelope(json_in, result); if (!result) { // On a subscription_expired failure the account's (now-past) true expiry rides top-level on // the envelope, alongside the same grace/renewal qualifiers as the success path @@ -271,17 +219,17 @@ GenerateProProofResponse parse_pro_proof(std::string_view json) { // grace`, and an expiry has typically NOT changed on this failure (it is a // lapse/cancellation) while the grace and flag have -- refreshing only the expiry would // leave a stale grace claiming coverage the backend has stopped honouring. Read leniently, - // absent -> {0}/{false} as on the success path (the envelope already parsed, so json_parse + // absent -> {0}/{false} as on the success path (the envelope already parsed, so json::parse // here can't throw). if (result.error_code == "subscription_expired") { - auto j = json_parse(json); - if (auto expiry = json_maybe(j, "account_expiry_ts")) + auto j = json::parse(json_in); + if (auto expiry = json::maybe(j, "account_expiry_ts")) result.account_expiry = *expiry; if (j.contains("account_grace_period_duration")) result.account_grace_period = - json_require(j, "account_grace_period_duration"); + json::require(j, "account_grace_period_duration"); if (j.contains("account_auto_renewing")) - result.account_auto_renewing = json_require(j, "account_auto_renewing"); + result.account_auto_renewing = json::require(j, "account_auto_renewing"); } return result; } @@ -293,38 +241,22 @@ namespace { // --- generate-proof (endpoint generate_pro_proof) --- - std::vector generate_proof_message( - std::span master_pubkey, - std::span rotating_pubkey, + std::vector generate_proof_message( + std::span master_pubkey, + std::span rotating_pubkey, std::chrono::sys_seconds unix_ts) { // Must match the generate-proof signed-request message in pro-wire-protocol.md §3.1, // built per §1.1. - return session::pro::signed_message( - session::GENERATE_PROOF_DOMAIN, - master_pubkey, - rotating_pubkey, - epoch_seconds(unix_ts)); - } - - MasterRotatingSignatures generate_proof_sign( - const cleared_uc64& master, - const cleared_uc64& rotating, - std::chrono::sys_seconds unix_ts) { - auto msg = generate_proof_message(pubkey_of(master), pubkey_of(rotating), unix_ts); - MasterRotatingSignatures result = {}; - crypto_sign_ed25519_detached( - result.master_sig.data(), nullptr, msg.data(), msg.size(), master.data()); - crypto_sign_ed25519_detached( - result.rotating_sig.data(), nullptr, msg.data(), msg.size(), rotating.data()); - return result; + return pro::signed_message( + GENERATE_PROOF_DOMAIN, master_pubkey, rotating_pubkey, epoch_seconds(unix_ts)); } std::string generate_proof_body( - std::span master_pubkey, - std::span rotating_pubkey, + std::span master_pubkey, + std::span rotating_pubkey, std::chrono::sys_seconds unix_ts, - std::span master_sig, - std::span rotating_sig) { + std::span master_sig, + std::span rotating_sig) { return nlohmann::json{ {"master_pkey", oxenc::to_hex(master_pubkey)}, {"rotating_pkey", oxenc::to_hex(rotating_pubkey)}, @@ -337,20 +269,18 @@ namespace { } // namespace ProRequest pro_proof_request( - std::span master_privkey, - std::span rotating_privkey, + const ed25519::PrivKeySpan& master_privkey, + const ed25519::PrivKeySpan& rotating_privkey, std::chrono::sys_seconds unix_ts) { - auto master = normalize_privkey(master_privkey, "master_privkey"); - auto rotating = normalize_privkey(rotating_privkey, "rotating_privkey"); - auto sigs = generate_proof_sign(master, rotating, unix_ts); + auto msg = generate_proof_message(master_privkey.pubkey(), rotating_privkey.pubkey(), unix_ts); return {generate_proof_endpoint, application_json, generate_proof_body( - pubkey_of(master), - pubkey_of(rotating), + master_privkey.pubkey(), + rotating_privkey.pubkey(), unix_ts, - sigs.master_sig, - sigs.rotating_sig)}; + ed25519::sign(master_privkey, msg), + ed25519::sign(rotating_privkey, msg))}; } ProRequest revocations_request(std::int64_t ticket) { @@ -359,16 +289,16 @@ ProRequest revocations_request(std::int64_t ticket) { return {get_pro_revocations_endpoint, application_json, j.dump()}; } -GetProRevocationsResponse parse_revocations(std::string_view json) { +GetProRevocationsResponse parse_revocations(std::string_view json_in) { GetProRevocationsResponse result = {}; - auto result_obj = read_envelope(json, result); + auto result_obj = read_envelope(json_in, result); if (!result) return result; // Parse payload - result.ticket = json_require(result_obj, "ticket"); - result.retry_in = json_require(result_obj, "retry_in"); - result.retain_for = json_require(result_obj, "retain_for"); + result.ticket = json::require(result_obj, "ticket"); + result.retry_in = json::require(result_obj, "retry_in"); + result.retain_for = json::require(result_obj, "retain_for"); // Clamp values against non-sensical/catastrophic values: retry_in of very small would result in // excess retries, and an overly long retry (e.g. 10 years) would make the client not properly @@ -377,7 +307,7 @@ GetProRevocationsResponse parse_revocations(std::string_view json) { result.retry_in = std::clamp(result.retry_in, 60s, 48h); result.retain_for = std::clamp(result.retain_for, 24h, 365 * 24h); - auto array = json_require(result_obj, "items"); + auto array = json::require(result_obj, "items"); result.items.reserve(array.size()); for (size_t index = 0; index < array.size(); index++) { const auto& it = array[index]; @@ -387,8 +317,8 @@ GetProRevocationsResponse parse_revocations(std::string_view json) { auto obj = it.get(); ProRevocationItem item = {}; - item.effective_at = json_require(obj, "effective_ts"); - json_require_hex(obj, "revocation_tag", item.revocation_tag); + item.effective_at = json::require(obj, "effective_ts"); + json::require_binary(obj, "revocation_tag", item.revocation_tag); result.items.emplace_back(std::move(item)); } @@ -399,24 +329,16 @@ namespace { // --- get-pro-status (endpoint get_pro_status) --- - std::vector pro_status_message( - std::span master_pubkey, std::chrono::sys_seconds unix_ts) { + std::vector pro_status_message( + std::span master_pubkey, std::chrono::sys_seconds unix_ts) { // Must match the get-pro-status signed-request message in pro-wire-protocol.md §3.2, built // per §1.1. - return session::pro::signed_message( - session::GET_PRO_STATUS_DOMAIN, master_pubkey, epoch_seconds(unix_ts)); - } - - array_uc64 pro_status_sign(const cleared_uc64& master, std::chrono::sys_seconds unix_ts) { - auto msg = pro_status_message(pubkey_of(master), unix_ts); - array_uc64 sig = {}; - crypto_sign_ed25519_detached(sig.data(), nullptr, msg.data(), msg.size(), master.data()); - return sig; + return pro::signed_message(GET_PRO_STATUS_DOMAIN, master_pubkey, epoch_seconds(unix_ts)); } std::string pro_status_body( - std::span master_pubkey, - std::span master_sig, + std::span master_pubkey, + std::span master_sig, std::chrono::sys_seconds unix_ts) { return nlohmann::json{ {"master_pkey", oxenc::to_hex(master_pubkey)}, @@ -427,36 +349,21 @@ namespace { // --- get-payment-details (endpoint get_payment_details) --- - std::vector payment_details_message( - std::span master_pubkey, + std::vector payment_details_message( + std::span master_pubkey, std::chrono::sys_seconds unix_ts, uint32_t limit, std::string_view before) { // Must match the get-payment-details signed-request message in pro-wire-protocol.md §3.3, // built per §1.1. `before` is the opaque pagination cursor (§5.3), empty for the newest // page. - return session::pro::signed_message( - session::GET_PAYMENT_DETAILS_DOMAIN, - master_pubkey, - epoch_seconds(unix_ts), - limit, - before); - } - - array_uc64 payment_details_sign( - const cleared_uc64& master, - std::chrono::sys_seconds unix_ts, - uint32_t limit, - std::string_view before) { - auto msg = payment_details_message(pubkey_of(master), unix_ts, limit, before); - array_uc64 sig = {}; - crypto_sign_ed25519_detached(sig.data(), nullptr, msg.data(), msg.size(), master.data()); - return sig; + return pro::signed_message( + GET_PAYMENT_DETAILS_DOMAIN, master_pubkey, epoch_seconds(unix_ts), limit, before); } std::string payment_details_body( - std::span master_pubkey, - std::span master_sig, + std::span master_pubkey, + std::span master_sig, std::chrono::sys_seconds unix_ts, uint32_t limit, std::string_view before) { @@ -472,28 +379,29 @@ namespace { // Parse one payment item object (shared by get-pro-status's `latest_payment` and // get-payment-details's `items`). Throws parse_error on any malformed field. ProPaymentItem parse_payment_item(const nlohmann::json::object_t& obj) { - auto status = json_require(obj, "status"); - auto plan_code = json_require(obj, "plan"); + auto status = json::require(obj, "status"); + auto plan_code = json::require(obj, "plan"); auto plan = parse_plan_period(plan_code); if (!plan) throw parse_error{ fmt::format("'plan' is not a recognized billing-period code: '{}'", plan_code)}; - auto payment_provider = json_require(obj, "payment_provider"); - auto payment_id = json_require(obj, "payment_id"); - auto auto_renewing = json_require(obj, "auto_renewing"); + auto payment_provider = json::require(obj, "payment_provider"); + auto payment_id = json::require(obj, "payment_id"); + auto auto_renewing = json::require(obj, "auto_renewing"); // purchased_ts and revoked_ts are upstream-provider event instants: floats on the wire // carrying sub-second precision (kept as millisecond-precision sys_ms). All other // timestamps are whole-second integers. - auto purchased_ts = json_require(obj, "purchased_ts"); - auto expiry_at = json_require(obj, "expiry_ts"); + auto purchased_ts = json::require(obj, "purchased_ts"); + auto expiry_at = json::require(obj, "expiry_ts"); auto grace_period_duration = - json_require(obj, "grace_period_duration"); + json::require(obj, "grace_period_duration"); auto platform_refund_expiry_at = - json_require(obj, "platform_refund_expiry_ts"); - auto revoked_ts = json_require(obj, "revoked_ts"); + json::require(obj, "platform_refund_expiry_ts"); + auto revoked_ts = json::require(obj, "revoked_ts"); ProPaymentItem item = {}; item.status = std::move(status); + // payment_provider / payment_id are opaque strings that pass through as-is. item.plan = *plan; item.payment_provider = std::move(payment_provider); item.payment_id = std::move(payment_id); @@ -509,40 +417,43 @@ namespace { } // namespace ProRequest pro_status_request( - std::span master_privkey, std::chrono::sys_seconds unix_ts) { - auto master = normalize_privkey(master_privkey, "master_privkey"); - auto sig = pro_status_sign(master, unix_ts); + const ed25519::PrivKeySpan& master_privkey, std::chrono::sys_seconds unix_ts) { + auto msg = pro_status_message(master_privkey.pubkey(), unix_ts); return {get_pro_status_endpoint, application_json, - pro_status_body(pubkey_of(master), sig, unix_ts)}; + pro_status_body(master_privkey.pubkey(), ed25519::sign(master_privkey, msg), unix_ts)}; } ProRequest payment_details_request( - std::span master_privkey, + const ed25519::PrivKeySpan& master_privkey, std::chrono::sys_seconds unix_ts, uint32_t limit, std::string_view before) { - auto master = normalize_privkey(master_privkey, "master_privkey"); - auto sig = payment_details_sign(master, unix_ts, limit, before); + auto msg = payment_details_message(master_privkey.pubkey(), unix_ts, limit, before); return {get_payment_details_endpoint, application_json, - payment_details_body(pubkey_of(master), sig, unix_ts, limit, before)}; + payment_details_body( + master_privkey.pubkey(), + ed25519::sign(master_privkey, msg), + unix_ts, + limit, + before)}; } -ProStatusResponse parse_pro_status(std::string_view json) { +ProStatusResponse parse_pro_status(std::string_view json_in) { ProStatusResponse result = {}; - auto result_obj = read_envelope(json, result); + auto result_obj = read_envelope(json_in, result); if (!result) return result; // Parse payload. The account Pro status is an opaque string code ("never"/"active"/"expired"); // an unknown value passes through unchanged (§1: enums are codes) rather than failing the - // parse. (Wire key `user_status`, disambiguated from the envelope `status` -- spec §5.2.) - result.user_status = json_require(result_obj, "user_status"); - result.auto_renewing = json_require(result_obj, "auto_renewing"); - result.expiry_at = json_require(result_obj, "expiry_ts"); + // parse. + result.user_status = json::require(result_obj, "user_status"); + result.auto_renewing = json::require(result_obj, "auto_renewing"); + result.expiry_at = json::require(result_obj, "expiry_ts"); result.grace_period_duration = - json_require(result_obj, "grace_period_duration"); + json::require(result_obj, "grace_period_duration"); // `latest_payment` is a single payment item, or null when the account has no payments. if (auto it = result_obj.find("latest_payment"); it == result_obj.end()) @@ -557,15 +468,15 @@ ProStatusResponse parse_pro_status(std::string_view json) { return result; } -PaymentDetailsResponse parse_payment_details(std::string_view json) { +PaymentDetailsResponse parse_payment_details(std::string_view json_in) { PaymentDetailsResponse result = {}; - auto result_obj = read_envelope(json, result); + auto result_obj = read_envelope(json_in, result); if (!result) return result; - result.payments_total = json_require(result_obj, "payments_total"); + result.payments_total = json::require(result_obj, "payments_total"); - auto array = json_require(result_obj, "items"); + auto array = json::require(result_obj, "items"); result.items.reserve(array.size()); for (size_t index = 0; index < array.size(); index++) { const auto& it = array[index]; @@ -590,6 +501,7 @@ PaymentDetailsResponse parse_payment_details(std::string_view json) { } // namespace session::pro_backend +using namespace session; using namespace session::pro_backend; // error / error_code strings libsession synthesizes when it cannot parse or build the C response. @@ -643,7 +555,7 @@ static void set_c_protocol_error(session_pro_backend_response_header& header, co // can't bind. static session_pro_backend_pro_revocation_item to_c(ProRevocationItem& src) { return { - .revocation_tag = src.revocation_tag.data(), + .revocation_tag = reinterpret_cast(src.revocation_tag.data()), .effective_ts = session::epoch_seconds(src.effective_at), }; } @@ -708,7 +620,7 @@ static session_pro_backend_request c_own_request(ProRequest&& req) { session_pro_backend_request result = {}; result.endpoint = owned->endpoint.data(); result.content_type = owned->content_type.data(); - result.data = span_u8{reinterpret_cast(owned->data.data()), owned->data.size()}; + result.data = span_u8{reinterpret_cast(owned->data.data()), owned->data.size()}; result.success = true; result.internal_ = owned.release(); return result; @@ -716,26 +628,52 @@ static session_pro_backend_request c_own_request(ProRequest&& req) { // Fill a session_pro_backend_request's error buffer from a caught exception. static void c_request_error(session_pro_backend_request& result, const std::exception& e) { - const std::string& error = e.what(); - result.error_count = snprintf_clamped( - result.error, - sizeof(result.error), - "%.*s", - static_cast(error.size()), - error.data()); + result.error_count = session::copy_c_str(result.error, sizeof(result.error), e.what()) - 1; +} + +LIBSESSION_C_API session_pro_backend_provider_urls +session_pro_backend_get_provider_urls(const char* provider_code) { + // Each present field is a static, null-terminated literal; an absent one is NULL. `found` + // distinguishes an unknown provider ({} -> found == false) from a recognised provider that + // simply has some/all URLs absent. + auto c = [](const std::optional& u) -> const char* { + return u ? u->data() : nullptr; + }; + if (auto u = provider_urls(provider_code)) + return {.found = true, + .refund_platform_url = c(u->refund_platform_url), + .refund_support_url = c(u->refund_support_url), + .refund_status_url = c(u->refund_status_url), + .update_subscription_url = c(u->update_subscription_url), + .cancel_subscription_url = c(u->cancel_subscription_url)}; + return {}; +} + +LIBSESSION_C_API const char* const* session_pro_backend_visible_platforms(size_t* count) { + // Derived from the C++ list (single source); each slug's data() is a static, null-terminated + // literal, so the pointer array is safe to hand out as static storage. + static const std::vector codes = [] { + std::vector v; + for (auto slug : visible_platforms()) + v.push_back(slug.data()); + return v; + }(); + if (count) + *count = codes.size(); + return codes.data(); } LIBSESSION_C_API session_pro_backend_request session_pro_backend_generate_pro_proof_request_build( - const uint8_t* master_privkey, + const unsigned char* master_privkey, size_t master_privkey_len, - const uint8_t* rotating_privkey, + const unsigned char* rotating_privkey, size_t rotating_privkey_len, int64_t ts) { session_pro_backend_request result = {}; try { result = c_own_request(pro_proof_request( - {master_privkey, master_privkey_len}, - {rotating_privkey, rotating_privkey_len}, + ed25519::PrivKeySpan{master_privkey, master_privkey_len}, + ed25519::PrivKeySpan{rotating_privkey, rotating_privkey_len}, session::as_sys_seconds(ts))); } catch (const std::exception& e) { c_request_error(result, e); @@ -755,11 +693,12 @@ session_pro_backend_get_pro_revocations_request_build(int64_t ticket) { } LIBSESSION_C_API session_pro_backend_request session_pro_backend_get_pro_status_request_build( - const uint8_t* master_privkey, size_t master_privkey_len, int64_t ts) { + const unsigned char* master_privkey, size_t master_privkey_len, int64_t ts) { session_pro_backend_request result = {}; try { result = c_own_request(pro_status_request( - {master_privkey, master_privkey_len}, session::as_sys_seconds(ts))); + ed25519::PrivKeySpan{master_privkey, master_privkey_len}, + session::as_sys_seconds(ts))); } catch (const std::exception& e) { c_request_error(result, e); } @@ -767,7 +706,7 @@ LIBSESSION_C_API session_pro_backend_request session_pro_backend_get_pro_status_ } LIBSESSION_C_API session_pro_backend_request session_pro_backend_get_payment_details_request_build( - const uint8_t* master_privkey, + const unsigned char* master_privkey, size_t master_privkey_len, int64_t ts, uint32_t limit, @@ -775,7 +714,7 @@ LIBSESSION_C_API session_pro_backend_request session_pro_backend_get_payment_det session_pro_backend_request result = {}; try { result = c_own_request(payment_details_request( - {master_privkey, master_privkey_len}, + ed25519::PrivKeySpan{master_privkey, master_privkey_len}, session::as_sys_seconds(ts), limit, before ? std::string_view{before} : std::string_view{})); diff --git a/src/pro_message.hpp b/src/pro_message.hpp index 09614ad2d..38acd38e9 100644 --- a/src/pro_message.hpp +++ b/src/pro_message.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -25,32 +26,37 @@ namespace session::pro { /// raw fields need no separator, and the domain prefix (also raw) never precedes one. The message /// is signed directly — there is no pre-hash (Ed25519 hashes internally). template -std::vector signed_message(std::string_view domain, const Fields&... fields) { - std::vector buf(domain.begin(), domain.end()); +std::vector signed_message(std::string_view domain, const Fields&... fields) { + std::vector buf; bool prev_var = false; // the previously-appended field was variable-length (int/string) + auto put_chars = [&](std::string_view s) { + for (char c : s) + buf.push_back(static_cast(static_cast(c))); + }; + put_chars(domain); // domain prefix: fixed-width, self-delimiting + auto append = [&](const auto& field) { using T = std::remove_cvref_t; if constexpr (std::is_integral_v) { if (prev_var) - buf.push_back('\0'); + buf.push_back(std::byte{0}); char tmp[24]; // enough for -9223372036854775808 (20 chars) auto [ptr, ec] = std::to_chars(tmp, tmp + sizeof(tmp), field); assert(ec == std::errc{}); // tmp is large enough for any integer, so this cannot fail - buf.insert(buf.end(), tmp, ptr); + put_chars({tmp, static_cast(ptr - tmp)}); prev_var = true; } else if constexpr (std::convertible_to) { if (prev_var) - buf.push_back('\0'); - std::string_view s = field; - buf.insert(buf.end(), s.begin(), s.end()); + buf.push_back(std::byte{0}); + put_chars(field); prev_var = true; } else { static_assert( - std::convertible_to>, + std::convertible_to>, "signed_message() fields must be an integer, a string_view, or a " "byte-spannable raw value (e.g. a public key or tag)"); - std::span b = field; + std::span b = field; buf.insert(buf.end(), b.begin(), b.end()); prev_var = false; } diff --git a/src/random.cpp b/src/random.cpp index 434b859ae..66e09c788 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -7,15 +7,27 @@ #include #include "session/export.h" +#include "session/random.h" #include "session/util.hpp" namespace session::random { -std::vector random(size_t size) { - std::vector result; - result.resize(size); - randombytes_buf(result.data(), size); +void fill(std::span buf) { + randombytes_buf(buf.data(), buf.size()); +} +void fill(std::span buf) { + fill(std::span{reinterpret_cast(buf.data()), buf.size()}); +} +void fill_deterministic(std::span buf, std::span seed) { + static_assert(seed.extent == randombytes_SEEDBYTES); + randombytes_buf_deterministic(to_unsigned(buf.data()), buf.size(), to_unsigned(seed.data())); +} + +std::vector random(size_t size) { + std::vector result; + result.resize(size); + fill(result); return result; } @@ -39,10 +51,14 @@ std::string random_base32(size_t size) { return result; } -std::string unique_id(std::string_view prefix) { - static std::atomic counter{0}; +static std::atomic unique_id_counter{0}; + +std::string unique_id(std::string_view prefix, size_t random_len) { return fmt::format( - "{}-{}-{}", prefix, counter.fetch_add(1, std::memory_order_relaxed), random_base32(4)); + "{}-{}-{}", + prefix, + unique_id_counter.fetch_add(1, std::memory_order_relaxed), + random_base32(random_len)); } } // namespace session::random @@ -50,9 +66,8 @@ std::string unique_id(std::string_view prefix) { extern "C" { LIBSESSION_C_API unsigned char* session_random(size_t size) { - auto result = session::random::random(size); auto* ret = static_cast(malloc(size)); - std::memcpy(ret, result.data(), result.size()); + session::random::fill(std::span{reinterpret_cast(ret), size}); return ret; } diff --git a/src/session_encrypt.cpp b/src/session_encrypt.cpp index a63d44ee5..09d2e0c2c 100644 --- a/src/session_encrypt.cpp +++ b/src/session_encrypt.cpp @@ -5,31 +5,30 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include #include +#include #include #include #include +#include "internal-util.hpp" #include "session/blinding.hpp" +#include "session/clock.hpp" +#include "session/crypto/ed25519.hpp" +#include "session/crypto/mlkem768.hpp" +#include "session/crypto/x25519.hpp" +#include "session/encrypt.hpp" +#include "session/hash.hpp" +#include "session/random.hpp" #include "session/sodium_array.hpp" #include "session/types.hpp" using namespace std::literals; +using namespace session::literals; namespace session { @@ -66,28 +65,80 @@ namespace detail { // some future version changes the format (and if not even try to load it). inline constexpr unsigned char BLINDED_ENCRYPT_VERSION = 0; -std::vector sign_for_recipient( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message) { - cleared_uc64 ed_sk_from_seed; - if (ed25519_privkey.size() == 32) { - uc32 ignore_pk; - crypto_sign_ed25519_seed_keypair( - ignore_pk.data(), ed_sk_from_seed.data(), ed25519_privkey.data()); - ed25519_privkey = {ed_sk_from_seed.data(), ed_sk_from_seed.size()}; - } else if (ed25519_privkey.size() != 64) { - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - } +// Constants for v2 PFS+PQ message encryption/decryption + +// BLAKE2b personalization for the key indicator shared secret (KISS): a 2-byte hash that lets the +// recipient cheaply identify which of their account keys was used without revealing it externally. +constexpr auto V2_KISS_PERS = "Session-Msg-KISS"_b2b_pers; + +// BLAKE2b personalization for the inner-message signature hash. +constexpr auto V2_MSG_SIG_PERS = "SessionV2Message"_b2b_pers; + +// X-Wing KDF domain separator from draft-connolly-cfrg-xwing-kem: the 6 ASCII bytes '\.//^\'. +// SHA3-256(ssₘ || ssₓ || E || X || V2_XWING_LABEL) produces the combined X-Wing shared secret. +constexpr auto V2_XWING_LABEL = // + R"(\./)" + R"(/^\)"_bytes; + +// SHAKE256 domain prefix for deriving the XChaCha20+Poly1305 key and nonce from the X-Wing SS. +constexpr auto V2_SS_DOMAIN = "SessionV2MessageSS"_bytes; + +// Shared v2 wire-format layout constants (used in both encrypt and decrypt) +static constexpr size_t V2_AEAD_OVERHEAD = encryption::XCHACHA20_ABYTES; +static constexpr size_t V2_NONCE_SIZE = encryption::XCHACHA20_NONCEBYTES; +static constexpr size_t V2_HEADER_SIZE = 2 + 2 + 32 + mlkem768::CIPHERTEXTBYTES; +static constexpr size_t V2_OUTER_OVERHEAD = V2_HEADER_SIZE + V2_AEAD_OVERHEAD; +static constexpr size_t V2_MIN_FINAL_SIZE = ((V2_OUTER_OVERHEAD + 256 + 255) / 256) * 256; + +// Validates the v2 ciphertext prefix and minimum size; throws std::runtime_error on failure. +static void v2_check_header(std::span ciphertext) { + if (ciphertext.size() < V2_MIN_FINAL_SIZE) + throw std::runtime_error{"v2 ciphertext is too short"}; + if (ciphertext[0] != std::byte{0x00} || ciphertext[1] != std::byte{0x02}) + throw std::runtime_error{"v2 ciphertext has wrong version prefix"}; +} + +// X-Wing KDF: computes ss = SHA3-256(ssm||ssx||E||X||V2_XWING_LABEL), then squeezes k (32B) into +// key_buf (overwriting ssm) and n (V2_NONCE_SIZE B) into nonce_out. Callers are responsible for +// storing these outputs in cleared buffers. E is the ephemeral X25519 pubkey; X is the PFS pubkey. +static void v2_derive_xwing_key_nonce( + std::span key_buf, + std::span nonce_out, + std::span ssx, + std::span E, + std::span X) { + cleared_b32 ss; + hash::sha3_256(ss, key_buf, ssx, E, X, V2_XWING_LABEL); + hash::shake256(V2_SS_DOMAIN, ss)(key_buf, nonce_out); +} + +// Computes the 2-byte Key Indicator Shared Secret (KISS): +// KISS = BLAKE2b_2(E || S, key=DH(sec, pub_for_dh), pers="Session-Msg-KISS") +// On encrypt: sender holds ephemeral secret e, DH partner is long-term S → call with +// encrypting=true On decrypt: recipient holds long-term secret s, DH partner is ephemeral E → call +// with encrypting=false +static std::array v2_kiss( + std::span sec, + std::span E, + std::span S, + bool encrypting) { + auto dh = x25519::scalarmult(sec, encrypting ? S : E); + return hash::blake2b_key_pers<2>(dh, V2_KISS_PERS, E, S); +} + +std::vector sign_for_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message) { // If prefixed, drop it (and do this for the caller, too) so that everything after this // doesn't need to worry about whether it is prefixed or not. - if (recipient_pubkey.size() == 33 && recipient_pubkey.front() == 0x05) + if (recipient_pubkey.size() == 33 && recipient_pubkey.front() == std::byte{0x05}) recipient_pubkey = recipient_pubkey.subspan(1); else if (recipient_pubkey.size() != 32) throw std::invalid_argument{ "Invalid recipient_pubkey: expected 32 bytes (33 with 05 prefix)"}; - std::vector buf; + std::vector buf; buf.reserve(message.size() + 96); // 32+32 now, but 32+64 when we reuse it for the sealed box buf.insert(buf.end(), message.begin(), message.end()); buf.insert( @@ -96,10 +147,7 @@ std::vector sign_for_recipient( ed25519_privkey.end()); // [32:] of a libsodium full seed value is the *pubkey* buf.insert(buf.end(), recipient_pubkey.begin(), recipient_pubkey.end()); - uc64 sig; - if (0 != crypto_sign_ed25519_detached( - sig.data(), nullptr, buf.data(), buf.size(), ed25519_privkey.data())) - throw std::runtime_error{"Failed to sign; perhaps the secret key is invalid?"}; + auto sig = ed25519::sign(ed25519_privkey, buf); // We have M||A||Y for the sig, but now we want M||A||SIG so drop Y then append SIG: buf.resize(buf.size() - 32); @@ -108,12 +156,12 @@ std::vector sign_for_recipient( return buf; } -static const std::span BOX_HASHKEY = to_span("SessionBoxEphemeralHashKey"); +static constexpr auto BOX_HASHKEY = "SessionBoxEphemeralHashKey"_bytes; -std::vector encrypt_for_recipient( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message) { +std::vector encrypt_for_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message) { auto signed_msg = sign_for_recipient(ed25519_privkey, recipient_pubkey, message); @@ -122,19 +170,17 @@ std::vector encrypt_for_recipient( recipient_pubkey.subspan(1); // sign_for_recipient already checked that this is the // proper 0x05 prefix when present. - std::vector result; - result.resize(signed_msg.size() + crypto_box_SEALBYTES); - if (0 != crypto_box_seal( - result.data(), signed_msg.data(), signed_msg.size(), recipient_pubkey.data())) - throw std::runtime_error{"Sealed box encryption failed"}; + std::vector result; + result.resize(signed_msg.size() + encryption::BOX_SEALBYTES); + encryption::box_seal(result, signed_msg, recipient_pubkey.first<32>()); return result; } -std::vector encrypt_for_recipient_deterministic( - std::span ed25519_privkey, - std::span recipient_pubkey, - std::span message) { +std::vector encrypt_for_recipient_deterministic( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span message) { auto signed_msg = sign_for_recipient(ed25519_privkey, recipient_pubkey, message); @@ -144,47 +190,335 @@ std::vector encrypt_for_recipient_deterministic( // To make our ephemeral seed we're going to hash: SENDER_SEED || RECIPIENT_PK || MESSAGE with a // keyed blake2b hash. - cleared_array seed; - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, BOX_HASHKEY.data(), BOX_HASHKEY.size(), seed.size()); - crypto_generichash_blake2b_update(&st, ed25519_privkey.data(), 32); - crypto_generichash_blake2b_update(&st, recipient_pubkey.data(), 32); - crypto_generichash_blake2b_update(&st, message.data(), message.size()); - crypto_generichash_blake2b_final(&st, seed.data(), seed.size()); - - cleared_array eph_sk; - cleared_array eph_pk; + cleared_b32 seed; + hash::blake2b_key( + seed, BOX_HASHKEY, ed25519_privkey.seed(), recipient_pubkey.first(32), message); - crypto_box_seed_keypair(eph_pk.data(), eph_sk.data(), seed.data()); + auto [eph_pk, eph_sk] = x25519::seed_keypair(seed); // The nonce for a sealed box is not passed but is implicitly defined as the (unkeyed) blake2b // hash of: // EPH_PUBKEY || RECIPIENT_PUBKEY - cleared_array nonce; - crypto_generichash_blake2b_init(&st, nullptr, 0, nonce.size()); - crypto_generichash_blake2b_update(&st, eph_pk.data(), eph_pk.size()); - crypto_generichash_blake2b_update(&st, recipient_pubkey.data(), recipient_pubkey.size()); - crypto_generichash_blake2b_final(&st, nonce.data(), nonce.size()); + std::array nonce; + hash::blake2b(nonce, eph_pk, recipient_pubkey); // A sealed box is a regular box (using the ephermal keys and nonce), but with the ephemeral // pubkey prepended: - static_assert(crypto_box_SEALBYTES == crypto_box_PUBLICKEYBYTES + crypto_box_MACBYTES); - - std::vector result; - result.resize(crypto_box_SEALBYTES + signed_msg.size()); - std::memcpy(result.data(), eph_pk.data(), crypto_box_PUBLICKEYBYTES); - if (0 != crypto_box_easy( - result.data() + crypto_box_PUBLICKEYBYTES, - signed_msg.data(), - signed_msg.size(), - nonce.data(), - recipient_pubkey.data(), - eph_sk.data())) - throw std::runtime_error{"Crypto box encryption failed"}; + static_assert( + encryption::BOX_SEALBYTES == encryption::BOX_PUBLICKEYBYTES + encryption::BOX_MACBYTES); + + std::vector result; + result.resize(encryption::BOX_SEALBYTES + signed_msg.size()); + std::ranges::copy(eph_pk, result.begin()); + encryption::box_easy( + std::span{result}.subspan(encryption::BOX_PUBLICKEYBYTES), + signed_msg, + nonce, + recipient_pubkey.first<32>(), + eph_sk); + + return result; +} + +// Builds and returns a complete v2 DM wire-format ciphertext from already-derived header fields +// and encryption key material. Used by both encrypt_for_recipient_v2 (PFS+PQ) and +// encrypt_for_recipient_v2_nopfs (non-PFS fallback). +static std::vector v2_encrypt_inner( + std::array ki, + std::span E, + std::span outer_ct, + std::span enc_key, + std::span enc_nonce, + const ed25519::PrivKeySpan& sender_ed25519_privkey, + std::span recipient_session_id, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_ed25519_privkey) { + + auto sender_ed_pk = sender_ed25519_privkey.pubkey(); + + // bt_bytes_encoded(n): total bytes to represent an n-byte bt string: decimal digits of n + 1 + + // n + constexpr auto bt_bytes_encoded = [](size_t n) constexpr -> size_t { + size_t sz = 1 + n; // ':' + n data bytes + do { + ++sz; + } while (n /= 10); // decimal digits of n + return sz; + }; + // Keys must be in ascending lexicographic order: "S" < "c" < "~" < "~P" + constexpr size_t S_KEY_VAL = 3 + bt_bytes_encoded(32); // "1:S" + "32:" + constexpr size_t SIG_KEY_VAL = 3 + bt_bytes_encoded(64); // "1:~" + "64:" + constexpr size_t PRO_KEY_VAL = 4 + bt_bytes_encoded(64); // "2:~P" + "64:" + size_t inner_dict_size = 2 // d...e dict delimiters + + S_KEY_VAL + 3 + + bt_bytes_encoded(content.size()) // "1:c" + "" + + SIG_KEY_VAL + (pro_ed25519_privkey ? PRO_KEY_VAL : 0); + + // Total message must be a multiple of 256 bytes and at least V2_MIN_FINAL_SIZE bytes. + size_t final_size = + (std::max(V2_MIN_FINAL_SIZE, V2_OUTER_OVERHEAD + inner_dict_size) + 255) & ~size_t{255}; + size_t padded_inner_size = final_size - V2_OUTER_OVERHEAD; + + // Allocate result (zero-initialized so padding bytes are already 0), write header, + // build inner dict directly into result buffer, then encrypt in-place. + // (c == m is explicitly supported by libsodium for AEAD functions) + std::vector result(final_size, std::byte{0}); + + result[0] = std::byte{0x00}; + result[1] = std::byte{0x02}; + result[2] = ki[0]; + result[3] = ki[1]; + std::memcpy(result.data() + 4, E.data(), 32); + std::memcpy(result.data() + 36, outer_ct.data(), mlkem768::CIPHERTEXTBYTES); + + { + oxenc::bt_dict_producer dict{ + reinterpret_cast(result.data() + V2_HEADER_SIZE), inner_dict_size}; + dict.append("S", sender_ed_pk); + dict.append("c", content); + // "~" signs BLAKE2b-64(body-so-far, key=recipient_session_id_33B, pers="SessionV2Message") + dict.append_signature("~", [&](std::span body) { + cleared_b64 h; + hash::blake2b_key_pers(h, recipient_session_id, V2_MSG_SIG_PERS, body); + return ed25519::sign(sender_ed25519_privkey, h); + }); + if (pro_ed25519_privkey) + dict.append_signature("~P", [&](std::span body) { + return ed25519::sign(*pro_ed25519_privkey, body); + }); + assert(dict.view().size() == inner_dict_size); + } + + // In-place AEAD encrypt (libsodium explicitly supports c == m) + encryption::xchacha20poly1305_encrypt( + std::span{result}.subspan(V2_HEADER_SIZE), + std::span{result}.subspan(V2_HEADER_SIZE, padded_inner_size), + enc_nonce, + enc_key); + + return result; +} + +std::vector encrypt_for_recipient_v2( + const ed25519::PrivKeySpan& sender_ed25519_privkey, + std::span recipient_session_id, + std::span recipient_account_x25519, + std::span recipient_account_mlkem768, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_ed25519_privkey) { + + // S = long-term X25519 pubkey of the recipient (session ID without the 0x05 prefix) + std::span S{recipient_session_id.data() + 1, 32}; + + // Step 1: Generate ephemeral X25519 keypair e/E + auto [E, e] = x25519::keypair(); + + // Three cleared buffers for key material: + // enc_key_buf: ML-KEM shared secret ssm (step 4) → SHAKE256-derived enc key k (step 6) + // ssx_buf: eS DH result (step 2) → ML-KEM coins (step 4) → ssx DH result (step 5) + // enc_nonce: SHAKE256-derived enc nonce n (step 6) + cleared_b32 enc_key_buf; + cleared_b32 ssx_buf; + + // Step 2: KISS = BLAKE2b_2(E || S, key=eS, pers="Session-Msg-KISS") + // eS is the X25519 DH with the long-term key, used only for cheap key indicator obfuscation + auto kiss = v2_kiss(e, E, S, /*encrypting=*/true); + + // Step 3: ki = M[0:2] ⊕ kiss (encrypted key indicator; lets recipient quickly identify key) + std::array ki{ + recipient_account_mlkem768[0] ^ kiss[0], recipient_account_mlkem768[1] ^ kiss[1]}; + + // Step 4: ML-KEM-768 encapsulate: ssₘ, mlkem_ct = Encapsulate(M) + std::array mlkem_ct; + random::fill(ssx_buf); // repurpose ssx_buf as random ML-KEM coins + mlkem768::encapsulate(mlkem_ct, enc_key_buf, recipient_account_mlkem768, ssx_buf); + + // Step 5: ssx = eX (X25519 DH with account PFS key X, not long-term key S) + x25519::scalarmult(ssx_buf, e, recipient_account_x25519); + + // Step 6: X-Wing KDF → enc key k (in enc_key_buf) and enc nonce n (in enc_nonce) + std::array enc_nonce; + v2_derive_xwing_key_nonce(enc_key_buf, enc_nonce, ssx_buf, E, recipient_account_x25519); + + return v2_encrypt_inner( + ki, + E, + mlkem_ct, + enc_key_buf, + enc_nonce, + sender_ed25519_privkey, + recipient_session_id, + content, + pro_ed25519_privkey); +} +std::array decrypt_incoming_v2_prefix( + std::span x25519_sec, + std::span x25519_pub, + std::span ciphertext) { + v2_check_header(ciphertext); + auto E = ciphertext.subspan<4, 32>(); + auto kiss = v2_kiss(x25519_sec, E, x25519_pub, /*encrypting=*/false); + return {ciphertext[2] ^ kiss[0], ciphertext[3] ^ kiss[1]}; +} + +// Decrypts the v2 AEAD payload and parses the inner bt-encoded dict. Used by both +// decrypt_incoming_v2 (PFS+PQ) and decrypt_incoming_v2_nopfs (non-PFS fallback). +// Throws DecryptV2Error on AEAD failure; std::runtime_error on structural/format errors. +static DecryptV2Result v2_aead_decrypt_and_parse( + std::span recipient_session_id, + std::span key, + std::span nonce, + std::span ciphertext) { + + size_t enc_size = ciphertext.size() - V2_HEADER_SIZE; + std::vector plain(enc_size - V2_AEAD_OVERHEAD); + if (!encryption::xchacha20poly1305_decrypt( + plain, ciphertext.subspan(V2_HEADER_SIZE, enc_size), nonce, key)) + throw DecryptV2Error{"v2 message decryption failed"}; + + // Strip zero padding from end (the plaintext was padded to a multiple of 256 bytes) + while (!plain.empty() && plain.back() == std::byte{0}) + plain.pop_back(); + + // Parse the bencoded inner dict + oxenc::bt_dict_consumer dict{plain}; + + auto sender_ed_pk = dict.require_span("S"); + auto content_sv = dict.require_span("c"); + + // Verify the Ed25519 signature over BLAKE2b(body, key=recipient_session_id, pers=…) + dict.require_signature( + "~", [&](std::span body, std::span sig) { + if (sig.size() != 64) + throw std::runtime_error{"v2 message signature has wrong size"}; + b64 h; + hash::blake2b_key_pers(h, recipient_session_id, V2_MSG_SIG_PERS, body); + if (!ed25519::verify(sig.first<64>(), sender_ed_pk, h)) + throw std::runtime_error{"v2 message signature verification failed"}; + }); + + // Optional "~P" pro signature. Extracted but not verified here — the Pro public key is + // inside the protobuf Content, so verification is deferred to the message parsing layer. + std::optional pro_sig; + if (dict.skip_until("~P")) + dict.consume_signature([&](std::span, std::span sig) { + if (sig.size() != 64) + throw std::runtime_error{"v2 ~P pro signature has wrong size"}; + std::memcpy(pro_sig.emplace().data(), sig.data(), 64); + }); + + dict.finish(); + + // Convert sender Ed25519 pubkey to X25519 and build the 33-byte session ID + b32 sender_x25519 = ed25519::pk_to_x25519(sender_ed_pk); + + DecryptV2Result result; + result.content.assign(content_sv.begin(), content_sv.end()); + result.sender_session_id[0] = std::byte{0x05}; + std::ranges::copy(sender_x25519, result.sender_session_id.begin() + 1); + if (pro_sig) + std::memcpy(result.pro_signature.emplace().data(), pro_sig->data(), 64); return result; } +DecryptV2Result decrypt_incoming_v2( + std::span recipient_session_id, + std::span account_pfs_x25519_sec, + std::span account_pfs_x25519_pub, + std::span account_pfs_mlkem768_sec, + std::span ciphertext) { + v2_check_header(ciphertext); + + auto E = ciphertext.subspan<4, 32>(); + auto mlkem_ct = ciphertext.subspan<36, mlkem768::CIPHERTEXTBYTES>(); + + cleared_b32 key_buf; // ssm → k + cleared_b32 ssx_buf; + std::array nonce; + + // Step 1: ML-KEM-768 decapsulate → shared secret ssm in key_buf + if (!mlkem768::decapsulate(key_buf, mlkem_ct, account_pfs_mlkem768_sec)) + throw DecryptV2Error{"ML-KEM-768 decapsulation failed"}; + + // Step 2: X25519 DH with account PFS key → shared secret ssx in ssx_buf + x25519::scalarmult(ssx_buf, account_pfs_x25519_sec, E); + + // Step 3: X-Wing KDF → enc key k (in key_buf) and enc nonce n (in nonce) + v2_derive_xwing_key_nonce(key_buf, nonce, ssx_buf, E, account_pfs_x25519_pub); + + return v2_aead_decrypt_and_parse(recipient_session_id, key_buf, nonce, ciphertext); +} + +// Non-PFS fallback key derivation domain labels (private to this translation unit). +// The outer wire format is identical to a PFS+PQ v2 message; only the key derivation differs. +constexpr auto V2_NONPFS_KDF_LABEL = "SessionV2NonPFS"_bytes; +constexpr auto V2_NONPFS_SS_DOMAIN = "SessionV2NonPFSSS"_bytes; + +std::vector encrypt_for_recipient_v2_nopfs( + const ed25519::PrivKeySpan& sender_ed25519_privkey, + std::span recipient_session_id, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_ed25519_privkey) { + + // R = long-term X25519 pubkey of the recipient (session ID without the 0x05 prefix) + auto R = recipient_session_id.last<32>(); + + // Generate ephemeral X25519 keypair e/E + auto [E, e] = x25519::keypair(); + + // ki and the outer "mlkem_ct" slot are random: the message is externally indistinguishable + // from a PFS+PQ v2 message, but carries no actual ML-KEM ciphertext. + std::array ki; + std::array outer_ct; + random::fill(ki); + random::fill(outer_ct); + + // ss = eR, then overwritten in-place with SHA3-256(ss || R || E || V2_NONPFS_KDF_LABEL). + cleared_b32 ss; + x25519::scalarmult(ss, e, R); + hash::sha3_256(ss, ss, R, E, V2_NONPFS_KDF_LABEL); + + // k, n = SHAKE256(V2_NONPFS_SS_DOMAIN, ss) → 32-byte key + 24-byte nonce + cleared_b32 enc_key; + std::array enc_nonce; + hash::shake256(V2_NONPFS_SS_DOMAIN, ss)(enc_key, enc_nonce); + + return v2_encrypt_inner( + ki, + E, + outer_ct, + enc_key, + enc_nonce, + sender_ed25519_privkey, + recipient_session_id, + content, + pro_ed25519_privkey); +} + +DecryptV2Result decrypt_incoming_v2_nopfs( + std::span recipient_session_id, + std::span x25519_sec, + std::span x25519_pub, + std::span ciphertext) { + v2_check_header(ciphertext); + + // E = ephemeral X25519 pubkey from bytes 4-35; bytes 36-1123 (fake mlkem_ct) are ignored. + auto E = ciphertext.subspan<4, 32>(); + + // ss = rE, then overwritten in-place with SHA3-256(ss || R || E || V2_NONPFS_KDF_LABEL). + cleared_b32 ss; + x25519::scalarmult(ss, x25519_sec, E); + hash::sha3_256(ss, ss, x25519_pub, E, V2_NONPFS_KDF_LABEL); + + // k, n = SHAKE256(V2_NONPFS_SS_DOMAIN, ss) → 32-byte key + 24-byte nonce + cleared_b32 key; + std::array nonce; + hash::shake256(V2_NONPFS_SS_DOMAIN, ss)(key, nonce); + + return v2_aead_decrypt_and_parse(recipient_session_id, key, nonce, ciphertext); +} + // Calculate the shared encryption key, sending from blinded sender kS (k = S's blinding factor) to // blinded receiver jR (j = R's blinding factor). // @@ -213,11 +547,11 @@ std::vector encrypt_for_recipient_deterministic( // jB -- A's 33-byte blinded id, beginning with 0x15 or 0x25 (must be the same prefix as kA). // server_pk -- the server's pubkey (needed to compute A's `k` value) // sending -- true if this for a message from A to B, false if this is from B to A. -static cleared_uc32 blinded_shared_secret( - std::span seed, - std::span kA, - std::span jB, - std::span server_pk, +static cleared_b32 blinded_shared_secret( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span kA_prefixed, + std::span jB_prefixed, + std::span server_pk, bool sending) { // Because we're doing this generically, we use notation a/A/k for ourselves and b/jB for the @@ -225,175 +559,111 @@ static cleared_uc32 blinded_shared_secret( // the BLAKE2b hashed value: there we have to use kA || jB if we are the sender, but reverse the // order to jB || kA if we are the receiver. - std::pair blinded_key_pair; - cleared_uc32 k; - - if (seed.size() != 64 && seed.size() != 32) - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - if (server_pk.size() != 32) - throw std::invalid_argument{"Invalid server_pk: expected 32 bytes"}; - if (kA.size() != 33) - throw std::invalid_argument{"Invalid local blinded id: expected 33 bytes"}; - if (jB.size() != 33) - throw std::invalid_argument{"Invalid remote blinded id: expected 33 bytes"}; - if (kA[0] == 0x15 && jB[0] == 0x15) - blinded_key_pair = blind15_key_pair(seed, server_pk, &k); - else if (kA[0] == 0x25 && jB[0] == 0x25) - blinded_key_pair = blind25_key_pair(seed, server_pk, &k); + std::pair blinded_key_pair; + cleared_b32 k; + + if (kA_prefixed[0] == std::byte{0x15} && jB_prefixed[0] == std::byte{0x15}) + blinded_key_pair = blind15_key_pair(ed25519_privkey, server_pk, &k); + else if (kA_prefixed[0] == std::byte{0x25} && jB_prefixed[0] == std::byte{0x25}) + blinded_key_pair = blind25_key_pair(ed25519_privkey, server_pk, &k); else throw std::invalid_argument{"Both ids must start with the same 0x15 or 0x25 prefix"}; - bool blind25 = kA[0] == 0x25; + bool blind25 = kA_prefixed[0] == std::byte{0x25}; - kA = kA.subspan(1); - jB = jB.subspan(1); + auto kA = kA_prefixed.subspan<1>(); + auto jB = jB_prefixed.subspan<1>(); - cleared_uc32 ka; - // Not really switching to x25519 here, this is just an easy way to compute `a` - crypto_sign_ed25519_sk_to_curve25519(ka.data(), seed.data()); + cleared_b32 ka = ed25519::sk_to_private(ed25519_privkey); if (blind25) // Multiply a by k, so that we end up computing kajB = kjaB, which the other side can // compute as jkbA. - crypto_core_ed25519_scalar_mul(ka.data(), ka.data(), k.data()); + ed25519::scalar_mul(ka, ka, k); // Else for 15 blinding we leave "ka" as just a, because j=k and so we don't need the // double-blind. - cleared_uc32 shared_secret; - if (0 != crypto_scalarmult_ed25519_noclamp(shared_secret.data(), ka.data(), jB.data())) - throw std::runtime_error{"Shared secret generation failed"}; + cleared_b32 shared_secret; + ed25519::scalarmult_noclamp(shared_secret, ka, jB); auto& sender = sending ? kA : jB; auto& recipient = sending ? jB : kA; // H(kjsR || kS || jR): - crypto_generichash_blake2b_state st; - crypto_generichash_blake2b_init(&st, nullptr, 0, 32); - crypto_generichash_blake2b_update(&st, shared_secret.data(), shared_secret.size()); - crypto_generichash_blake2b_update(&st, sender.data(), sender.size()); - crypto_generichash_blake2b_update(&st, recipient.data(), recipient.size()); - crypto_generichash_blake2b_final(&st, shared_secret.data(), shared_secret.size()); + hash::blake2b(shared_secret, shared_secret, sender, recipient); return shared_secret; } -std::vector encrypt_for_blinded_recipient( - std::span ed25519_privkey, - std::span server_pk, - std::span recipient_blinded_id, - std::span message) { - if (ed25519_privkey.size() != 64 && ed25519_privkey.size() != 32) - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - if (server_pk.size() != 32) - throw std::invalid_argument{"Invalid server_pk: expected 32 bytes"}; - if (recipient_blinded_id.size() != 33) - throw std::invalid_argument{"Invalid recipient_blinded_id: expected 33 bytes"}; +std::vector encrypt_for_blinded_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span server_pk, + std::span recipient_blinded_id, + std::span message) { // Generate the blinded key pair & shared encryption key - std::pair blinded_key_pair; - switch (recipient_blinded_id[0]) { - case 0x15: blinded_key_pair = blind15_key_pair(ed25519_privkey, server_pk); break; - - case 0x25: blinded_key_pair = blind25_key_pair(ed25519_privkey, server_pk); break; + std::pair blinded_key_pair; + if (recipient_blinded_id[0] == std::byte{0x15}) + blinded_key_pair = blind15_key_pair(ed25519_privkey, server_pk); + else if (recipient_blinded_id[0] == std::byte{0x25}) + blinded_key_pair = blind25_key_pair(ed25519_privkey, server_pk); + else + throw std::invalid_argument{"Invalid recipient_blinded_id: must start with 0x15 or 0x25"}; - default: - throw std::invalid_argument{ - "Invalid recipient_blinded_id: must start with 0x15 or 0x25"}; - } - std::vector blinded_id; - blinded_id.reserve(33); - blinded_id.insert( - blinded_id.end(), recipient_blinded_id.begin(), recipient_blinded_id.begin() + 1); - blinded_id.insert( - blinded_id.end(), blinded_key_pair.first.begin(), blinded_key_pair.first.end()); + std::array blinded_id; + blinded_id[0] = recipient_blinded_id[0]; + std::ranges::copy(blinded_key_pair.first, blinded_id.begin() + 1); auto enc_key = blinded_shared_secret( ed25519_privkey, blinded_id, recipient_blinded_id, server_pk, true); // Inner data: msg || A (i.e. the sender's ed25519 master pubkey, *not* kA blinded pubkey) - std::vector buf; + std::vector buf; buf.reserve(message.size() + 32); buf.insert(buf.end(), message.begin(), message.end()); // append A (pubkey) - if (ed25519_privkey.size() == 64) { - buf.insert(buf.end(), ed25519_privkey.begin() + 32, ed25519_privkey.end()); - } else { - cleared_uc64 ed_sk_from_seed; - uc32 ed_pk_buf; - crypto_sign_ed25519_seed_keypair( - ed_pk_buf.data(), ed_sk_from_seed.data(), ed25519_privkey.data()); - buf.insert(buf.end(), ed_pk_buf.begin(), ed_pk_buf.end()); - } - - // Encrypt using xchacha20-poly1305 - cleared_array nonce; - randombytes_buf(nonce.data(), nonce.size()); + auto pk = ed25519_privkey.pubkey(); + buf.insert(buf.end(), pk.begin(), pk.end()); - std::vector ciphertext; - unsigned long long outlen = 0; + // Layout: version(1) || ciphertext(buf+ABYTES) || nonce(NPUBBYTES) + std::vector ciphertext; ciphertext.resize( - 1 + buf.size() + crypto_aead_xchacha20poly1305_ietf_ABYTES + - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); + 1 + buf.size() + encryption::XCHACHA20_ABYTES + encryption::XCHACHA20_NONCEBYTES); // Prepend with a version byte, so that the recipient can reliably detect if a future version is // no longer encrypting things the way it expects. - ciphertext[0] = BLINDED_ENCRYPT_VERSION; + ciphertext[0] = std::byte{BLINDED_ENCRYPT_VERSION}; - if (0 != crypto_aead_xchacha20poly1305_ietf_encrypt( - ciphertext.data() + 1, - &outlen, - buf.data(), - buf.size(), - nullptr, - 0, - nullptr, - nonce.data(), - enc_key.data())) - throw std::runtime_error{"Crypto aead encryption failed"}; + auto nonce = std::span{ciphertext}.last(); + random::fill(nonce); - assert(outlen == ciphertext.size() - 1 - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - - // append the nonce, so that we have: data = b'\x00' + ciphertext + nonce - std::memcpy(ciphertext.data() + (1 + outlen), nonce.data(), nonce.size()); + encryption::xchacha20poly1305_encrypt( + std::span{ciphertext}.subspan(1, buf.size() + encryption::XCHACHA20_ABYTES), + buf, + nonce, + enc_key); return ciphertext; } static constexpr size_t GROUPS_ENCRYPT_OVERHEAD = - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + crypto_aead_xchacha20poly1305_ietf_ABYTES; + encryption::XCHACHA20_NONCEBYTES + encryption::XCHACHA20_ABYTES; -std::vector encrypt_for_group( - std::span user_ed25519_privkey, - std::span group_ed25519_pubkey, - std::span group_enc_key, - std::span plaintext, +std::vector encrypt_for_group( + const ed25519::PrivKeySpan& user_ed25519_privkey, + std::span group_ed25519_pubkey, + std::span group_enc_key, + std::span plaintext, bool compress, size_t padding) { if (plaintext.size() > GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE) throw std::runtime_error{"Cannot encrypt plaintext: message size is too large"}; - // Generate the user's pubkey if they passed in a 32 byte secret key instead of the - // libsodium-style 64 byte secret key. - cleared_uc64 user_ed25519_privkey_from_seed; - if (user_ed25519_privkey.size() == 32) { - uc32 ignore_pk; - crypto_sign_ed25519_seed_keypair( - ignore_pk.data(), - user_ed25519_privkey_from_seed.data(), - user_ed25519_privkey.data()); - user_ed25519_privkey = { - user_ed25519_privkey_from_seed.data(), user_ed25519_privkey_from_seed.size()}; - } else if (user_ed25519_privkey.size() != 64) { - throw std::invalid_argument{"Invalid user_ed25519_privkey: expected 32 or 64 bytes"}; - } - if (group_enc_key.size() != 32 && group_enc_key.size() != 64) throw std::invalid_argument{"Invalid group_enc_key: expected 32 or 64 bytes"}; - if (group_ed25519_pubkey.size() != crypto_sign_ed25519_PUBLICKEYBYTES) - throw std::invalid_argument{"Invalid group_ed25519_pubkey: expected 32 bytes"}; - std::vector _compressed; + std::vector _compressed; if (compress) { _compressed = zstd_compress(plaintext); if (_compressed.size() < plaintext.size()) @@ -415,9 +685,8 @@ std::vector encrypt_for_group( // components to this validation: first the regular signature validation of the "s" signature we // add below, but then also validation that this Ed25519 converts to the Session ID of the // claimed sender of the message inside the encoded message data. - dict.append( - "a", - std::string_view{reinterpret_cast(user_ed25519_privkey.data()) + 32, 32}); + auto sender_pk = user_ed25519_privkey.pubkey(); + dict.append("a", to_string_view(sender_pk)); if (!compress) dict.append("d", to_string_view(plaintext)); @@ -426,16 +695,14 @@ std::vector encrypt_for_group( // encrypted data will not validate if cross-posted to any other group. We don't actually // include the pubkey alongside, because that is implicitly known by the group members that // receive it. - std::vector to_sign(plaintext.size() + group_ed25519_pubkey.size()); + std::vector to_sign(plaintext.size() + group_ed25519_pubkey.size()); std::memcpy(to_sign.data(), plaintext.data(), plaintext.size()); std::memcpy( to_sign.data() + plaintext.size(), group_ed25519_pubkey.data(), group_ed25519_pubkey.size()); - std::array signature; - crypto_sign_ed25519_detached( - signature.data(), nullptr, to_sign.data(), to_sign.size(), user_ed25519_privkey.data()); + auto signature = ed25519::sign(user_ed25519_privkey, to_sign); dict.append("s", to_string_view(signature)); if (compress) @@ -453,119 +720,82 @@ std::vector encrypt_for_group( encoded.resize(encoded.size() + to_append); } - std::vector ciphertext; + std::vector ciphertext; ciphertext.resize(GROUPS_ENCRYPT_OVERHEAD + encoded.size()); - randombytes_buf(ciphertext.data(), crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - std::span nonce{ - ciphertext.data(), crypto_aead_xchacha20poly1305_ietf_NPUBBYTES}; - if (0 != crypto_aead_xchacha20poly1305_ietf_encrypt( - ciphertext.data() + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, - nullptr, - to_unsigned(encoded.data()), - encoded.size(), - nullptr, - 0, - nullptr, - nonce.data(), - group_enc_key.data())) - throw std::runtime_error{"Encryption failed"}; + auto nonce = std::span{ciphertext}.first(); + random::fill(nonce); + + encryption::xchacha20poly1305_encrypt( + std::span{ciphertext}.subspan(encryption::XCHACHA20_NONCEBYTES), + to_span(encoded), + nonce, + group_enc_key.first()); return ciphertext; } -std::pair, std::string> decrypt_incoming_session_id( - std::span ed25519_privkey, std::span ciphertext) { +std::pair, std::string> decrypt_incoming_session_id( + const ed25519::PrivKeySpan& ed25519_privkey, std::span ciphertext) { auto [buf, sender_ed_pk] = decrypt_incoming(ed25519_privkey, ciphertext); // Convert the sender_ed_pk to the sender's session ID - std::array sender_x_pk; - - if (0 != crypto_sign_ed25519_pk_to_curve25519(sender_x_pk.data(), sender_ed_pk.data())) - throw std::runtime_error{"Sender ed25519 pubkey to x25519 pubkey conversion failed"}; + auto sender_x_pk = ed25519::pk_to_x25519(sender_ed_pk); // Everything is good, so just drop A and Y off the message and prepend the '05' prefix to // the sender session ID - std::string sender_session_id; - sender_session_id.reserve(66); - sender_session_id += "05"; - oxenc::to_hex(sender_x_pk.begin(), sender_x_pk.end(), std::back_inserter(sender_session_id)); + auto sender_session_id = "05{:x}"_format(sender_x_pk); return {buf, sender_session_id}; } -std::pair, std::string> decrypt_incoming_session_id( - std::span x25519_pubkey, - std::span x25519_seckey, - std::span ciphertext) { +std::pair, std::string> decrypt_incoming_session_id( + std::span x25519_pubkey, + std::span x25519_seckey, + std::span ciphertext) { auto [buf, sender_ed_pk] = decrypt_incoming(x25519_pubkey, x25519_seckey, ciphertext); // Convert the sender_ed_pk to the sender's session ID - std::array sender_x_pk; - - if (0 != crypto_sign_ed25519_pk_to_curve25519(sender_x_pk.data(), sender_ed_pk.data())) - throw std::runtime_error{"Sender ed25519 pubkey to x25519 pubkey conversion failed"}; + auto sender_x_pk = ed25519::pk_to_x25519(sender_ed_pk); // Everything is good, so just drop A and Y off the message and prepend the '05' prefix to // the sender session ID - std::string sender_session_id; - sender_session_id.reserve(66); - sender_session_id += "05"; - oxenc::to_hex(sender_x_pk.begin(), sender_x_pk.end(), std::back_inserter(sender_session_id)); + auto sender_session_id = "05{:x}"_format(sender_x_pk); return {buf, sender_session_id}; } -std::pair, std::vector> decrypt_incoming( - std::span ed25519_privkey, std::span ciphertext) { - cleared_uc64 ed_sk_from_seed; - if (ed25519_privkey.size() == 32) { - uc32 ignore_pk; - crypto_sign_ed25519_seed_keypair( - ignore_pk.data(), ed_sk_from_seed.data(), ed25519_privkey.data()); - ed25519_privkey = {ed_sk_from_seed.data(), ed_sk_from_seed.size()}; - } else if (ed25519_privkey.size() != 64) { - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - } - - cleared_uc32 x_sec; - uc32 x_pub; - crypto_sign_ed25519_sk_to_curve25519(x_sec.data(), ed25519_privkey.data()); - crypto_scalarmult_base(x_pub.data(), x_sec.data()); - +std::pair, b32> decrypt_incoming( + const ed25519::PrivKeySpan& ed25519_privkey, std::span ciphertext) { + auto x_sec = ed25519::sk_to_x25519(ed25519_privkey); + auto x_pub = x25519::scalarmult_base(x_sec); return decrypt_incoming(x_pub, x_sec, ciphertext); } -std::pair, std::vector> decrypt_incoming( - std::span x25519_pubkey, - std::span x25519_seckey, - std::span ciphertext) { +std::pair, b32> decrypt_incoming( + std::span x25519_pubkey, + std::span x25519_seckey, + std::span ciphertext) { - if (ciphertext.size() < crypto_box_SEALBYTES + 32 + 64) + if (ciphertext.size() < encryption::BOX_SEALBYTES + 32 + 64) throw std::runtime_error{"Invalid incoming message: ciphertext is too small"}; - const size_t outer_size = ciphertext.size() - crypto_box_SEALBYTES; + const size_t outer_size = ciphertext.size() - encryption::BOX_SEALBYTES; const size_t msg_size = outer_size - 32 - 64; - std::pair, std::vector> result; + std::pair, b32> result; auto& [buf, sender_ed_pk] = result; buf.resize(outer_size); - int opened = crypto_box_seal_open( - buf.data(), - ciphertext.data(), - ciphertext.size(), - x25519_pubkey.data(), - x25519_seckey.data()); - if (opened != 0) + if (!encryption::box_seal_open(buf, ciphertext, x25519_pubkey, x25519_seckey)) throw std::runtime_error{"Decryption failed"}; - uc64 sig; - sender_ed_pk.assign(buf.begin() + msg_size, buf.begin() + msg_size + 32); - std::memcpy(sig.data(), buf.data() + msg_size + 32, 64); + auto tail = std::span{buf}.subspan(msg_size); // A(32) || SIG(64) + std::ranges::copy(tail.first<32>(), sender_ed_pk.begin()); + b64 sig; + std::ranges::copy(tail.last<64>(), sig.begin()); buf.resize(buf.size() - 64); // Remove SIG, then append Y so that we get M||A||Y to verify - buf.insert(buf.end(), x25519_pubkey.begin(), x25519_pubkey.begin() + 32); + buf.insert(buf.end(), x25519_pubkey.begin(), x25519_pubkey.end()); - if (0 != crypto_sign_ed25519_verify_detached( - sig.data(), buf.data(), buf.size(), sender_ed_pk.data())) + if (!ed25519::verify(sig, sender_ed_pk, buf)) throw std::runtime_error{"Signature verification failed"}; // Everything is good, so just drop A and Y off the message @@ -574,97 +804,66 @@ std::pair, std::vector> decrypt_incomi return result; } -std::pair, std::string> decrypt_from_blinded_recipient( - std::span ed25519_privkey, - std::span server_pk, - std::span sender_id, - std::span recipient_id, - std::span ciphertext) { - uc32 ed_pk_from_seed; - cleared_uc64 ed_sk_from_seed; - if (ed25519_privkey.size() == 32) { - crypto_sign_ed25519_seed_keypair( - ed_pk_from_seed.data(), ed_sk_from_seed.data(), ed25519_privkey.data()); - ed25519_privkey = {ed_sk_from_seed.data(), ed_sk_from_seed.size()}; - } else if (ed25519_privkey.size() == 64) - std::memcpy(ed_pk_from_seed.data(), ed25519_privkey.data() + 32, 32); - else - throw std::invalid_argument{"Invalid ed25519_privkey: expected 32 or 64 bytes"}; - if (ciphertext.size() < crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + 1 + - crypto_aead_xchacha20poly1305_ietf_ABYTES) +std::pair, std::string> decrypt_from_blinded_recipient( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span server_pk, + std::span sender_id, + std::span recipient_id, + std::span ciphertext) { + auto ed_pk = ed25519_privkey.pubkey(); + if (ciphertext.size() < encryption::XCHACHA20_NONCEBYTES + 1 + encryption::XCHACHA20_ABYTES) throw std::invalid_argument{ "Invalid ciphertext: too short to contain valid encrypted data"}; - cleared_uc32 dec_key; - auto blinded_id = recipient_id[0] == 0x25 - ? blinded25_id_from_ed(to_span(ed_pk_from_seed), server_pk) - : blinded15_id_from_ed(to_span(ed_pk_from_seed), server_pk); + cleared_b32 dec_key; + auto blinded_id = recipient_id[0] == std::byte{0x25} ? blinded25_id_from_ed(ed_pk, server_pk) + : blinded15_id_from_ed(ed_pk, server_pk); if (to_string_view(sender_id) == to_string_view(blinded_id)) dec_key = blinded_shared_secret(ed25519_privkey, sender_id, recipient_id, server_pk, true); else dec_key = blinded_shared_secret(ed25519_privkey, recipient_id, sender_id, server_pk, false); - std::pair, std::string> result; + std::pair, std::string> result; auto& [buf, sender_session_id] = result; // v, ct, nc = data[0], data[1:-24], data[-24:] - if (ciphertext[0] != BLINDED_ENCRYPT_VERSION) + if (ciphertext[0] != std::byte{BLINDED_ENCRYPT_VERSION}) throw std::invalid_argument{ - "Invalid ciphertext: version is not " + std::to_string(BLINDED_ENCRYPT_VERSION)}; + fmt::format("Invalid ciphertext: version is not {}", BLINDED_ENCRYPT_VERSION)}; - std::vector nonce; const size_t msg_size = - (ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES - 1 - - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); + (ciphertext.size() - encryption::XCHACHA20_ABYTES - 1 - + encryption::XCHACHA20_NONCEBYTES); if (msg_size < 32) throw std::invalid_argument{"Invalid ciphertext: innerBytes too short"}; buf.resize(msg_size); - unsigned long long buf_len = 0; - - nonce.resize(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - std::memcpy( - nonce.data(), - ciphertext.data() + msg_size + 1 + crypto_aead_xchacha20poly1305_ietf_ABYTES, - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - - if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt( - buf.data(), - &buf_len, - nullptr, - ciphertext.data() + 1, - msg_size + crypto_aead_xchacha20poly1305_ietf_ABYTES, - nullptr, - 0, - nonce.data(), - dec_key.data())) + auto nonce = ciphertext.last(); + if (!encryption::xchacha20poly1305_decrypt( + buf, + ciphertext.subspan(1, msg_size + encryption::XCHACHA20_ABYTES), + nonce, + dec_key)) throw std::invalid_argument{"Decryption failed"}; - assert(buf_len == buf.size()); - // Split up: the last 32 bytes are the sender's *unblinded* ed25519 key - uc32 sender_ed_pk; - std::memcpy(sender_ed_pk.data(), buf.data() + (buf.size() - 32), 32); + b32 sender_ed_pk; + std::ranges::copy(std::span{buf}.last<32>(), sender_ed_pk.begin()); // Convert the sender_ed_pk to the sender's session ID - uc32 sender_x_pk; - if (0 != crypto_sign_ed25519_pk_to_curve25519(sender_x_pk.data(), sender_ed_pk.data())) - throw std::runtime_error{"Sender ed25519 pubkey to x25519 pubkey conversion failed"}; - - std::vector session_id; // Gets populated by the following ..._from_ed calls + auto sender_x_pk = ed25519::pk_to_x25519(sender_ed_pk); // Verify that the inner sender_ed_pk (A) yields the same outer kA we got with the message - auto extracted_sender = - recipient_id[0] == 0x25 - ? blinded25_id_from_ed(to_span(sender_ed_pk), server_pk, &session_id) - : blinded15_id_from_ed(to_span(sender_ed_pk), server_pk, &session_id); + auto extracted_sender = recipient_id[0] == std::byte{0x25} + ? blinded25_id_from_ed(sender_ed_pk, server_pk) + : blinded15_id_from_ed(sender_ed_pk, server_pk); bool matched = to_string_view(sender_id) == to_string_view(extracted_sender); - if (!matched && extracted_sender[0] == 0x15) { + if (!matched && extracted_sender[0] == std::byte{0x15}) { // With 15-blinding we might need the negative instead: - extracted_sender[31] ^= 0x80; + extracted_sender[31] ^= std::byte{0x80}; matched = to_string_view(sender_id) == to_string_view(extracted_sender); } if (!matched) @@ -673,50 +872,36 @@ std::pair, std::string> decrypt_from_blinded_recipien // Everything is good, so just drop the sender_ed_pk off the message and prepend the '05' prefix // to the sender session ID buf.resize(buf.size() - 32); - sender_session_id.reserve(66); - sender_session_id += "05"; - oxenc::to_hex(sender_x_pk.begin(), sender_x_pk.end(), std::back_inserter(sender_session_id)); + sender_session_id = "05{:x}"_format(sender_x_pk); return result; } DecryptGroupMessage decrypt_group_message( - std::span> decrypt_ed25519_privkey_list, - std::span group_ed25519_pubkey, - std::span ciphertext) { + std::span> group_enc_keys, + std::span group_ed25519_pubkey, + std::span ciphertext) { DecryptGroupMessage result = {}; + auto& [res_index, session_id, data] = result; if (ciphertext.size() < GROUPS_ENCRYPT_OVERHEAD) throw std::runtime_error{"ciphertext is too small to be encrypted data"}; - if (group_ed25519_pubkey.size() != crypto_sign_ed25519_PUBLICKEYBYTES) - throw std::invalid_argument{"Invalid decrypt_ed25519_privkey: expected 32 bytes"}; - // Note we only use the secret key of the decrypt_ed25519_privkey so we don't care about - // generating the pubkey component if the user only passed in a 32 byte libsodium-style secret - // key. + // Each group encryption key is a 32-byte symmetric XChaCha20-Poly1305 key. Multiple keys + // are tried because the group key rotates and recently-received messages may still be + // encrypted with a pre-rotation key. - std::vector plain; + std::vector plain; - auto nonce = ciphertext.subspan(0, crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - ciphertext = ciphertext.subspan(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - plain.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES); + auto nonce = ciphertext.first(); + ciphertext = ciphertext.subspan(encryption::XCHACHA20_NONCEBYTES); + plain.resize(ciphertext.size() - encryption::XCHACHA20_ABYTES); bool decrypt_success = false; - for (size_t index = 0; index < decrypt_ed25519_privkey_list.size(); index++) { - const auto& decrypt_ed25519_privkey = decrypt_ed25519_privkey_list[index]; - if (decrypt_ed25519_privkey.size() != 32 && decrypt_ed25519_privkey.size() != 64) - throw std::invalid_argument{"Invalid decrypt_ed25519_privkey: expected 32 or 64 bytes"}; - decrypt_success = 0 == crypto_aead_xchacha20poly1305_ietf_decrypt( - plain.data(), - nullptr, - nullptr, - ciphertext.data(), - ciphertext.size(), - nullptr, - 0, - nonce.data(), - decrypt_ed25519_privkey.data()); + for (size_t index = 0; index < group_enc_keys.size(); index++) { + decrypt_success = encryption::xchacha20poly1305_decrypt( + plain, ciphertext, nonce, group_enc_keys[index]); if (decrypt_success) { - result.index = index; + res_index = index; break; } } @@ -727,44 +912,33 @@ DecryptGroupMessage decrypt_group_message( // // Removing any null padding bytes from the end // - if (auto it = - std::find_if(plain.rbegin(), plain.rend(), [](unsigned char c) { return c != 0; }); - it != plain.rend()) - plain.resize(plain.size() - std::distance(plain.rbegin(), it)); + trim_trailing(plain); // // Now what we have less should be a bt_dict // - if (plain.empty() || plain.front() != 'd' || plain.back() != 'e') + if (plain.empty() || plain.front() != std::byte{'d'} || plain.back() != std::byte{'e'}) throw std::runtime_error{"decrypted data is not a bencoded dict"}; oxenc::bt_dict_consumer dict{to_string_view(plain)}; if (auto v = dict.require(""); v != 1) throw std::runtime_error{ - "group message version tag (" + std::to_string(v) + - ") is not compatible (we support v1)"}; + fmt::format("group message version tag ({}) is not compatible (we support v1)", v)}; - auto ed_pk = dict.require_span("a"); + auto ed_pk = dict.require_span("a"); + auto x_pk = ed25519::pk_to_x25519(ed_pk); - std::array x_pk; - if (0 != crypto_sign_ed25519_pk_to_curve25519(x_pk.data(), ed_pk.data())) - throw std::runtime_error{ - "author ed25519 pubkey is invalid (unable to convert it to a session id)"}; - - auto& [_, session_id, data] = result; - session_id.reserve(66); - session_id += "05"; - oxenc::to_hex(x_pk.begin(), x_pk.end(), std::back_inserter(session_id)); + session_id = "05{:x}"_format(x_pk); - auto plain_data = dict.maybe>("d"); + auto plain_data = dict.maybe>("d"); if (plain_data && plain_data->empty()) - throw std::runtime_error{"uncompressed message data (\"d\") cannot be empty"}; + throw std::runtime_error{"uncompressed message data (d) cannot be empty"}; - auto ed_sig = dict.require_span("s"); + auto ed_sig = dict.require_span("s"); bool compressed = false; - auto comp_data = dict.maybe>("z"); + auto comp_data = dict.maybe>("z"); if (comp_data) { if (comp_data->empty()) throw std::runtime_error{"compressed message data (z) cannot be empty"}; @@ -781,20 +955,19 @@ DecryptGroupMessage decrypt_group_message( // The value we verify is the raw data *followed by* the group Ed25519 pubkey. (See the comment // in encrypt_message). - std::vector to_verify; + std::vector to_verify; to_verify.reserve(raw_data.size() + group_ed25519_pubkey.size()); to_verify.insert(to_verify.end(), raw_data.begin(), raw_data.end()); to_verify.insert(to_verify.end(), group_ed25519_pubkey.begin(), group_ed25519_pubkey.end()); - if (0 != crypto_sign_ed25519_verify_detached( - ed_sig.data(), to_verify.data(), to_verify.size(), ed_pk.data())) + if (!ed25519::verify(ed_sig.first<64>(), ed_pk.first<32>(), to_verify)) throw std::runtime_error{"message signature failed validation"}; if (compressed) { - if (auto decomp = zstd_decompress(raw_data, GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE)) + if (auto decomp = zstd_decompress(raw_data, GROUPS_MAX_PLAINTEXT_MESSAGE_SIZE)) { data = std::move(*decomp); - else + } else throw std::runtime_error{"message decompression failed"}; } else data.assign(raw_data.begin(), raw_data.end()); @@ -802,219 +975,102 @@ DecryptGroupMessage decrypt_group_message( return result; } +// The old Argon2-based ONS encryption always used an all-zero salt and all-zero secretbox nonce. +static constexpr std::array ONS_ARGON2_SALT = {}; +static constexpr std::array ONS_SECRETBOX_NONCE = {}; + std::string decrypt_ons_response( std::string_view lowercase_name, - std::span ciphertext, - std::optional> nonce) { + std::span ciphertext, + std::optional> nonce) { // Handle old Argon2-based encryption used before HF16 if (!nonce) { - if (ciphertext.size() < crypto_secretbox_MACBYTES) + if (ciphertext.size() < encryption::SECRETBOX_MACBYTES) throw std::invalid_argument{"Invalid ciphertext: expected to be greater than 16 bytes"}; - uc32 key; - std::array salt = {0}; - - if (0 != crypto_pwhash( - key.data(), - key.size(), - lowercase_name.data(), - lowercase_name.size(), - salt.data(), - crypto_pwhash_OPSLIMIT_MODERATE, - crypto_pwhash_MEMLIMIT_MODERATE, - crypto_pwhash_ALG_ARGON2ID13)) - throw std::runtime_error{"Failed to generate key"}; - - std::vector msg; - msg.resize(ciphertext.size() - crypto_secretbox_MACBYTES); - std::array nonce = {0}; - - if (0 != - crypto_secretbox_open_easy( - msg.data(), ciphertext.data(), ciphertext.size(), nonce.data(), key.data())) + b32 key; + hash::argon2( + key, + {lowercase_name.data(), lowercase_name.size()}, + ONS_ARGON2_SALT, + hash::ARGON2_OPSLIMIT_MODERATE, + hash::ARGON2_MEMLIMIT_MODERATE, + hash::ARGON2ID13); + + std::vector msg; + msg.resize(ciphertext.size() - encryption::SECRETBOX_MACBYTES); + + if (!encryption::secretbox_open_easy(msg, ciphertext, ONS_SECRETBOX_NONCE, key)) throw std::runtime_error{"Failed to decrypt"}; - std::string session_id = oxenc::to_hex(msg.begin(), msg.end()); - return session_id; + return oxenc::to_hex(msg); } - if (ciphertext.size() < crypto_aead_xchacha20poly1305_ietf_ABYTES) - throw std::invalid_argument{"Invalid ciphertext: expected to be greater than 16 bytes"}; - if (nonce->size() != crypto_aead_xchacha20poly1305_ietf_NPUBBYTES) - throw std::invalid_argument{"Invalid nonce: expected to be 24 bytes"}; + static_assert(encryption::XCHACHA20_NONCEBYTES == 24); + if (ciphertext.size() != 33 + encryption::XCHACHA20_ABYTES) + throw std::invalid_argument{"Invalid ciphertext: expected exactly 49 bytes"}; // Hash the ONS name using BLAKE2b // // xchacha-based encryption // key = H(name, key=H(name)) - uc32 key; - uc32 name_hash; - auto name_bytes = to_unsigned(lowercase_name.data()); - crypto_generichash_blake2b( - name_hash.data(), name_hash.size(), name_bytes, lowercase_name.size(), nullptr, 0); - crypto_generichash_blake2b( - key.data(), - key.size(), - name_bytes, - lowercase_name.size(), - name_hash.data(), - name_hash.size()); - - std::vector buf; - unsigned long long buf_len = 0; - buf.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES); - - if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt( - buf.data(), - &buf_len, - nullptr, - ciphertext.data(), - ciphertext.size(), - nullptr, - 0, - nonce->data(), - key.data())) - throw std::runtime_error{"Failed to decrypt"}; + b32 name_hash; + hash::blake2b(name_hash, lowercase_name); + auto key = hash::blake2b_key<32>(name_hash, lowercase_name); - if (buf_len != 33) - throw std::runtime_error{"Invalid decrypted value: expected to be 33 bytes"}; + std::array buf; + if (!encryption::xchacha20poly1305_decrypt(buf, ciphertext, *nonce, key)) + throw std::runtime_error{"Failed to decrypt"}; - std::string session_id = oxenc::to_hex(buf.begin(), buf.end()); - return session_id; + return oxenc::to_hex(buf); } -std::vector decrypt_push_notification( - std::span payload, std::span enc_key) { - if (payload.size() < - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + crypto_aead_xchacha20poly1305_ietf_ABYTES) +std::vector decrypt_push_notification( + std::span payload, std::span enc_key) { + if (payload.size() < encryption::XCHACHA20_NONCEBYTES + encryption::XCHACHA20_ABYTES) throw std::invalid_argument{"Invalid payload: too short to contain valid encrypted data"}; - if (enc_key.size() != 32) - throw std::invalid_argument{"Invalid enc_key: expected 32 bytes"}; - std::vector buf; - std::vector nonce; - const size_t msg_size = - (payload.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES - - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - unsigned long long buf_len = 0; - buf.resize(msg_size); - nonce.resize(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - std::memcpy(nonce.data(), payload.data(), crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - - if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt( - buf.data(), - &buf_len, - nullptr, - payload.data() + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, - payload.size() - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, - nullptr, - 0, - nonce.data(), - enc_key.data())) + auto nonce = payload.first(); + auto ct = payload.subspan(encryption::XCHACHA20_NONCEBYTES); + + std::vector buf(ct.size() - encryption::XCHACHA20_ABYTES); + + if (!encryption::xchacha20poly1305_decrypt(buf, ct, nonce, enc_key)) throw std::runtime_error{"Failed to decrypt; perhaps the secret key is invalid?"}; // Removing any null padding bytes from the end - if (auto it = std::find_if(buf.rbegin(), buf.rend(), [](unsigned char c) { return c != 0; }); - it != buf.rend()) - buf.resize(buf.size() - std::distance(buf.rbegin(), it)); + trim_trailing(buf); return buf; } -template -std::string compute_hash(Func hasher, const T&... args) { - // Allocate a buffer of 20 bytes per integral value (which is the largest the any integral - // value can be when stringified). - std::array< - char, - (0 + ... + - (std::is_integral_v || std::is_same_v - ? 20 - : 0))> - buffer; - auto* b = buffer.data(); - return hasher({detail::to_hashable(args, b)...}); -} +std::vector encrypt_xchacha20( + std::span plaintext, std::span key) { -std::string compute_hash_blake2b_b64(std::vector parts) { - constexpr size_t HASH_SIZE = 32; - crypto_generichash_state state; - crypto_generichash_init(&state, nullptr, 0, HASH_SIZE); - for (const auto& s : parts) - crypto_generichash_update( - &state, reinterpret_cast(s.data()), s.size()); - std::array hash; - crypto_generichash_final(&state, hash.data(), HASH_SIZE); - - std::string b64hash = oxenc::to_base64(hash.begin(), hash.end()); - // Trim padding: - while (!b64hash.empty() && b64hash.back() == '=') - b64hash.pop_back(); - return b64hash; -} + std::vector ciphertext( + encryption::XCHACHA20_NONCEBYTES + plaintext.size() + encryption::XCHACHA20_ABYTES); -std::vector encrypt_xchacha20( - std::span plaintext, std::span enc_key) { - if (enc_key.size() != 32) - throw std::invalid_argument{"Invalid enc_key: expected 32 bytes"}; + auto nonce = std::span{ciphertext}.first(); + random::fill(nonce); - std::vector ciphertext; - ciphertext.resize( - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + plaintext.size() + - crypto_aead_xchacha20poly1305_ietf_ABYTES); - - // Generate random nonce, and stash it at the beginning of ciphertext: - randombytes_buf(ciphertext.data(), crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); - - auto* c = reinterpret_cast(ciphertext.data()) + - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES; - unsigned long long clen; - - crypto_aead_xchacha20poly1305_ietf_encrypt( - c, - &clen, - plaintext.data(), - plaintext.size(), - nullptr, - 0, // additional data - nullptr, // nsec (always unused) - reinterpret_cast(ciphertext.data()), - enc_key.data()); - assert(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + clen <= ciphertext.size()); - ciphertext.resize(crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + clen); + encryption::xchacha20poly1305_encrypt( + std::span{ciphertext}.subspan(encryption::XCHACHA20_NONCEBYTES), plaintext, nonce, key); return ciphertext; } -std::vector decrypt_xchacha20( - std::span ciphertext, std::span enc_key) { - if (ciphertext.size() < - crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + crypto_aead_xchacha20poly1305_ietf_ABYTES) +std::vector decrypt_xchacha20( + std::span ciphertext, std::span key) { + if (ciphertext.size() < encryption::XCHACHA20_NONCEBYTES + encryption::XCHACHA20_ABYTES) throw std::invalid_argument{ "Invalid ciphertext: too short to contain valid encrypted data"}; - if (enc_key.size() != 32) - throw std::invalid_argument{"Invalid enc_key: expected 32 bytes"}; // Extract nonce from the beginning of the ciphertext: - auto nonce = ciphertext.subspan(0, crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); + auto nonce = ciphertext.first(); ciphertext = ciphertext.subspan(nonce.size()); - std::vector plaintext; - plaintext.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES); - auto* m = reinterpret_cast(plaintext.data()); - unsigned long long mlen; - if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt( - m, - &mlen, - nullptr, // nsec (always unused) - ciphertext.data(), - ciphertext.size(), - nullptr, - 0, // additional data - nonce.data(), - enc_key.data())) + std::vector plaintext(ciphertext.size() - encryption::XCHACHA20_ABYTES); + if (!encryption::xchacha20poly1305_decrypt(plaintext, ciphertext, nonce, key)) throw std::runtime_error{"Could not decrypt (XChaCha20-Poly1305)"}; - assert(mlen <= plaintext.size()); - plaintext.resize(mlen); return plaintext; } @@ -1033,9 +1089,9 @@ LIBSESSION_C_API bool session_encrypt_for_recipient_deterministic( size_t* ciphertext_len) { try { auto ciphertext = session::encrypt_for_recipient_deterministic( - std::span{ed25519_privkey, 64}, - std::span{recipient_pubkey, 32}, - std::span{plaintext_in, plaintext_len}); + to_byte_span<64>(ed25519_privkey), + to_byte_span<32>(recipient_pubkey), + to_byte_span(plaintext_in, plaintext_len)); *ciphertext_out = static_cast(malloc(ciphertext.size())); *ciphertext_len = ciphertext.size(); @@ -1056,10 +1112,10 @@ LIBSESSION_C_API bool session_encrypt_for_blinded_recipient( size_t* ciphertext_len) { try { auto ciphertext = session::encrypt_for_blinded_recipient( - std::span{ed25519_privkey, 64}, - std::span{community_pubkey, 32}, - std::span{recipient_blinded_id, 33}, - std::span{plaintext_in, plaintext_len}); + to_byte_span<64>(ed25519_privkey), + to_byte_span<32>(community_pubkey), + to_byte_span<33>(recipient_blinded_id), + to_byte_span(plaintext_in, plaintext_len)); *ciphertext_out = static_cast(malloc(ciphertext.size())); *ciphertext_len = ciphertext.size(); @@ -1084,11 +1140,11 @@ LIBSESSION_C_API session_encrypt_group_message session_encrypt_for_group( size_t error_len) { session_encrypt_group_message result = {}; try { - std::vector result_cpp = encrypt_for_group( + std::vector result_cpp = encrypt_for_group( {user_ed25519_privkey, user_ed25519_privkey_len}, - {group_ed25519_pubkey, crypto_sign_ed25519_PUBLICKEYBYTES}, - {group_enc_key, group_enc_key_len}, - {plaintext, plaintext_len}, + to_byte_span<32>(group_ed25519_pubkey), + to_byte_span(group_enc_key, group_enc_key_len), + to_byte_span(plaintext, plaintext_len), compress, padding); result = { @@ -1096,11 +1152,7 @@ LIBSESSION_C_API session_encrypt_group_message session_encrypt_for_group( .ciphertext = session::span_u8_copy_or_throw(result_cpp.data(), result_cpp.size()), }; } catch (const std::exception& e) { - std::string error_cpp = e.what(); - result.error_len_incl_null_terminator = - snprintf_clamped( - error, error_len, "%.*s", (int)error_cpp.size(), error_cpp.data()) + - 1; + result.error_len_incl_null_terminator = copy_c_str(error, error_len, e.what()); } return result; } @@ -1114,8 +1166,7 @@ LIBSESSION_C_API bool session_decrypt_incoming( size_t* plaintext_len) { try { auto result = session::decrypt_incoming_session_id( - std::span{ed25519_privkey, 64}, - std::span{ciphertext_in, ciphertext_len}); + to_byte_span<64>(ed25519_privkey), to_byte_span(ciphertext_in, ciphertext_len)); auto [plaintext, session_id] = result; std::memcpy(session_id_out, session_id.c_str(), session_id.size() + 1); @@ -1138,9 +1189,9 @@ LIBSESSION_C_API bool session_decrypt_incoming_legacy_group( size_t* plaintext_len) { try { auto result = session::decrypt_incoming_session_id( - std::span{x25519_pubkey, 32}, - std::span{x25519_seckey, 32}, - std::span{ciphertext_in, ciphertext_len}); + to_byte_span<32>(x25519_pubkey), + to_byte_span<32>(x25519_seckey), + to_byte_span(ciphertext_in, ciphertext_len)); auto [plaintext, session_id] = result; std::memcpy(session_id_out, session_id.c_str(), session_id.size() + 1); @@ -1165,11 +1216,11 @@ LIBSESSION_C_API bool session_decrypt_for_blinded_recipient( size_t* plaintext_len) { try { auto result = session::decrypt_from_blinded_recipient( - std::span{ed25519_privkey, 64}, - std::span{community_pubkey, 32}, - std::span{sender_id, 33}, - std::span{recipient_id, 33}, - std::span{ciphertext_in, ciphertext_len}); + to_byte_span<64>(ed25519_privkey), + to_byte_span<32>(community_pubkey), + to_byte_span<33>(sender_id), + to_byte_span<33>(recipient_id), + to_byte_span(ciphertext_in, ciphertext_len)); auto [plaintext, session_id] = result; std::memcpy(session_id_out, session_id.c_str(), session_id.size() + 1); @@ -1191,32 +1242,27 @@ LIBSESSION_C_API session_decrypt_group_message_result session_decrypt_group_mess char* error, size_t error_len) { session_decrypt_group_message_result result = {}; - for (size_t index = 0; index < decrypt_ed25519_privkey_len; index++) { - std::span key = { - decrypt_ed25519_privkey_list[index].data, decrypt_ed25519_privkey_list[index].size}; - - DecryptGroupMessage result_cpp = {}; - try { - result_cpp = decrypt_group_message( - {&key, 1}, - {group_ed25519_pubkey, crypto_sign_ed25519_PUBLICKEYBYTES}, - {ciphertext, ciphertext_len}); - result = { - .success = true, - .index = index, - .plaintext = session::span_u8_copy_or_throw( - result.plaintext.data, result.plaintext.size), - }; - assert(result_cpp.session_id.size() == sizeof(result.session_id)); - std::memcpy(result.session_id, result_cpp.session_id.data(), sizeof(result.session_id)); - break; - } catch (const std::exception& e) { - std::string error_cpp = e.what(); - result.error_len_incl_null_terminator = - snprintf_clamped( - error, error_len, "%.*s", (int)error_cpp.size(), error_cpp.data()) + - 1; + try { + std::vector> keys; + keys.reserve(decrypt_ed25519_privkey_len); + for (size_t i = 0; i < decrypt_ed25519_privkey_len; i++) { + if (decrypt_ed25519_privkey_list[i].size != 32) + throw std::invalid_argument{fmt::format( + "Invalid group encryption key: expected 32 bytes, got {}", + decrypt_ed25519_privkey_list[i].size)}; + keys.push_back(to_byte_span<32>(decrypt_ed25519_privkey_list[i].data)); } + auto [index, session_id, plaintext] = decrypt_group_message( + keys, + to_byte_span<32>(group_ed25519_pubkey), + to_byte_span(ciphertext, ciphertext_len)); + result.success = true; + result.index = index; + result.plaintext = session::span_u8_copy_or_throw(plaintext.data(), plaintext.size()); + assert(session_id.size() == sizeof(result.session_id)); + std::memcpy(result.session_id, session_id.data(), sizeof(result.session_id)); + } catch (const std::exception& e) { + result.error_len_incl_null_terminator = format_c_str(error, error_len, "{}", e.what()); } return result; } @@ -1228,13 +1274,12 @@ LIBSESSION_C_API bool session_decrypt_ons_response( const unsigned char* nonce_in, char* session_id_out) { try { - std::optional> nonce; + std::optional> nonce; if (nonce_in) - nonce = std::span{ - nonce_in, crypto_aead_xchacha20poly1305_ietf_NPUBBYTES}; + nonce = to_byte_span(nonce_in); auto session_id = session::decrypt_ons_response( - name_in, std::span{ciphertext_in, ciphertext_len}, nonce); + name_in, to_byte_span(ciphertext_in, ciphertext_len), nonce); std::memcpy(session_id_out, session_id.c_str(), session_id.size() + 1); return true; @@ -1251,8 +1296,7 @@ LIBSESSION_C_API bool session_decrypt_push_notification( size_t* plaintext_len) { try { auto plaintext = session::decrypt_push_notification( - std::span{payload_in, payload_len}, - std::span{enc_key_in, 32}); + to_byte_span(payload_in, payload_len), to_byte_span<32>(enc_key_in)); *plaintext_out = static_cast(malloc(plaintext.size())); *plaintext_len = plaintext.size(); @@ -1266,13 +1310,12 @@ LIBSESSION_C_API bool session_decrypt_push_notification( LIBSESSION_C_API bool session_encrypt_xchacha20( const unsigned char* plaintext_in, size_t plaintext_len, - const unsigned char* enc_key_in, + const unsigned char* key_in, unsigned char** ciphertext_out, size_t* ciphertext_len) { try { auto ciphertext = session::encrypt_xchacha20( - std::span{plaintext_in, plaintext_len}, - std::span{enc_key_in, 32}); + to_byte_span(plaintext_in, plaintext_len), to_byte_span<32>(key_in)); *ciphertext_out = static_cast(malloc(ciphertext.size())); *ciphertext_len = ciphertext.size(); @@ -1286,13 +1329,12 @@ LIBSESSION_C_API bool session_encrypt_xchacha20( LIBSESSION_C_API bool session_decrypt_xchacha20( const unsigned char* ciphertext_in, size_t ciphertext_len, - const unsigned char* enc_key_in, + const unsigned char* key_in, unsigned char** plaintext_out, size_t* plaintext_len) { try { auto plaintext = session::decrypt_xchacha20( - std::span{ciphertext_in, ciphertext_len}, - std::span{enc_key_in, 32}); + to_byte_span(ciphertext_in, ciphertext_len), to_byte_span<32>(key_in)); *plaintext_out = static_cast(malloc(plaintext.size())); *plaintext_len = plaintext.size(); diff --git a/src/session_protocol.cpp b/src/session_protocol.cpp index 0f9d50c1a..8525f6756 100644 --- a/src/session_protocol.cpp +++ b/src/session_protocol.cpp @@ -1,10 +1,6 @@ #include -#include #include #include -#include -#include -#include #include #include @@ -16,10 +12,10 @@ #include #include #include -#include #include "SessionProtos.pb.h" #include "WebSocketResources.pb.h" +#include "internal-util.hpp" #include "pro_message.hpp" #include "session/export.h" @@ -54,9 +50,9 @@ const session_protocol_strings SESSION_PROTOCOL_STRINGS = { // clang-format on namespace { -std::vector proof_signed_message( - std::span revocation_tag, - std::span rotating_pubkey, +std::vector proof_signed_message( + std::span revocation_tag, + std::span rotating_pubkey, std::int64_t expiry_ts) { // This must match the Pro proof signed message in pro-wire-protocol.md §2 (built per §1.1). No @@ -66,29 +62,11 @@ std::vector proof_signed_message( session::BUILD_PROOF_DOMAIN, revocation_tag, rotating_pubkey, expiry_ts); } -bool proof_verify_message_internal( - std::span rotating_pubkey, - std::span sig, - std::span msg) { - // C++ throws on bad size, C uses a fixed sized array - assert(rotating_pubkey.size() == crypto_sign_ed25519_PUBLICKEYBYTES); - if (sig.size() != crypto_sign_ed25519_BYTES) - return false; - - int verify_result = crypto_sign_ed25519_verify_detached( - reinterpret_cast(sig.data()), - msg.data(), - msg.size(), - reinterpret_cast(rotating_pubkey.data())); - bool result = verify_result == 0; - return result; -} - // Copies an optional 32-byte value out of a possibly-null C pointer. A null pointer yields a // zero-filled value (the "not provided" case); a non-null pointer with the wrong length yields // nullopt to signal an error. -static std::optional maybe_uc32_from_ptr(const void* ptr, size_t len) { - session::array_uc32 out = {}; +static std::optional maybe_uc32_from_ptr(const void* ptr, size_t len) { + session::b32 out = {}; if (ptr) { if (len != out.max_size()) return std::nullopt; @@ -121,46 +99,49 @@ static session_protocol_decoded_pro decoded_pro_from_cpp(const session::DecodedP cpp.proof.rotating_pubkey.max_size()); result.proof.expiry_ts = session::epoch_seconds(cpp.proof.expiry_at); std::memcpy(result.proof.sig.data, cpp.proof.sig.data(), cpp.proof.sig.max_size()); - result.msg_bitset.data = cpp.msg_bitset.data; - result.profile_bitset.data = cpp.profile_bitset.data; + result.msg_bitset = static_cast(cpp.msg_flags); + result.profile_bitset = static_cast(cpp.profile_flags); return result; } + +// Builds a C++ ProProof from the C proof struct (the inverse of the proof half of +// decoded_pro_from_cpp). +static session::ProProof proof_from_c(const session_protocol_pro_proof& c) { + session::ProProof proof = {}; + std::memcpy( + proof.revocation_tag.data(), c.revocation_tag.data, proof.revocation_tag.max_size()); + std::memcpy( + proof.rotating_pubkey.data(), c.rotating_pubkey.data, proof.rotating_pubkey.max_size()); + proof.expiry_at = session::as_sys_seconds(c.expiry_ts); + std::memcpy(proof.sig.data(), c.sig.data, proof.sig.max_size()); + return proof; +} } // namespace namespace session { static_assert(sizeof(ProProof::revocation_tag) == 32); -static_assert(sizeof(ProProof::rotating_pubkey) == crypto_sign_ed25519_PUBLICKEYBYTES); -static_assert(sizeof(ProProof::sig) == crypto_sign_ed25519_BYTES); +static_assert(sizeof(ProProof::rotating_pubkey) == 32); +static_assert(sizeof(ProProof::sig) == 64); -bool ProProof::verify_signature(const std::span& verify_pubkey) const { - if (verify_pubkey.size() != crypto_sign_ed25519_PUBLICKEYBYTES) - throw std::invalid_argument{fmt::format( - "Invalid verify_pubkey: Must be 32 byte Ed25519 public key (was: {})", - verify_pubkey.size())}; - - auto msg = proof_signed_message(revocation_tag, rotating_pubkey, epoch_seconds(expiry_at)); - bool result = proof_verify_message_internal(verify_pubkey, sig, msg); - return result; +bool ProProof::verify_signature(std::span verify_pubkey) const { + return ed25519::verify(sig, verify_pubkey, signed_message()); } -bool ProProof::verify_message(std::span sig, std::span msg) const { - if (sig.size() != crypto_sign_ed25519_BYTES) - throw std::invalid_argument{fmt::format( - "Invalid signed_msg: Signature must be 64 bytes (was: {})", sig.size())}; - bool result = proof_verify_message_internal(rotating_pubkey, sig, msg); - return result; +bool ProProof::verify_message( + std::span sig, std::span msg) const { + return ed25519::verify(sig, rotating_pubkey, msg); } -bool ProProof::is_active(sys_seconds unix_ts) const { +bool ProProof::is_active(std::chrono::sys_seconds unix_ts) const { return unix_ts <= expiry_at; } ProStatus ProProof::status( - std::span verify_pubkey, - sys_seconds unix_ts, - std::optional> user_sig, - std::span signed_msg) const { + std::span verify_pubkey, + std::chrono::sys_seconds unix_ts, + std::optional> user_sig, + std::span signed_msg) const { ProStatus result = ProStatus::Valid; // Verify the at the proof is verified by the Session Pro Backend key (e.g.: It was // issued by an authoritative backend) @@ -179,12 +160,12 @@ ProStatus ProProof::status( return result; } -std::vector ProProof::signed_message() const { - return proof_signed_message(revocation_tag, rotating_pubkey, epoch_seconds(expiry_at)); +std::vector ProProof::signed_message() const { + return proof_signed_message(revocation_tag, rotating_pubkey, session::epoch_seconds(expiry_at)); } -cleared_uc32 ProProof::rotating_seed( - std::span master_seed, std::chrono::sys_seconds now) { +cleared_b32 ProProof::rotating_seed( + std::span master_seed, std::chrono::sys_seconds now) { if (master_seed.size() != 32 && master_seed.size() != 64) throw std::invalid_argument{ "Invalid master_seed: expected a 32-byte Ed25519 seed or 64-byte libsodium key"}; @@ -197,61 +178,28 @@ cleared_uc32 ProProof::rotating_seed( char dec[20]; auto [ptr, ec] = std::to_chars(dec, dec + sizeof(dec), epoch_seconds(period_start)); assert(ec == std::errc{}); // dec is large enough for any int64, so this cannot fail + std::string_view dec_sv{dec, static_cast(ptr - dec)}; - auto seed = master_seed.first(32); - std::vector msg; - msg.reserve(seed.size() + static_cast(ptr - dec)); - msg.insert(msg.end(), seed.begin(), seed.end()); - msg.insert(msg.end(), dec, ptr); - - static constexpr std::string_view personal = "ProRotatingSeed_"; - static_assert(personal.size() == crypto_generichash_blake2b_PERSONALBYTES); - - cleared_uc32 out = {}; - crypto_generichash_blake2b_salt_personal( - out.data(), - out.size(), - msg.data(), - msg.size(), - nullptr, // key - 0, - nullptr, // salt - reinterpret_cast(personal.data())); - return out; -} + // Unkeyed, personalised (salt=null) BLAKE2b-256 -- byte-for-byte the libsodium + // crypto_generichash_blake2b_salt_personal this replaces; the wrapper concatenates its args. + auto out = session::hash::blake2b_pers<32>( + "ProRotatingSeed_"_b2b_pers, master_seed.first(32), dec_sv); -void ProProfileBitset::set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) { - data |= (1ULL << static_cast(features)); -} - -void ProProfileBitset::unset(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) { - data &= ~(1ULL << static_cast(features)); -} - -bool ProProfileBitset::is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) const { - bool result = data & (1ULL << static_cast(features)); + cleared_b32 result = {}; + std::memcpy(result.data(), out.data(), result.size()); return result; } -void ProMessageBitset::set(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) { - data |= (1ULL << static_cast(features)); -} - -void ProMessageBitset::unset(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) { - data &= ~(1ULL << static_cast(features)); -} +}; // namespace session -bool ProMessageBitset::is_set(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) const { - bool result = data & (1ULL << static_cast(features)); - return result; -} +namespace session { ProFeaturesForMsg pro_features_for_message(size_t codepoint_count) { ProFeaturesForMsg result = {}; result.status = ProFeaturesForMsgStatus::Success; if (codepoint_count > STANDARD_CHARACTER_LIMIT) { if (codepoint_count <= PRO_HIGHER_CHARACTER_LIMIT) { - result.bitset.set(SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT); + result.flags |= ProMessageFlags::CharLimit10k; } else { result.error = "Message exceeds the maximum character limit allowed"; result.status = ProFeaturesForMsgStatus::ExceedsCharacterLimit; @@ -260,80 +208,8 @@ ProFeaturesForMsg pro_features_for_message(size_t codepoint_count) { return result; } -std::vector encode_for_1o1( - std::span plaintext, - std::span ed25519_privkey, - std::chrono::milliseconds sent_timestamp, - const array_uc33& recipient_pubkey, - std::optional> pro_rotating_ed25519_privkey) { - Destination dest = {}; - dest.type = DestinationType::SyncOr1o1; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey ? *pro_rotating_ed25519_privkey - : std::span{}; - dest.sent_timestamp_ms = sent_timestamp; - dest.recipient_pubkey = recipient_pubkey; - std::vector result = encode_for_destination(plaintext, ed25519_privkey, dest); - return result; -} - -std::vector encode_for_community_inbox( - std::span plaintext, - std::span ed25519_privkey, - const array_uc33& recipient_pubkey, - const array_uc32& community_pubkey, - std::optional> pro_rotating_ed25519_privkey) { - Destination dest = {}; - dest.type = DestinationType::CommunityInbox; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey ? *pro_rotating_ed25519_privkey - : std::span{}; - // Unused on the CommunityInbox path (only the envelope-based Group/1o1 path consumes it). - dest.sent_timestamp_ms = std::chrono::milliseconds{0}; - dest.recipient_pubkey = recipient_pubkey; - dest.community_inbox_server_pubkey = community_pubkey; - std::vector result = encode_for_destination(plaintext, ed25519_privkey, dest); - return result; -} - -std::vector encode_for_community( - std::span plaintext, - std::optional> pro_rotating_ed25519_privkey) { - Destination dest = {}; - dest.type = DestinationType::Community; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey ? *pro_rotating_ed25519_privkey - : std::span{}; - std::span nil_ed25519_privkey; - std::vector result = encode_for_destination(plaintext, nil_ed25519_privkey, dest); - return result; -} - -std::vector encode_for_group( - std::span plaintext, - std::span ed25519_privkey, - std::chrono::milliseconds sent_timestamp, - const array_uc33& group_ed25519_pubkey, - const cleared_uc32& group_enc_key, - std::optional> pro_rotating_ed25519_privkey) { - Destination dest = {}; - dest.type = DestinationType::Group; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey ? *pro_rotating_ed25519_privkey - : std::span{}; - dest.sent_timestamp_ms = sent_timestamp; - dest.group_ed25519_pubkey = group_ed25519_pubkey; - dest.group_enc_key = group_enc_key; - std::vector result = encode_for_destination(plaintext, ed25519_privkey, dest); - return result; -} - -// Interop between the C and CPP API. The C api will request malloc which writes to `ciphertext_c`. -// This pointer is taken verbatim and avoids requiring a copy from the CPP vector. The CPP api will -// steal the contents from `ciphertext_cpp`. -struct EncryptedForDestinationInternal { - std::vector ciphertext_cpp; - span_u8 ciphertext_c; -}; - -constexpr char PADDING_TERMINATING_BYTE = 0x80; -std::vector pad_message(std::span payload) { +constexpr std::byte PADDING_TERMINATING_BYTE{0x80}; +std::vector pad_message(std::span payload) { // Calculate amount of padding required size_t padded_content_size = payload.size() + 1 /*padding byte*/; @@ -343,366 +219,182 @@ std::vector pad_message(std::span payload) { assert(padded_content_size % COMMUNITY_OR_1O1_MSG_PADDING == 0); // Do the padding - std::vector result; + std::vector result; result.resize(padded_content_size); std::memcpy(result.data(), payload.data(), payload.size()); result[payload.size()] = PADDING_TERMINATING_BYTE; return result; } -static std::span unpad_message(std::span payload) { - // Strip padding from content - size_t size_without_padding = payload.size(); - while (size_without_padding) { - char ch = payload[size_without_padding - 1]; - if (ch != 0 && ch != PADDING_TERMINATING_BYTE) { - // Non-zero padding encountered, terminate the loop and assume message is not - // padded - // TODO: We should enforce this but no client enforces it right now. - break; - } +static std::span unpad_message(std::span payload) { + auto size = payload.size() - count_trailing(payload); - size_without_padding--; - if (ch == PADDING_TERMINATING_BYTE) - break; - } + // The 0x80 terminator is required by the padding scheme, so its absence means the message was + // not padded at all. + // TODO: We should enforce this but no client enforces it right now. + if (size > 0 && payload[size - 1] == PADDING_TERMINATING_BYTE) + size--; - assert(size_without_padding <= payload.size()); - auto result = std::span(payload.data(), payload.data() + size_without_padding); - return result; + return payload.first(size); } -enum class UseMalloc { No, Yes }; -static EncryptedForDestinationInternal encode_for_destination_internal( - std::span plaintext, - std::span ed25519_privkey, - DestinationType dest_type, - std::span dest_pro_rotating_ed25519_privkey, - std::span dest_recipient_pubkey, - std::chrono::milliseconds dest_sent_timestamp_ms, - std::span dest_community_inbox_server_pubkey, - std::span dest_group_ed25519_pubkey, - std::span dest_group_enc_key, - UseMalloc use_malloc) { - // The following arguments are passed in from structs with fixed-sized arrays so we expect the - // sizes to be correct. It being wrong would be a development error - // - // The ed25519_privkey is passed into the lower level layer, session encrypt which has its own - // private key normalisation to 64 bytes for us. - assert(dest_recipient_pubkey.size() == 1 + crypto_sign_ed25519_PUBLICKEYBYTES); - assert(dest_community_inbox_server_pubkey.size() == crypto_sign_ed25519_PUBLICKEYBYTES); - assert(dest_group_ed25519_pubkey.size() == 1 + crypto_sign_ed25519_PUBLICKEYBYTES); - assert(dest_group_enc_key.size() == 32 || dest_group_enc_key.size() == 64); - - bool is_group = dest_type == DestinationType::Group; - bool is_1o1 = dest_type == DestinationType::SyncOr1o1; - bool is_community_inbox = dest_type == DestinationType::CommunityInbox; - bool is_community = dest_type == DestinationType::Community; - if (!is_community) { - assert(ed25519_privkey.size() == crypto_sign_ed25519_SECRETKEYBYTES || - ed25519_privkey.size() == crypto_sign_ed25519_SEEDBYTES); - } - - // Ensure the Session Pro rotating key is a 64 byte key if given - cleared_uc64 pro_ed_sk_from_seed; - if (dest_pro_rotating_ed25519_privkey.size()) { - if (dest_pro_rotating_ed25519_privkey.size() == 32) { - uc32 ignore_pk; - crypto_sign_ed25519_seed_keypair( - ignore_pk.data(), - pro_ed_sk_from_seed.data(), - dest_pro_rotating_ed25519_privkey.data()); - dest_pro_rotating_ed25519_privkey = to_span(pro_ed_sk_from_seed); - } else if (dest_pro_rotating_ed25519_privkey.size() == 64) { - dest_pro_rotating_ed25519_privkey = to_span(dest_pro_rotating_ed25519_privkey); - } else { - throw std::runtime_error{fmt::format( - "Invalid dest_pro_rotating_ed25519_privkey: expected 32 or 64 bytes, received " - "{}", - dest_pro_rotating_ed25519_privkey.size())}; - } - } - - std::span content = plaintext; - - EncryptedForDestinationInternal result = {}; - switch (dest_type) { - case DestinationType::Group: /*FALLTHRU*/ - case DestinationType::SyncOr1o1: { - if (is_group && - dest_group_ed25519_pubkey[0] != static_cast(SessionIDPrefix::group)) { - // Legacy groups which have a 05 prefixed key - throw std::runtime_error{ - "Unsupported configuration, encrypting for a legacy group (0x05 prefix) is " - "no longer supported"}; - } +// Attaches a Session Pro signature to an envelope. With no pro key a decoy signature is attached +// instead, so that pro and non-pro messages are indistinguishable on the wire (see +// ed25519::decoy_signature). +static void attach_pro_sig_to_envelope( + SessionProtos::Envelope& envelope, + std::span content, + const ed25519::OptionalPrivKeySpan& pro_key) { + b64 signature = pro_key ? ed25519::sign(*pro_key, content) : ed25519::decoy_signature(); + std::string* pro_sig = envelope.mutable_prosig(); + pro_sig->assign(reinterpret_cast(signature.data()), signature.size()); +} - // For Sync or 1o1 mesasges, we need to pad the contents to 160 bytes, see: - // https://github.com/session-foundation/session-desktop/blob/a04e62427034a6b6fee39dcff7dbabf0d0131b13/ts/session/crypto/BufferPadding.ts#L49 - std::vector tmp_content_buffer; - if (is_1o1) { // Encrypt the padded output - std::vector padded_payload = pad_message(content); - tmp_content_buffer = encrypt_for_recipient( - ed25519_privkey, dest_recipient_pubkey, padded_payload); - content = tmp_content_buffer; - } +// TODO: We don't need to actually pad the community message since that's unencrypted, +// there's no need to make the message sizes uniform but we need it for backwards +// compat. We can remove this eventually, first step is to unify the clients. +std::vector encode_for_community( + std::span plaintext, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey) { + if (!pro_rotating_ed25519_privkey) + return pad_message(plaintext); + + // TODO: Sub-optimal, but we parse the content again to make sure it's valid. Sign + // the blob then, fill in the signature in-place as part of the transitioning of + // open groups messages to envelopes. As part of that, libsession is going to take + // responsibility of constructing community messages so that eventually all + // platforms switch over to envelopes and we can change the implementation across + // all platforms in one swoop and remove this. + // + // Parse the content blob + SessionProtos::Content content_w_sig; + if (!content_w_sig.ParseFromArray(plaintext.data(), plaintext.size())) + throw std::runtime_error{"Parsing community message failed"}; - // Create envelope - // Set sourcedevice to 1 as per: - // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Utilities/MessageWrapper.swift#L57 - SessionProtos::Envelope envelope = {}; - envelope.set_type( - is_1o1 ? SessionProtos::Envelope_Type_SESSION_MESSAGE - : SessionProtos::Envelope_Type_CLOSED_GROUP_MESSAGE); - envelope.set_sourcedevice(1); - envelope.set_timestamp(dest_sent_timestamp_ms.count()); - envelope.set_content(content.data(), content.size()); - - // Generate the session pro signature. If there's no pro ed25519 key specified, we still - // fill out the pro signature with a decoy (validly-encoded but unverifiable) signature. - // This makes pro and non-pro messages indistinguishable on the wire. - { - std::string* pro_sig = envelope.mutable_prosig(); - pro_sig->resize(crypto_sign_ed25519_BYTES); - - if (dest_pro_rotating_ed25519_privkey.empty()) { - // No pro key: attach a decoy signature -- a validly-encoded Ed25519 signature - // (R = r·B for a random scalar r, so a prime-order-subgroup point like a real - // R; s a second random scalar) that verifies against nothing. Keeps pro and - // non-pro envelopes indistinguishable on the wire without signing throwaway - // data. NOT a real signature; never verified. - auto* sig = reinterpret_cast(pro_sig->data()); - std::array r; - crypto_core_ed25519_scalar_random(r.data()); - crypto_scalarmult_ed25519_base_noclamp(sig, r.data()); - crypto_core_ed25519_scalar_random(sig + crypto_core_ed25519_BYTES); - } else { - crypto_sign_ed25519_detached( - reinterpret_cast(pro_sig->data()), - nullptr, - content.data(), - content.size(), - dest_pro_rotating_ed25519_privkey.data()); - } - } + if (content_w_sig.has_prosigforcommunitymessageonly()) + throw std::runtime_error{ + "Pro signature for community message must not be set. Libsession's " + "responsible for generating the signature and setting it"}; + + // We need to sign the padded content, so we pad the `Content` then sign it + std::vector padded = pad_message(plaintext); + auto pro_sig = ed25519::sign(*pro_rotating_ed25519_privkey, padded); + + // Now assign the community specific pro signature field, reserialize it and we have + // to, yes, pad it again. This is all temporary wasted work whilst transitioning + // open groups. + content_w_sig.set_prosigforcommunitymessageonly( + reinterpret_cast(pro_sig.data()), pro_sig.size()); + std::vector reserialized(content_w_sig.ByteSizeLong()); + [[maybe_unused]] bool ok = + content_w_sig.SerializeToArray(reserialized.data(), reserialized.size()); + assert(ok); + return pad_message(reserialized); +} - if (is_group) { - std::string bytes = envelope.SerializeAsString(); - if (dest_group_ed25519_pubkey.size() == crypto_sign_ed25519_PUBLICKEYBYTES + 1) - dest_group_ed25519_pubkey = dest_group_ed25519_pubkey.subspan(1); - - std::vector ciphertext = encrypt_for_group( - ed25519_privkey, - dest_group_ed25519_pubkey, - dest_group_enc_key, - to_span(bytes), - /*compress*/ true, - /*padding*/ 256); - - if (use_malloc == UseMalloc::Yes) { - result.ciphertext_c = - session::span_u8_copy_or_throw(ciphertext.data(), ciphertext.size()); - } else { - result.ciphertext_cpp = std::move(ciphertext); - } - } else { - // 1o1, Wrap in websocket message - WebSocketProtos::WebSocketMessage msg = {}; - msg.set_type(WebSocketProtos::WebSocketMessage_Type::WebSocketMessage_Type_REQUEST); - - // Make request - WebSocketProtos::WebSocketRequestMessage* req_msg = msg.mutable_request(); - req_msg->set_verb(""); // Required but unused on iOS - req_msg->set_path(""); // Required but unused on iOS - req_msg->set_requestid(0); // Required but unused on iOS - req_msg->set_body(envelope.SerializeAsString()); - - // Write message as ciphertext - [[maybe_unused]] bool serialized = false; - if (use_malloc == UseMalloc::Yes) { - result.ciphertext_c = span_u8_alloc_or_throw(msg.ByteSizeLong()); - serialized = msg.SerializeToArray( - result.ciphertext_c.data, result.ciphertext_c.size); - } else { - result.ciphertext_cpp.resize(msg.ByteSizeLong()); - serialized = msg.SerializeToArray( - result.ciphertext_cpp.data(), result.ciphertext_cpp.size()); - } - assert(serialized); - } - } break; - - case DestinationType::Community: /*FALLTHRU*/ - case DestinationType::CommunityInbox: { - // Setup the pro signature for the community message - std::vector tmp_content_buffer; - - // Sign the message with the Session Pro key if given and then pad the message (both - // community message types require it) - // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Sending%20%26%20Receiving/MessageSender.swift#L398 - if (dest_pro_rotating_ed25519_privkey.size()) { - // Key should be verified by the time we hit this branch - assert(dest_pro_rotating_ed25519_privkey.size() == - crypto_sign_ed25519_SECRETKEYBYTES); - - // TODO: Sub-optimal, but we parse the content again to make sure it's valid. Sign - // the blob then, fill in the signature in-place as part of the transitioning of - // open groups messages to envelopes. As part of that, libsession is going to take - // responsibility of constructing community messages so that eventually all - // platforms switch over to envelopes and we can change the implementation across - // all platforms in one swoop and remove this. - // - // Parse the content blob - SessionProtos::Content content_w_sig = {}; - if (!content_w_sig.ParseFromArray(content.data(), content.size())) - throw std::runtime_error{"Parsing community message failed"}; - - if (content_w_sig.has_prosigforcommunitymessageonly()) - throw std::runtime_error{ - "Pro signature for community message must not be set. Libsession's " - "responsible for generating the signature and setting it"}; - - // We need to sign the padded content, so we pad the `Content` then sign it - tmp_content_buffer = pad_message(content); - array_uc64 pro_sig; - bool was_signed = crypto_sign_ed25519_detached( - pro_sig.data(), - nullptr, - tmp_content_buffer.data(), - tmp_content_buffer.size(), - dest_pro_rotating_ed25519_privkey.data()) == 0; - assert(was_signed); - - // Now assign the community specific pro signature field, reserialize it and we have - // to, yes, pad it again. This is all temporary wasted work whilst transitioning - // open groups. - content_w_sig.set_prosigforcommunitymessageonly(pro_sig.data(), pro_sig.size()); - tmp_content_buffer.resize(content_w_sig.ByteSizeLong()); - bool serialized = content_w_sig.SerializeToArray( - tmp_content_buffer.data(), tmp_content_buffer.size()); - assert(serialized); - - tmp_content_buffer = pad_message(tmp_content_buffer); - content = tmp_content_buffer; - } else { - tmp_content_buffer = pad_message(to_span(content)); - content = tmp_content_buffer; - } +std::vector encode_for_community_inbox( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, + std::span recipient_pubkey, + std::span community_pubkey, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey) { + std::vector content = encode_for_community(plaintext, pro_rotating_ed25519_privkey); + return encrypt_for_blinded_recipient( + ed25519_privkey, community_pubkey, recipient_pubkey, content); +} - // TODO: We don't need to actually pad the community message since that's unencrypted, - // there's no need to make the message sizes uniform but we need it for backwards - // compat. We can remove this eventually, first step is to unify the clients. - - if (is_community_inbox) { - std::vector ciphertext = encrypt_for_blinded_recipient( - ed25519_privkey, - dest_community_inbox_server_pubkey, - dest_recipient_pubkey, // recipient blinded pubkey - content); - - if (use_malloc == UseMalloc::Yes) { - result.ciphertext_c = - span_u8_copy_or_throw(ciphertext.data(), ciphertext.size()); - } else { - result.ciphertext_cpp = std::move(ciphertext); - } - } else { - if (use_malloc == UseMalloc::Yes) { - result.ciphertext_c = span_u8_copy_or_throw(content.data(), content.size()); - } else { - result.ciphertext_cpp = std::vector(content.begin(), content.end()); - } - } - } break; - } +std::vector encode_dm_v1( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, + sys_ms sent_timestamp, + std::span recipient_pubkey, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey) { + // For 1o1 messages, encrypt the padded payload for the recipient. See: + // https://github.com/session-foundation/session-desktop/blob/a04e62427034a6b6fee39dcff7dbabf0d0131b13/ts/session/crypto/BufferPadding.ts#L49 + std::vector encrypted = + encrypt_for_recipient(ed25519_privkey, recipient_pubkey, pad_message(plaintext)); + + // Create envelope. + // Set sourcedevice to 1 as per: + // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Utilities/MessageWrapper.swift#L57 + SessionProtos::Envelope envelope; + envelope.set_type(SessionProtos::Envelope_Type_SESSION_MESSAGE); + envelope.set_sourcedevice(1); + envelope.set_timestamp(epoch_ms(sent_timestamp)); + envelope.set_content(encrypted.data(), encrypted.size()); + attach_pro_sig_to_envelope(envelope, encrypted, pro_rotating_ed25519_privkey); + + // Wrap in websocket message + WebSocketProtos::WebSocketMessage msg; + msg.set_type(WebSocketProtos::WebSocketMessage_Type::WebSocketMessage_Type_REQUEST); + WebSocketProtos::WebSocketRequestMessage* req_msg = msg.mutable_request(); + req_msg->set_verb(""); // Required but unused on iOS + req_msg->set_path(""); // Required but unused on iOS + req_msg->set_requestid(0); // Required but unused on iOS + req_msg->set_body(envelope.SerializeAsString()); + + std::vector result(msg.ByteSizeLong()); + [[maybe_unused]] bool ok = msg.SerializeToArray(result.data(), result.size()); + assert(ok); return result; } -std::vector encode_for_destination( - std::span plaintext, - std::span ed25519_privkey, - const Destination& dest) { - - EncryptedForDestinationInternal result_internal = encode_for_destination_internal( - /*plaintext=*/plaintext, - /*ed25519_privkey=*/ed25519_privkey, - /*dest_type=*/dest.type, - /*dest_pro_rotating_ed25519_privkey=*/dest.pro_rotating_ed25519_privkey, - /*dest_recipient_pubkey=*/dest.recipient_pubkey, - /*dest_sent_timestamp_ms=*/dest.sent_timestamp_ms, - /*dest_community_inbox_server_pubkey=*/dest.community_inbox_server_pubkey, - /*dest_group_ed25519_pubkey=*/dest.group_ed25519_pubkey, - /*dest_group_enc_key=*/dest.group_enc_key, - /*use_malloc=*/UseMalloc::No); - - std::vector result = std::move(result_internal.ciphertext_cpp); - return result; +std::vector encode_for_group( + std::span plaintext, + const ed25519::PrivKeySpan& ed25519_privkey, + std::chrono::milliseconds sent_timestamp, + std::span group_ed25519_pubkey, + std::span group_enc_key, + const ed25519::OptionalPrivKeySpan& pro_rotating_ed25519_privkey) { + if (group_ed25519_pubkey[0] != std::byte{static_cast(SessionIDPrefix::group)}) { + // Legacy groups which have a 05 prefixed key + throw std::runtime_error{ + "Unsupported configuration, encrypting for a legacy group (0x05 prefix) is " + "no longer supported"}; + } + + // Create envelope. + // Set sourcedevice to 1 as per: + // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Utilities/MessageWrapper.swift#L57 + SessionProtos::Envelope envelope; + envelope.set_type(SessionProtos::Envelope_Type_CLOSED_GROUP_MESSAGE); + envelope.set_sourcedevice(1); + envelope.set_timestamp(sent_timestamp.count()); + envelope.set_content(plaintext.data(), plaintext.size()); + attach_pro_sig_to_envelope(envelope, plaintext, pro_rotating_ed25519_privkey); + + std::string bytes = envelope.SerializeAsString(); + return encrypt_for_group( + ed25519_privkey, + group_ed25519_pubkey.subspan<1>(), + group_enc_key, + to_span(bytes), + /*compress*/ true, + /*padding*/ 256); } -DecodedEnvelope decode_envelope( - const DecodeEnvelopeKey& keys, - std::span envelope_payload, - const array_uc32& pro_backend_pubkey) { - DecodedEnvelope result = {}; - SessionProtos::Envelope envelope = {}; - std::span envelope_plaintext = envelope_payload; - - // The caller is indicating that the envelope_payload is encrypted, if the group keys are - // provided. We will decrypt the payload to get the plaintext. In all other cases, the envelope - // is assumed to be websocket wrapped - std::vector envelope_from_decrypted_groups; - std::string envelope_from_websocket_message; - if (keys.group_ed25519_pubkey) { - // Decrypt using the keys - DecryptGroupMessage decrypt = decrypt_group_message( - keys.decrypt_keys, *keys.group_ed25519_pubkey, envelope_plaintext); - - if (decrypt.session_id.size() != ((crypto_sign_ed25519_PUBLICKEYBYTES + 1) * 2)) - throw std::runtime_error{fmt::format( - "Parse encrypted envelope failed, extracted session ID was wrong size: " - "{}", - decrypt.session_id.size())}; - - // Update the plaintext to use the decrypted envelope - envelope_from_decrypted_groups = std::move(decrypt.plaintext); - envelope_plaintext = envelope_from_decrypted_groups; - - // Copy keys out - assert(decrypt.session_id.starts_with("05")); - oxenc::from_hex( - decrypt.session_id.begin() + 2, - decrypt.session_id.end(), - result.sender_x25519_pubkey.begin()); - } else { - // Assumed to be a 1o1/sync message which is wrapped in a websocket message - WebSocketProtos::WebSocketMessage ws_msg; - if (!ws_msg.ParseFromArray(envelope_plaintext.data(), envelope_plaintext.size())) - throw std::runtime_error{fmt::format( - "Parse websocket wrapped envelope from payload failed: {}", - envelope_plaintext.size())}; - - if (!ws_msg.has_request()) - throw std::runtime_error{"Parse websocket wrapped envelope failed, missing request"}; - - if (!ws_msg.request().has_body()) - throw std::runtime_error{ - "Parse websocket wrapped envelope failed, missing request body"}; - - WebSocketProtos::WebSocketRequestMessage* request = ws_msg.mutable_request(); - std::string* body = request->mutable_body(); - envelope_from_websocket_message = std::move(*body); - envelope_plaintext = to_span(envelope_from_websocket_message); +// Parses the optional envelope metadata fields that are encoded identically for every message type +// (source device and server timestamp) into an Envelope. +static void parse_common_envelope_fields(Envelope& env, const SessionProtos::Envelope& pb) { + if (pb.has_sourcedevice()) { + env.source_device = pb.sourcedevice(); + env.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SOURCE_DEVICE; + } + if (pb.has_servertimestamp()) { + env.server_timestamp = pb.servertimestamp(); + env.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SERVER_TIMESTAMP; } +} - if (!envelope.ParseFromArray(envelope_plaintext.data(), envelope_plaintext.size())) - throw std::runtime_error{"Parse envelope from plaintext failed"}; +// Shared helper 1: parses envelope metadata fields (timestamp, source, etc.) from an +// already-parsed Envelope protobuf. +static void parse_envelope_fields( + DecodedEnvelope& result, const SessionProtos::Envelope& envelope) { - // TODO: We do not parse the envelop type anymore, we infer the type from - // the namespace. Deciding whether or not we decrypt the envelope vs the content depends on - // whether or not the group keys were passed in so we don't care about the type anymore. - // - // When the type is removed, we can remove this TODO. This is just a reminder as to why we skip - // over that field but it's still in the schema and still being set on the sending side. + // TODO: We do not parse the envelope type anymore, we infer the type from the namespace. + // Deciding whether or not we decrypt the envelope vs the content depends on the function + // called (dm vs group) so we don't care about the type anymore. When the type is removed + // from the schema, we can remove this TODO. // Parse timestamp if (envelope.has_timestamp()) { @@ -739,69 +431,60 @@ DecodedEnvelope decode_envelope( } } - // Parse source device (optional) - if (envelope.has_sourcedevice()) { - result.envelope.source_device = envelope.sourcedevice(); - result.envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SOURCE_DEVICE; - } + parse_common_envelope_fields(result.envelope, envelope); +} - // Parse server timestamp (optional) - if (envelope.has_servertimestamp()) { - result.envelope.server_timestamp = envelope.servertimestamp(); - result.envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SERVER_TIMESTAMP; +// Parses the proof and feature flags embedded in a protobuf ProMessage into a DecodedPro. A proof +// we can't read is NOT fatal: it degrades to a non-pro message flagged ProStatus::Invalid (the +// caller then skips signature evaluation), so a future proof format -- which arrives as its own +// field/message, not a version bump on this one -- cannot make an older client silently drop the +// whole message. Throws only when a proof that IS present is structurally malformed (corruption, +// not forward-compat). The caller evaluates `.status` for a proof that parsed. +static DecodedPro parse_pro_message(const SessionProtos::ProMessage& pro_msg) { + DecodedPro pro = {}; + pro.msg_flags = static_cast(pro_msg.msgbitset()); + pro.profile_flags = static_cast(pro_msg.profilebitset()); + + // No proof to read: either the sender attached none, or it is in a format we don't know -- a + // new proof format is a new field/message rather than a version bump on this one, so to this + // client it simply isn't here. Nothing to evaluate, so flag Invalid and let the caller deliver + // a non-pro message rather than dropping it. + if (!pro_msg.has_proof()) { + pro.status = ProStatus::Invalid; + return pro; } - // Parse content - if (!envelope.has_content()) - throw std::runtime_error{"Parse decrypted message failed, missing content"}; - - // Decrypt content - // The envelope is encrypted in GroupsV2, contents unencrypted. In 1o1 and legacy groups, the - // envelope is encrypted, contents is encrypted. - if (keys.group_ed25519_pubkey) { - result.content_plaintext.resize(envelope.content().size()); - std::memcpy( - result.content_plaintext.data(), - envelope.content().data(), - envelope.content().size()); - } else { - const std::string& content = envelope.content(); - bool decrypt_success = false; - std::vector content_plaintext; - std::vector sender_ed25519_pubkey; - for (const auto& privkey_it : keys.decrypt_keys) { - try { - std::tie(content_plaintext, sender_ed25519_pubkey) = - session::decrypt_incoming(privkey_it, to_span(content)); - assert(result.sender_ed25519_pubkey.size() == crypto_sign_ed25519_PUBLICKEYBYTES); - decrypt_success = true; - break; - } catch (...) { - } - } - - if (!decrypt_success) { - throw std::runtime_error{fmt::format( - "Envelope content decryption failed, tried {} key(s)", - keys.decrypt_keys.size())}; - } + // A proof that is present but the wrong shape is corruption, not forward-compat: hard error. + const SessionProtos::ProProof& proto_proof = pro_msg.proof(); + ProProof& proof = pro.proof; + bool valid = proto_proof.has_revocationtag() && + proto_proof.revocationtag().size() == proof.revocation_tag.max_size() && + proto_proof.has_rotatingpublickey() && + proto_proof.rotatingpublickey().size() == proof.rotating_pubkey.max_size() && + proto_proof.has_expiryunixts() && proto_proof.has_sig() && + proto_proof.sig().size() == proof.sig.max_size(); + if (!valid) + throw std::runtime_error{"Parse failed, pro metadata was malformed"}; - // Strip padding from content - std::span unpadded_content = unpad_message(content_plaintext); - content_plaintext.resize(unpadded_content.size()); - result.content_plaintext = std::move(content_plaintext); - - std::memcpy( - result.sender_ed25519_pubkey.data(), - sender_ed25519_pubkey.data(), - result.sender_ed25519_pubkey.size()); + std::memcpy( + proof.revocation_tag.data(), + proto_proof.revocationtag().data(), + proto_proof.revocationtag().size()); + std::memcpy( + proof.rotating_pubkey.data(), + proto_proof.rotatingpublickey().data(), + proto_proof.rotatingpublickey().size()); + proof.expiry_at = session::as_sys_seconds(proto_proof.expiryunixts()); + std::memcpy(proof.sig.data(), proto_proof.sig().data(), proto_proof.sig().size()); + return pro; +} - if (crypto_sign_ed25519_pk_to_curve25519( - result.sender_x25519_pubkey.data(), result.sender_ed25519_pubkey.data()) != 0) - throw std::runtime_error( - "Parse content failed, ed25519 public key could not be converted to x25519 " - "key."); - } +// Shared helper 2: parses Content protobuf from result.content_plaintext (which must already be +// set) and extracts pro metadata/verification. +static void parse_content_and_pro( + DecodedEnvelope& result, + const SessionProtos::Envelope& envelope, + std::span pro_backend_pubkey) { // TODO: We parse the content in libsession to extract pro metadata but we return the unparsed // blob back to the caller. This is temporary, eventually we will return a proxy structure for @@ -828,9 +511,9 @@ DecodedEnvelope decode_envelope( if (envelope.has_prosig()) { // Copy (maybe dummy) pro signature into our result struct const std::string& pro_sig = envelope.prosig(); - if (pro_sig.size() != crypto_sign_ed25519_BYTES) + if (pro_sig.size() != 64) throw std::runtime_error("Parse envelope failed, pro signature has wrong size"); - static_assert(sizeof(result.envelope.pro_sig) == crypto_sign_ed25519_BYTES); + static_assert(sizeof(result.envelope.pro_sig) == 64); std::memcpy(result.envelope.pro_sig.data(), pro_sig.data(), pro_sig.size()); if (content.has_promessage()) { @@ -842,72 +525,113 @@ DecodedEnvelope decode_envelope( // Mark the envelope as having a pro signature that the caller can use. result.envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_PRO_SIG; - DecodedPro& pro = result.pro.emplace(); - - // Extract the pro message - const SessionProtos::ProMessage& pro_msg = content.promessage(); - session::ProProof& proof = pro.proof; - pro.msg_bitset.data = pro_msg.msgbitset(); - pro.profile_bitset.data = pro_msg.profilebitset(); - std::memcpy(result.envelope.pro_sig.data(), pro_sig.data(), pro_sig.size()); - - // No proof to read: either the sender attached none, or it is in a format we don't - // know -- a new proof format is a new field/message rather than a version bump on this - // one, so to this client it simply isn't here. Nothing to evaluate, so the proof is - // flagged Invalid and the message is delivered as non-pro rather than dropped: a future - // proof format costs the sender their Pro affordances here, not the whole message. - if (!pro_msg.has_proof()) { - pro.status = ProStatus::Invalid; - } else { - // Parse the proof from protobufs - const SessionProtos::ProProof& proto_proof = pro_msg.proof(); - // A proof that is present but the wrong shape is corruption, not forward-compat: - // hard error. - size_t proof_errors = 0; - proof_errors += - !proto_proof.has_revocationtag() || - proto_proof.revocationtag().size() != proof.revocation_tag.max_size(); - proof_errors += - !proto_proof.has_rotatingpublickey() || - proto_proof.rotatingpublickey().size() != proof.rotating_pubkey.max_size(); - proof_errors += !proto_proof.has_expiryunixts(); - proof_errors += - !proto_proof.has_sig() || proto_proof.sig().size() != proof.sig.max_size(); - if (proof_errors) - throw std::runtime_error( - "Parse decrypted message failed, pro metadata was malformed"); - - std::memcpy( - proof.revocation_tag.data(), - proto_proof.revocationtag().data(), - proto_proof.revocationtag().size()); - std::memcpy( - proof.rotating_pubkey.data(), - proto_proof.rotatingpublickey().data(), - proto_proof.rotatingpublickey().size()); - proof.expiry_at = as_sys_seconds(proto_proof.expiryunixts()); - std::memcpy(proof.sig.data(), proto_proof.sig().data(), proto_proof.sig().size()); + DecodedPro& pro = result.pro.emplace(parse_pro_message(content.promessage())); + // A proof we couldn't read is already flagged ProStatus::Invalid, with no proof to + // check -- leave it as non-pro. + if (pro.status != ProStatus::Invalid) { // Evaluate the pro status given the extracted components (was it signed, is it // expired, was the message signed validly?) - // // Note that we sign the envelope content wholesale. For 1o1 which are padded to 160 // bytes, this means that we expected the user to have signed the padding as well. auto unix_ts = std::chrono::floor( std::chrono::sys_time( std::chrono::milliseconds(content.sigtimestamp()))); - pro.status = proof.status( - pro_backend_pubkey, unix_ts, to_span(pro_sig), to_span(envelope.content())); + // pro_sig.size() validated == 64 above + pro.status = pro.proof.status( + pro_backend_pubkey, + unix_ts, + to_byte_span<64>(pro_sig.data()), + to_span(envelope.content())); } } } +} + +DecodedEnvelope decode_dm_envelope( + const ed25519::PrivKeySpan& ed25519_privkey, + std::span envelope_payload, + std::span pro_backend_pubkey) { + DecodedEnvelope result = {}; + + // 1-on-1/sync messages are wrapped in a WebSocket message protobuf + WebSocketProtos::WebSocketMessage ws_msg; + if (!ws_msg.ParseFromArray(envelope_payload.data(), envelope_payload.size())) + throw std::runtime_error{fmt::format( + "Parse websocket wrapped envelope from payload failed: {}", + envelope_payload.size())}; + if (!ws_msg.has_request()) + throw std::runtime_error{"Parse websocket wrapped envelope failed, missing request"}; + if (!ws_msg.request().has_body()) + throw std::runtime_error{"Parse websocket wrapped envelope failed, missing request body"}; + + SessionProtos::Envelope envelope = {}; + if (!envelope.ParseFromArray(ws_msg.request().body().data(), ws_msg.request().body().size())) + throw std::runtime_error{"Parse envelope from plaintext failed"}; + + parse_envelope_fields(result, envelope); + + if (!envelope.has_content()) + throw std::runtime_error{"Parse decrypted message failed, missing content"}; + + // The inner content is encrypted with Session protocol (Ed25519 DH) + auto [content_plaintext, sender_ed25519_pubkey] = + session::decrypt_incoming(ed25519_privkey, to_span(envelope.content())); + + // Strip padding from content + auto unpadded = unpad_message(content_plaintext); + content_plaintext.resize(unpadded.size()); + result.content_plaintext = std::move(content_plaintext); + + result.sender_ed25519_pubkey = sender_ed25519_pubkey; + result.sender_x25519_pubkey = ed25519::pk_to_x25519(sender_ed25519_pubkey); + + parse_content_and_pro(result, envelope, pro_backend_pubkey); + return result; +} + +DecodedEnvelope decode_group_envelope( + std::span> group_keys, + std::span group_ed25519_pubkey, + std::span envelope_payload, + std::span pro_backend_pubkey) { + DecodedEnvelope result = {}; + + // Groups v2: the entire envelope payload is encrypted with a group symmetric key + DecryptGroupMessage decrypt = + decrypt_group_message(group_keys, group_ed25519_pubkey, envelope_payload); + + if (decrypt.session_id.size() != 66) + throw std::runtime_error{fmt::format( + "Parse encrypted envelope failed, extracted session ID was wrong size: {}", + decrypt.session_id.size())}; + + assert(decrypt.session_id.starts_with("05")); + oxenc::from_hex( + decrypt.session_id.begin() + 2, + decrypt.session_id.end(), + result.sender_x25519_pubkey.begin()); + + SessionProtos::Envelope envelope = {}; + if (!envelope.ParseFromArray(decrypt.plaintext.data(), decrypt.plaintext.size())) + throw std::runtime_error{"Parse envelope from decrypted group data failed"}; + + parse_envelope_fields(result, envelope); + + if (!envelope.has_content()) + throw std::runtime_error{"Parse decrypted message failed, missing content"}; + + // Group content is plaintext (the envelope itself was the encrypted layer) + result.content_plaintext = to_vector(envelope.content()); + + parse_content_and_pro(result, envelope, pro_backend_pubkey); return result; } DecodedCommunityMessage decode_for_community( - std::span content_or_envelope_payload, - sys_seconds unix_ts, - const array_uc32& pro_backend_pubkey) { + std::span content_or_envelope_payload, + std::chrono::sys_seconds unix_ts, + std::span pro_backend_pubkey) { // TODO: Community message parsing requires a custom code path for now as we are planning to // migrate from sending plain `Content` to `Content` with a pro signature embedded in `Content` // (added exclusively for communities usecase), then, transitioning to sending an `Envelope` to @@ -924,7 +648,7 @@ DecodedCommunityMessage decode_for_community( DecodedCommunityMessage result = {}; // Attempt to parse the blob as an envelope - std::optional> pro_sig; + std::optional> pro_sig; SessionProtos::Envelope pb_envelope = {}; { bool envelope_parsed = pb_envelope.ParseFromArray( @@ -933,8 +657,7 @@ DecodedCommunityMessage decode_for_community( if (envelope_parsed) { // Create the envelope Envelope& envelope = result.envelope.emplace(); - result.content_plaintext = std::vector( - pb_envelope.content().begin(), pb_envelope.content().end()); + result.content_plaintext = to_vector(pb_envelope.content()); // Extract the envelope into our type // Parse source (optional) @@ -950,17 +673,7 @@ DecodedCommunityMessage decode_for_community( envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SOURCE; } - // Parse source device (optional) - if (pb_envelope.has_sourcedevice()) { - envelope.source_device = pb_envelope.sourcedevice(); - envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SOURCE_DEVICE; - } - - // Parse server timestamp (optional) - if (pb_envelope.has_servertimestamp()) { - envelope.server_timestamp = pb_envelope.servertimestamp(); - envelope.flags |= SESSION_PROTOCOL_ENVELOPE_FLAGS_SERVER_TIMESTAMP; - } + parse_common_envelope_fields(envelope, pb_envelope); // Parse pro signature (optional) if (pb_envelope.has_prosig()) { @@ -969,13 +682,13 @@ DecodedCommunityMessage decode_for_community( } } else { // TODO: Do wasteful copy in the interim whilst transitioning protocol - result.content_plaintext = std::vector( + result.content_plaintext = std::vector( content_or_envelope_payload.begin(), content_or_envelope_payload.end()); } } // Parse the content blob - std::span unpadded_content = unpad_message(result.content_plaintext); + std::span unpadded_content = unpad_message(result.content_plaintext); SessionProtos::Content content = {}; if (!content.ParseFromArray(unpadded_content.data(), unpadded_content.size())) throw std::runtime_error{ @@ -997,7 +710,7 @@ DecodedCommunityMessage decode_for_community( // If there was a pro signature in one of the payloads, verify and copy it to our result struct if (pro_sig) { - if (pro_sig->size() != crypto_sign_ed25519_BYTES) + if (pro_sig->size() != 64) throw std::runtime_error( "Decoding community message failed, pro signature has wrong size"); @@ -1012,49 +725,11 @@ DecodedCommunityMessage decode_for_community( } if (result.pro_sig && content.has_promessage()) { - // Extract the pro message - DecodedPro& pro = result.pro.emplace(); - const SessionProtos::ProMessage& pro_msg = content.promessage(); - session::ProProof& proof = pro.proof; - pro.msg_bitset.data = pro_msg.msgbitset(); - pro.profile_bitset.data = pro_msg.profilebitset(); - - // No proof to read: either the sender attached none, or it is in a format we don't know -- - // a new proof format is a new field/message rather than a version bump on this one, so to - // this client it simply isn't here. Nothing to evaluate, so the proof is flagged Invalid - // and the message is delivered as non-pro rather than dropped: a future proof format costs - // the sender their Pro affordances here, not the whole message. - if (!pro_msg.has_proof()) { - pro.status = ProStatus::Invalid; - } else { - // Parse the proof from protobufs - const SessionProtos::ProProof& proto_proof = pro_msg.proof(); - // A proof that is present but the wrong shape is corruption, not forward-compat: hard - // error. - size_t proof_errors = 0; - proof_errors += !proto_proof.has_revocationtag() || - proto_proof.revocationtag().size() != proof.revocation_tag.max_size(); - proof_errors += - !proto_proof.has_rotatingpublickey() || - proto_proof.rotatingpublickey().size() != proof.rotating_pubkey.max_size(); - proof_errors += !proto_proof.has_expiryunixts(); - proof_errors += - !proto_proof.has_sig() || proto_proof.sig().size() != proof.sig.max_size(); - if (proof_errors) - throw std::runtime_error( - "Decoding community message failed, pro metadata was malformed"); - - std::memcpy( - proof.revocation_tag.data(), - proto_proof.revocationtag().data(), - proto_proof.revocationtag().size()); - std::memcpy( - proof.rotating_pubkey.data(), - proto_proof.rotatingpublickey().data(), - proto_proof.rotatingpublickey().size()); - proof.expiry_at = as_sys_seconds(proto_proof.expiryunixts()); - std::memcpy(proof.sig.data(), proto_proof.sig().data(), proto_proof.sig().size()); + DecodedPro& pro = result.pro.emplace(parse_pro_message(content.promessage())); + // A proof we couldn't read is already flagged ProStatus::Invalid, with no proof to + // check -- leave it as non-pro. + if (pro.status != ProStatus::Invalid) { // Evaluate the pro status given the extracted components (was it signed, is it expired, // was the message signed validly?) // @@ -1066,11 +741,8 @@ DecodedCommunityMessage decode_for_community( // Entering the `pro_sig` and `result.envelope` branch means that the envelope must // have a pro signature. assert(result.envelope->flags & SESSION_PROTOCOL_ENVELOPE_FLAGS_PRO_SIG); - pro.status = proof.status( - pro_backend_pubkey, - unix_ts, - to_span(*result.pro_sig), - result.content_plaintext); + pro.status = pro.proof.status( + pro_backend_pubkey, unix_ts, *result.pro_sig, result.content_plaintext); } else { SessionProtos::Content content_copy_without_sig = content; assert(content_copy_without_sig.has_prosigforcommunitymessageonly()); @@ -1080,13 +752,13 @@ DecodedCommunityMessage decode_for_community( assert(!content_copy_without_sig.has_prosigforcommunitymessageonly()); // Reserialise the payload without the signature, repad it then verify the signature - std::vector content_copy_without_sig_payload = + std::vector content_copy_without_sig_payload = pad_message(to_span(content_copy_without_sig.SerializeAsString())); - pro.status = proof.status( + pro.status = pro.proof.status( pro_backend_pubkey, unix_ts, - to_span(*result.pro_sig), + *result.pro_sig, to_span(content_copy_without_sig_payload)); } } @@ -1102,6 +774,7 @@ DecodedCommunityMessage decode_for_community( return result; } + } // namespace session using namespace session; @@ -1119,69 +792,37 @@ LIBSESSION_EXPORT extern const int SESSION_PROTOCOL_COMMUNITY_OR_1O1_MSG_PADDING } static_assert(sizeof(session_protocol_pro_proof::revocation_tag) == 32); -static_assert( - sizeof(session_protocol_pro_proof::rotating_pubkey) == crypto_sign_ed25519_PUBLICKEYBYTES); -static_assert(sizeof(session_protocol_pro_proof::sig) == crypto_sign_ed25519_BYTES); - -static_assert( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_COUNT <= - sizeof(session_protocol_pro_profile_bitset::data) * 8 /*bits per byte*/, - "There are more feature flags than is available in the bitset, the bitset needs to be " - "upgraded into an array of bytes"); - -LIBSESSION_C_API bool session_protocol_pro_profile_bitset_is_set( - session_protocol_pro_profile_bitset value, SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) { - bool result = value.data & (1ULL << features); - return result; -} - -LIBSESSION_C_API void session_protocol_pro_profile_bitset_set( - session_protocol_pro_profile_bitset* value, - SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) { - value->data |= (1ULL << features); -} - -LIBSESSION_C_API void session_protocol_pro_profile_bitset_unset( - session_protocol_pro_profile_bitset* value, - SESSION_PROTOCOL_PRO_PROFILE_FEATURES features) { - value->data &= ~(1ULL << features); -} - -LIBSESSION_C_API bool session_protocol_pro_message_bitset_is_set( - session_protocol_pro_message_bitset value, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) { - bool result = value.data & (1ULL << features); - return result; -} - -LIBSESSION_C_API void session_protocol_pro_message_bitset_set( - session_protocol_pro_message_bitset* value, - SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) { - value->data |= (1ULL << features); -} - -LIBSESSION_C_API void session_protocol_pro_message_bitset_unset( - session_protocol_pro_message_bitset* value, - SESSION_PROTOCOL_PRO_MESSAGE_FEATURES features) { - value->data &= ~(1ULL << features); -} +static_assert(sizeof(session_protocol_pro_proof::rotating_pubkey) == 32); +static_assert(sizeof(session_protocol_pro_proof::sig) == 64); + +// Session Pro feature flag bit constants exposed to the C API. The C++ enum classes +// (session::ProProfileFlags / session::ProMessageFlags) are the source of truth; these mirror their +// underlying values so C callers can OR/test them against a plain uint64_t bitset. +const uint64_t SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE = + static_cast(ProProfileFlags::ProBadge); +const uint64_t SESSION_PROTOCOL_PRO_PROFILE_FEATURE_ANIMATED_AVATAR = + static_cast(ProProfileFlags::AnimatedAvatar); +const uint64_t SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT = + static_cast(ProMessageFlags::CharLimit10k); LIBSESSION_C_API bool session_protocol_pro_proof_verify_signature( session_protocol_pro_proof const* proof, uint8_t const* verify_pubkey, size_t verify_pubkey_len) { - if (verify_pubkey_len != crypto_sign_ed25519_PUBLICKEYBYTES) + if (verify_pubkey_len != 32) return false; - auto verify_pubkey_span = std::span(verify_pubkey, verify_pubkey_len); auto msg = proof_signed_message( - proof->revocation_tag.data, proof->rotating_pubkey.data, proof->expiry_ts); - bool result = proof_verify_message_internal(verify_pubkey_span, proof->sig.data, msg); - return result; + to_byte_span(proof->revocation_tag.data), + to_byte_span(proof->rotating_pubkey.data), + proof->expiry_ts); + return ed25519::verify(to_byte_span(proof->sig.data), to_byte_span<32>(verify_pubkey), msg); } LIBSESSION_C_API void session_protocol_pro_rotating_seed( const unsigned char* master_seed, int64_t now_unix_ts, unsigned char* rotating_seed_out) { auto seed = session::ProProof::rotating_seed( - {master_seed, 32}, std::chrono::sys_seconds{std::chrono::seconds{now_unix_ts}}); + to_byte_span(master_seed, 32), + std::chrono::sys_seconds{std::chrono::seconds{now_unix_ts}}); std::memcpy(rotating_seed_out, seed.data(), seed.size()); } @@ -1191,10 +832,12 @@ LIBSESSION_C_API bool session_protocol_pro_proof_verify_message( size_t sig_len, uint8_t const* msg, size_t msg_len) { - std::span sig_span = {sig, sig_len}; - std::span msg_span = {msg, msg_len}; - bool result = proof_verify_message_internal(proof->rotating_pubkey.data, sig_span, msg_span); - return result; + if (sig_len != 64) + return false; + return ed25519::verify( + to_byte_span<64>(sig), + to_byte_span(proof->rotating_pubkey.data), + to_byte_span(msg, msg_len)); } LIBSESSION_C_API bool session_protocol_pro_proof_is_active( @@ -1208,69 +851,86 @@ LIBSESSION_C_API SESSION_PROTOCOL_PRO_STATUS session_protocol_pro_proof_status( size_t verify_pubkey_len, int64_t ts, const session_protocol_pro_signed_message* signed_msg) { - SESSION_PROTOCOL_PRO_STATUS result = SESSION_PROTOCOL_PRO_STATUS_VALID; - if (!session_protocol_pro_proof_verify_signature(proof, verify_pubkey, verify_pubkey_len)) - result = SESSION_PROTOCOL_PRO_STATUS_INVALID_PRO_BACKEND_SIG; - - // Check if the message was signed if the user passed one in to verify against - if (result == SESSION_PROTOCOL_PRO_STATUS_VALID && signed_msg) { - if (!session_protocol_pro_proof_verify_message( - proof, - signed_msg->sig.data, - signed_msg->sig.size, - signed_msg->msg.data, - signed_msg->msg.size)) - result = SESSION_PROTOCOL_PRO_STATUS_INVALID_USER_SIG; + // ProProof::status is the single source of truth for the backend-sig -> user-sig -> expiry + // evaluation. The C API additionally validates the caller's buffer lengths (which the C++ API + // encodes as fixed-size spans), so handle those here and delegate the rest. + if (verify_pubkey_len != 32) + return SESSION_PROTOCOL_PRO_STATUS_INVALID_PRO_BACKEND_SIG; + + std::optional> user_sig; + std::span user_msg; + bool bad_user_sig = false; + if (signed_msg) { + if (signed_msg->sig.size == 64) { + user_sig = to_byte_span<64>(signed_msg->sig.data); + user_msg = to_byte_span(signed_msg->msg.data, signed_msg->msg.size); + } else + bad_user_sig = true; // a wrong-length signature can never verify } - // Check if the proof has expired - if (result == SESSION_PROTOCOL_PRO_STATUS_VALID && - !session_protocol_pro_proof_is_active(proof, ts)) - result = SESSION_PROTOCOL_PRO_STATUS_EXPIRED; - return result; + ProStatus status = proof_from_c(*proof).status( + to_byte_span<32>(verify_pubkey), as_sys_seconds(ts), user_sig, user_msg); + + // ProProof::status can't see a wrong-length signature (it takes a fixed-size span), so surface + // the C API's length check here while keeping the ordering: a bad user signature supersedes a + // valid or expired result, but not a failed backend signature. + if (bad_user_sig && status != ProStatus::InvalidProBackendSig) + status = ProStatus::InvalidUserSig; + + return static_cast(status); } LIBSESSION_C_API session_protocol_pro_features_for_msg session_protocol_pro_features_for_message( size_t codepoint_count) { - ProFeaturesForMsg result_cpp = pro_features_for_message(codepoint_count); - session_protocol_pro_features_for_msg result = { + auto result_cpp = pro_features_for_message(codepoint_count); + return session_protocol_pro_features_for_msg{ .status = static_cast(result_cpp.status), .error = result_cpp.error.data(), - .bitset = {result_cpp.bitset.data}, + .bitset = static_cast(result_cpp.flags), }; +} + +// Shared try/catch wrapper for all C encode functions. +template +static session_protocol_encoded_for_destination c_encode_impl( + char* error, size_t error_len, Fn&& fn) { + session_protocol_encoded_for_destination result = {}; + try { + auto ciphertext = fn(); + result = { + .success = true, + .ciphertext = span_u8_copy_or_throw(ciphertext.data(), ciphertext.size()), + }; + } catch (const std::exception& e) { + result.error_len_incl_null_terminator = copy_c_str(error, error_len, e.what()); + } return result; } LIBSESSION_C_API -session_protocol_encoded_for_destination session_protocol_encode_for_1o1( +session_protocol_encoded_for_destination session_protocol_encode_dm_v1( const void* plaintext, size_t plaintext_len, const void* ed25519_privkey, size_t ed25519_privkey_len, uint64_t sent_timestamp_ms, - const bytes33* recipient_pubkey, + const cbytes33* recipient_pubkey, const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, char* error, size_t error_len) { - - session_protocol_destination dest = {}; - dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_SYNC_OR_1O1; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey; - dest.pro_rotating_ed25519_privkey_len = pro_rotating_ed25519_privkey_len; - dest.recipient_pubkey = *recipient_pubkey; - dest.sent_timestamp_ms = sent_timestamp_ms; - - session_protocol_encoded_for_destination result = session_protocol_encode_for_destination( - plaintext, - plaintext_len, - ed25519_privkey, - ed25519_privkey_len, - &dest, - error, - error_len); - return result; + return c_encode_impl(error, error_len, [&] { + return encode_dm_v1( + std::span{static_cast(plaintext), plaintext_len}, + ed25519::PrivKeySpan{ + static_cast(ed25519_privkey), ed25519_privkey_len}, + from_epoch_ms(sent_timestamp_ms), + to_byte_span(recipient_pubkey->data), + ed25519::OptionalPrivKeySpan{ + static_cast(pro_rotating_ed25519_privkey), + pro_rotating_ed25519_privkey_len}); + }); } LIBSESSION_C_API @@ -1279,31 +939,23 @@ session_protocol_encoded_for_destination session_protocol_encode_for_community_i size_t plaintext_len, const void* ed25519_privkey, size_t ed25519_privkey_len, - const bytes33* recipient_pubkey, - const bytes32* community_pubkey, + const cbytes33* recipient_pubkey, + const cbytes32* community_pubkey, const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, char* error, size_t error_len) { - - session_protocol_destination dest = {}; - dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey; - dest.pro_rotating_ed25519_privkey_len = pro_rotating_ed25519_privkey_len; - // Unused on the CommunityInbox path (only the envelope-based Group/1o1 path consumes it). - dest.sent_timestamp_ms = 0; - dest.recipient_pubkey = *recipient_pubkey; - dest.community_inbox_server_pubkey = *community_pubkey; - - session_protocol_encoded_for_destination result = session_protocol_encode_for_destination( - plaintext, - plaintext_len, - ed25519_privkey, - ed25519_privkey_len, - &dest, - error, - error_len); - return result; + return c_encode_impl(error, error_len, [&] { + return encode_for_community_inbox( + std::span{static_cast(plaintext), plaintext_len}, + ed25519::PrivKeySpan{ + static_cast(ed25519_privkey), ed25519_privkey_len}, + to_byte_span(recipient_pubkey->data), + to_byte_span(community_pubkey->data), + ed25519::OptionalPrivKeySpan{ + static_cast(pro_rotating_ed25519_privkey), + pro_rotating_ed25519_privkey_len}); + }); } LIBSESSION_C_API @@ -1314,15 +966,13 @@ session_protocol_encoded_for_destination session_protocol_encode_for_community( size_t pro_rotating_ed25519_privkey_len, char* error, size_t error_len) { - - session_protocol_destination dest = {}; - dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey; - dest.pro_rotating_ed25519_privkey_len = pro_rotating_ed25519_privkey_len; - - session_protocol_encoded_for_destination result = session_protocol_encode_for_destination( - plaintext, plaintext_len, nullptr, 0, &dest, error, error_len); - return result; + return c_encode_impl(error, error_len, [&] { + return encode_for_community( + std::span{static_cast(plaintext), plaintext_len}, + ed25519::OptionalPrivKeySpan{ + static_cast(pro_rotating_ed25519_privkey), + pro_rotating_ed25519_privkey_len}); + }); } LIBSESSION_C_API @@ -1332,80 +982,24 @@ session_protocol_encoded_for_destination session_protocol_encode_for_group( const void* ed25519_privkey, size_t ed25519_privkey_len, uint64_t sent_timestamp_ms, - const bytes33* group_ed25519_pubkey, - const bytes32* group_enc_key, + const cbytes33* group_ed25519_pubkey, + const cbytes32* group_enc_key, const void* pro_rotating_ed25519_privkey, size_t pro_rotating_ed25519_privkey_len, char* error, size_t error_len) { - - session_protocol_destination dest = {}; - dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_GROUP; - dest.pro_rotating_ed25519_privkey = pro_rotating_ed25519_privkey; - dest.pro_rotating_ed25519_privkey_len = pro_rotating_ed25519_privkey_len; - dest.group_ed25519_pubkey = *group_ed25519_pubkey; - dest.group_enc_key = *group_enc_key; - dest.sent_timestamp_ms = sent_timestamp_ms; - - session_protocol_encoded_for_destination result = session_protocol_encode_for_destination( - plaintext, - plaintext_len, - ed25519_privkey, - ed25519_privkey_len, - &dest, - error, - error_len); - return result; -} - -LIBSESSION_C_API session_protocol_encoded_for_destination session_protocol_encode_for_destination( - const void* plaintext, - size_t plaintext_len, - const void* ed25519_privkey, - size_t ed25519_privkey_len, - const session_protocol_destination* dest, - char* error, - size_t error_len) { - - session_protocol_encoded_for_destination result = {}; - - try { - std::span dest_pro_rotating_ed25519_privkey = std::span( - reinterpret_cast(dest->pro_rotating_ed25519_privkey), - reinterpret_cast(dest->pro_rotating_ed25519_privkey) + - dest->pro_rotating_ed25519_privkey_len); - - EncryptedForDestinationInternal result_internal = encode_for_destination_internal( - /*plaintext=*/{static_cast(plaintext), plaintext_len}, - /*ed25519_privkey=*/ - {static_cast(ed25519_privkey), ed25519_privkey_len}, - /*dest_type=*/static_cast(dest->type), - /*dest_pro_rotating_ed25519_privkey=*/dest_pro_rotating_ed25519_privkey, - /*dest_recipient_pubkey=*/dest->recipient_pubkey.data, - /*dest_sent_timestamp_ms=*/ - std::chrono::milliseconds(dest->sent_timestamp_ms), - /*dest_community_inbox_server_pubkey=*/ - dest->community_inbox_server_pubkey.data, - /*dest_group_ed25519_pubkey=*/dest->group_ed25519_pubkey.data, - /*dest_group_enc_key=*/dest->group_enc_key.data, - /*use_malloc=*/UseMalloc::Yes); - - result = { - .success = true, - .ciphertext = result_internal.ciphertext_c, - }; - } catch (const std::exception& e) { - std::string error_cpp = e.what(); - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "%.*s", - static_cast(error_cpp.size()), - error_cpp.data()) + - 1; - } - - return result; + return c_encode_impl(error, error_len, [&] { + return encode_for_group( + std::span{static_cast(plaintext), plaintext_len}, + ed25519::PrivKeySpan{ + static_cast(ed25519_privkey), ed25519_privkey_len}, + std::chrono::milliseconds(sent_timestamp_ms), + to_byte_span(group_ed25519_pubkey->data), + to_byte_span(group_enc_key->data), + ed25519::OptionalPrivKeySpan{ + static_cast(pro_rotating_ed25519_privkey), + pro_rotating_ed25519_privkey_len}); + }); } LIBSESSION_C_API void session_protocol_encode_for_destination_free( @@ -1430,50 +1024,65 @@ session_protocol_decoded_envelope session_protocol_decode_envelope( // Setup the pro backend pubkey auto pro_backend_pubkey_cpp = maybe_uc32_from_ptr(pro_backend_pubkey, pro_backend_pubkey_len); if (!pro_backend_pubkey_cpp) { - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "Invalid pro_backend_pubkey: Key was " - "set but was not 32 bytes, was: %zu", - pro_backend_pubkey_len) + - 1; + result.error_len_incl_null_terminator = format_c_str( + error, + error_len, + "Invalid pro_backend_pubkey: Key was set but was not 32 bytes, was: {}", + pro_backend_pubkey_len); return result; } - // Setup decryption keys and decrypt - DecodeEnvelopeKey keys_cpp = {}; - if (keys->group_ed25519_pubkey.size) { - keys_cpp.group_ed25519_pubkey = std::span( - keys->group_ed25519_pubkey.data, keys->group_ed25519_pubkey.size); - } + std::span payload{ + static_cast(envelope_plaintext), envelope_plaintext_len}; DecodedEnvelope result_cpp = {}; - for (size_t index = 0; index < keys->decrypt_keys_len; index++) { - std::span key = { - keys->decrypt_keys[index].data, keys->decrypt_keys[index].size}; - keys_cpp.decrypt_keys = {&key, 1}; + if (keys->group_ed25519_pubkey.size == 32) { + // Groups v2 path: decrypt with group symmetric keys + auto group_pk = to_byte_span<32>(keys->group_ed25519_pubkey.data); + + std::vector> group_keys; + group_keys.reserve(keys->decrypt_keys_len); + for (size_t i = 0; i < keys->decrypt_keys_len; i++) { + if (keys->decrypt_keys[i].size != 32) + throw std::invalid_argument{fmt::format( + "Invalid group encryption key: expected 32 bytes, got {}", + keys->decrypt_keys[i].size)}; + group_keys.emplace_back(to_byte_span<32>(keys->decrypt_keys[i].data)); + } + try { - result_cpp = decode_envelope( - keys_cpp, - {static_cast(envelope_plaintext), envelope_plaintext_len}, - *pro_backend_pubkey_cpp); + result_cpp = + decode_group_envelope(group_keys, group_pk, payload, *pro_backend_pubkey_cpp); result.success = true; - break; } catch (const std::exception& e) { - std::string error_cpp = e.what(); - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "%.*s", - static_cast(error_cpp.size()), - error_cpp.data()) + - 1; + result.error_len_incl_null_terminator = format_c_str(error, error_len, "{}", e.what()); + } + } else if (keys->group_ed25519_pubkey.size) { + result.error_len_incl_null_terminator = format_c_str( + error, + error_len, + "Invalid group_ed25519_pubkey: must be exactly 32 bytes, was: {}", + keys->group_ed25519_pubkey.size); + return result; + } else { + // DM path: decrypt with Ed25519 private key(s) + for (size_t index = 0; index < keys->decrypt_keys_len; index++) { + try { + ed25519::PrivKeySpan privkey{ + keys->decrypt_keys[index].data, keys->decrypt_keys[index].size}; + result_cpp = decode_dm_envelope(privkey, payload, *pro_backend_pubkey_cpp); + result.success = true; + break; + } catch (const std::exception& e) { + result.error_len_incl_null_terminator = + format_c_str(error, error_len, "{}", e.what()); + } } - } - if (keys->decrypt_keys_len == 0) { - result.error_len_incl_null_terminator = - snprintf_clamped(error, error_len, "No keys ed25519_privkeys were provided") + 1; + if (keys->decrypt_keys_len == 0) { + result.error_len_incl_null_terminator = + format_c_str(error, error_len, "No ed25519 private keys were provided"); + } } // Marshall into c type @@ -1481,15 +1090,8 @@ session_protocol_decoded_envelope session_protocol_decode_envelope( result.content_plaintext = session::span_u8_copy_or_throw( result_cpp.content_plaintext.data(), result_cpp.content_plaintext.size()); } catch (const std::exception& e) { - std::string error_cpp = e.what(); result.success = false; - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "%.*s", - static_cast(error_cpp.size()), - error_cpp.data()) + - 1; + result.error_len_incl_null_terminator = copy_c_str(error, error_len, e.what()); } result.envelope = envelope_from_cpp(result_cpp.envelope); @@ -1533,19 +1135,17 @@ session_protocol_decoded_community_message session_protocol_decode_for_community OPTIONAL char* error, size_t error_len) { session_protocol_decoded_community_message result = {}; - auto content_or_envelope_payload_span = std::span( - reinterpret_cast(content_or_envelope_payload), - content_or_envelope_payload_len); - auto unix_ts = as_sys_seconds(ts); + std::span content_or_envelope_payload_span{ + static_cast(content_or_envelope_payload), + content_or_envelope_payload_len}; + auto unix_ts = session::as_sys_seconds(ts); auto pro_backend_pubkey_cpp = maybe_uc32_from_ptr(pro_backend_pubkey, pro_backend_pubkey_len); if (!pro_backend_pubkey_cpp) { - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "Invalid pro_backend_pubkey: Key was " - "set but was not 32 bytes, was: %zu", - pro_backend_pubkey_len) + - 1; + result.error_len_incl_null_terminator = format_c_str( + error, + error_len, + "Invalid pro_backend_pubkey: Key was set but was not 32 bytes, was: {}", + pro_backend_pubkey_len); return result; } @@ -1564,15 +1164,8 @@ session_protocol_decoded_community_message session_protocol_decode_for_community result.pro = decoded_pro_from_cpp(*decoded.pro); result.success = true; } catch (const std::exception& e) { - std::string error_cpp = e.what(); result.success = false; - result.error_len_incl_null_terminator = snprintf_clamped( - error, - error_len, - "%.*s", - static_cast(error_cpp.size()), - error_cpp.data()) + - 1; + result.error_len_incl_null_terminator = copy_c_str(error, error_len, e.what()); } return result; diff --git a/src/types.cpp b/src/types.cpp index 230dee60d..9da1d21a9 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1,14 +1,13 @@ #include #include -#include #include namespace session { span_u8 span_u8_alloc_or_throw(size_t size) { span_u8 result = {}; result.size = size; - result.data = static_cast(malloc(size)); + result.data = static_cast(malloc(size)); if (!result.data) throw std::runtime_error( fmt::format("Failed to allocate {} bytes for span, out of memory", size)); @@ -20,17 +19,4 @@ span_u8 span_u8_copy_or_throw(const void* data, size_t size) { std::memcpy(result.data, data, result.size); return result; } - }; // namespace session - -int snprintf_clamped(char* buffer, size_t size, char const* fmt, ...) { - va_list args; - va_start(args, fmt); - int bytes_required_not_incl_null = vsnprintf(buffer, size, fmt, args); - va_end(args); - - int result = bytes_required_not_incl_null; - if (buffer && size && bytes_required_not_incl_null >= (size - 1)) - result = size - 1; - return result; -} diff --git a/src/util.cpp b/src/util.cpp index a5e548193..9cd33f2e3 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,10 +1,12 @@ #include +#include #include #include #include #include #include +#include #include #include @@ -39,6 +41,19 @@ std::vector split(std::string_view str, const std::string_view return results; } +std::string format_as(human_size s) { + if (s.bytes < 1000) + return fmt::format("{} B", s.bytes); + constexpr std::array prefixes = {'k', 'M', 'G', 'T'}; + double b = s.bytes; + for (auto prefix : prefixes) { + b /= 1000.; + if (b < 1000.) + return fmt::format("{:.{}f} {}B", b, b < 10. ? 2 : b < 100. ? 1 : 0, prefix); + } + return fmt::format("{:.0f} {}B", b, prefixes.back()); +} + std::tuple, std::optional> parse_url( std::string_view url) { std::tuple, std::optional> @@ -112,9 +127,9 @@ namespace { using zstd_decomp_ptr = std::unique_ptr; } // namespace -std::vector zstd_compress( - std::span data, int level, std::span prefix) { - std::vector compressed; +std::vector zstd_compress( + std::span data, int level, std::span prefix) { + std::vector compressed; if (prefix.empty()) compressed.resize(ZSTD_compressBound(data.size())); else { @@ -128,14 +143,14 @@ std::vector zstd_compress( data.size(), level); if (ZSTD_isError(size)) - throw std::runtime_error{"Compression failed: " + std::string{ZSTD_getErrorName(size)}}; + throw std::runtime_error{"Compression failed: {}"_format(ZSTD_getErrorName(size))}; compressed.resize(prefix.size() + size); return compressed; } -std::optional> zstd_decompress( - std::span data, size_t max_size) { +std::optional> zstd_decompress( + std::span data, size_t max_size) { zstd_decomp_ptr z_decompressor{ZSTD_createDStream()}; auto* zds = z_decompressor.get(); @@ -144,7 +159,7 @@ std::optional> zstd_decompress( std::array out_buf; ZSTD_outBuffer output{/*.dst=*/out_buf.data(), /*.size=*/out_buf.size(), /*.pos=*/0}; - std::vector decompressed; + std::vector decompressed; size_t ret; do { @@ -155,7 +170,10 @@ std::optional> zstd_decompress( if (max_size > 0 && decompressed.size() + output.pos > max_size) return std::nullopt; - decompressed.insert(decompressed.end(), out_buf.begin(), out_buf.begin() + output.pos); + decompressed.insert( + decompressed.end(), + reinterpret_cast(out_buf.data()), + reinterpret_cast(out_buf.data()) + output.pos); } while (ret > 0 || input.pos < input.size); return decompressed; diff --git a/src/xed25519-tweetnacl.cpp b/src/xed25519-tweetnacl.cpp new file mode 100644 index 000000000..cd1157464 --- /dev/null +++ b/src/xed25519-tweetnacl.cpp @@ -0,0 +1,150 @@ +// This file contains a subset of TweetNaCl (https://tweetnacl.cr.yp.to/software.html) public domain +// code to perform the X25519 -> Ed25519 conversion; libsodium doesn't provide enough access to +// internals to compute this without hacking up libsodium's build, which is fragile. Hence we use +// this subset of the portable TweetNaCl for that single function, and libsodium for everything +// else. + +#include +#include +#include + +#include "session/xed25519.hpp" + +namespace session::xed25519 { + +namespace { + + // clang-format off + +#define FOR(i,n) for (i = 0;i < n;++i) + +using gf = int64_t[16]; + +const gf gf1 = {1}; + +void car25519(gf o) +{ + int i; + int64_t c; + FOR(i,16) { + o[i]+=(1LL<<16); + c=o[i]>>16; + o[(i+1)*(i<15)]+=c-1+37*(c-1)*(i==15); + o[i]-=c<<16; + } +} + +void sel25519(gf p,gf q,int b) +{ + int64_t t,i,c=~(b-1); + FOR(i,16) { + t= c&(p[i]^q[i]); + p[i]^=t; + q[i]^=t; + } +} + +void pack25519(uint8_t *o,const gf n) +{ + int i,j,b; + gf m,t; + FOR(i,16) t[i]=n[i]; + car25519(t); + car25519(t); + car25519(t); + FOR(j,2) { + m[0]=t[0]-0xffed; + for(i=1;i<15;i++) { + m[i]=t[i]-0xffff-((m[i-1]>>16)&1); + m[i-1]&=0xffff; + } + m[15]=t[15]-0x7fff-((m[14]>>16)&1); + b=(m[15]>>16)&1; + m[14]&=0xffff; + sel25519(t,m,1-b); + } + FOR(i,16) { + o[2*i]=t[i]&0xff; + o[2*i+1]=t[i]>>8; + } +} + +void unpack25519(gf o, const uint8_t *n) +{ + int i; + FOR(i,16) o[i]=n[2*i]+((int64_t)n[2*i+1]<<8); + o[15]&=0x7fff; +} + +void A(gf o,const gf a,const gf b) +{ + int i; + FOR(i,16) o[i]=a[i]+b[i]; +} + +void Z(gf o,const gf a,const gf b) +{ + int i; + FOR(i,16) o[i]=a[i]-b[i]; +} + +void M(gf o,const gf a,const gf b) +{ + int64_t i,j,t[31]; + FOR(i,31) t[i]=0; + FOR(i,16) FOR(j,16) t[i+j]+=a[i]*b[j]; + FOR(i,15) t[i]+=38*t[i+16]; + FOR(i,16) o[i]=t[i]; + car25519(o); + car25519(o); +} + +void S(gf o,const gf a) +{ + M(o,a,a); +} + +// Y +void inv25519(gf o,const gf i) +{ + gf c; + int a; + FOR(a,16) c[a]=i[a]; + for(a=253;a>=0;a--) { + S(c,c); + if(a!=2&&a!=4) M(c,c,i); + } + FOR(a,16) o[a]=c[a]; +} + + // clang-format on + +} // namespace + +std::array pubkey(std::span x_pk) noexcept { + gf u; + unpack25519(u, reinterpret_cast(x_pk.data())); + + // u - 1 + gf u_minus_one; + Z(u_minus_one, u, gf1); + + // Compute: u + 1 + gf u_plus_one; + A(u_plus_one, u, gf1); + + // Compute: (u + 1)^-1 + gf u_plus_one_inv; + inv25519(u_plus_one_inv, u_plus_one); + + // Compute: y = (u - 1) * (u + 1)^-1 + gf y; + M(y, u_minus_one, u_plus_one_inv); + + // Encode to 32 bytes (sign bit is naturally 0) + std::array ed_pk; + pack25519(reinterpret_cast(ed_pk.data()), y); + return ed_pk; +} + +} // namespace session::xed25519 diff --git a/src/xed25519.cpp b/src/xed25519.cpp index ce1e9e5fb..0da260646 100644 --- a/src/xed25519.cpp +++ b/src/xed25519.cpp @@ -1,36 +1,30 @@ #include "session/xed25519.hpp" #include -#include -#include #include #include #include +#include #include #include #include #include "session/export.h" +#include "session/hash.hpp" #include "session/util.hpp" +#include "session/xed25519.h" namespace session::xed25519 { +using namespace session::literals; + +// Internal unsigned char arrays; kept as unsigned char for direct C API use template -using bytes = std::array; +using uchars = std::array; namespace { - void fe25519_montx_to_edy(fe25519 y, const fe25519 u) { - fe25519 one; - crypto_internal_fe25519_1(one); - fe25519 um1, up1; - crypto_internal_fe25519_sub(um1, u, one); - crypto_internal_fe25519_add(up1, u, one); - crypto_internal_fe25519_invert(up1, up1); - crypto_internal_fe25519_mul(y, um1, up1); - } - // We construct an Ed25519-like signature with one important difference: where Ed25519 // calculates `r = H(S || M) mod L` (where S is the second half of the SHA-512 hash of the // secret key) we instead calculate `r = H(a || M || Z) mod L`. @@ -38,24 +32,16 @@ namespace { // This deviates from Signal's XEd25519 specified derivation of r in that we use a personalized // Black2b hash (for better performance and cryptographic properties), rather than a // custom-prefixed SHA-512 hash. - bytes<32> xed25519_compute_r(const bytes<32>& a, std::span msg) { - bytes<64> random; + uchars<32> xed25519_compute_r(const uchars<32>& a, std::span msg) { + uchars<64> random; randombytes_buf(random.data(), random.size()); - constexpr static bytes<16> personality = { - 'x', 'e', 'd', '2', '5', '5', '1', '9', 's', 'i', 'g', 'n', 'a', 't', 'u', 'r'}; + constexpr static auto personality = "xed25519signatur"_b2b_pers; - crypto_generichash_blake2b_state st; - static_assert(personality.size() == crypto_generichash_blake2b_PERSONALBYTES); - crypto_generichash_blake2b_init_salt_personal( - &st, nullptr, 0, 64, nullptr, personality.data()); - crypto_generichash_blake2b_update(&st, a.data(), a.size()); - crypto_generichash_blake2b_update(&st, msg.data(), msg.size()); - crypto_generichash_blake2b_update(&st, random.data(), random.size()); - bytes<64> h_aMZ; - crypto_generichash_blake2b_final(&st, h_aMZ.data(), h_aMZ.size()); + uchars<64> h_aMZ; + hash::blake2b_pers(h_aMZ, personality, a, msg, random); - bytes<32> r; + uchars<32> r; crypto_core_ed25519_scalar_reduce(r.data(), h_aMZ.data()); return r; } @@ -64,45 +50,58 @@ namespace { void ed25519_hram( unsigned char* S, const unsigned char* R, - const bytes<32>& A, - std::span msg) { - bytes<64> hram; + const uchars<32>& A, + std::span msg) { + uchars<64> hram; crypto_hash_sha512_state st; crypto_hash_sha512_init(&st); crypto_hash_sha512_update(&st, R, 32); crypto_hash_sha512_update(&st, A.data(), A.size()); - crypto_hash_sha512_update(&st, msg.data(), msg.size()); + crypto_hash_sha512_update(&st, to_unsigned(msg.data()), msg.size()); crypto_hash_sha512_final(&st, hram.data()); crypto_core_ed25519_scalar_reduce(S, hram.data()); } -} // namespace + // The string_view overloads are the only place a caller can get the length wrong: the binary + // API takes fixed-extent spans, so a bad size there is a compile error. `name` is the + // parameter name as it appears in the public API, for the exception message. + template + std::span require_bytes(std::string_view val, std::string_view name) { + if (val.size() != N) + throw std::invalid_argument{ + "Invalid " + std::string{name} + ": expected " + std::to_string(N) + " bytes"}; + return to_byte_span(val.data()); + } -bytes<64> sign( - std::span curve25519_privkey, std::span msg) { +} // namespace - bytes<32> A; +b64 sign(std::span curve25519_privkey, std::span msg) { + uchars<32> A; // Convert the x25519 privkey to an ed25519 pubkey: - crypto_scalarmult_ed25519_base(A.data(), curve25519_privkey.data()); + crypto_scalarmult_ed25519_base(A.data(), to_unsigned(curve25519_privkey.data())); // Signal's XEd25519 spec requires that the sign bit be zero, so if it isn't we negate. bool negative = A[31] >> 7; - A[31] &= 0x7f; - bytes<32> a, neg_a; + uchars<32> a, neg_a; std::memcpy(a.data(), curve25519_privkey.data(), a.size()); crypto_core_ed25519_scalar_negate(neg_a.data(), a.data()); - constant_time_conditional_assign(a, neg_a, negative); + + // constant_time_conditional_assign works on std::byte arrays; use bit_cast for uchars + auto ba = std::bit_cast>(a); + auto bna = std::bit_cast>(neg_a); + constant_time_conditional_assign(ba, bna, negative); + a = std::bit_cast>(ba); // We now have our a, A privkey/public. (Note that a is just the private key scalar, *not* the // ed25519 secret key). - bytes<32> r = xed25519_compute_r(a, msg); - bytes<64> signature; // R || S - auto* R = signature.data(); - auto* S = signature.data() + 32; + uchars<32> r = xed25519_compute_r(a, msg); + uchars<64> sig_uc; // R || S + auto* R = sig_uc.data(); + auto* S = sig_uc.data() + 32; crypto_scalarmult_ed25519_base_noclamp(R, r.data()); @@ -111,56 +110,38 @@ bytes<64> sign( crypto_core_ed25519_scalar_mul(S, S, a.data()); // S *= a crypto_core_ed25519_scalar_add(S, S, r.data()); // S += r - return signature; + return std::bit_cast(sig_uc); } std::string sign(std::string_view curve25519_privkey, std::string_view msg) { - auto privkey = to_span(curve25519_privkey); - if (privkey.size() != 32) - throw std::invalid_argument{"Invalid curve25519_privkey: expected 32 bytes"}; - - auto sig = sign(privkey.first<32>(), to_span(msg)); + auto sig = sign( + require_bytes<32>(curve25519_privkey, "curve25519_privkey"), to_span(msg)); return std::string{reinterpret_cast(sig.data()), sig.size()}; } bool verify( - std::span signature, - std::span curve25519_pubkey, - std::span msg) { + std::span signature, + std::span curve25519_pubkey, + std::span msg) { auto ed_pubkey = pubkey(curve25519_pubkey); return 0 == crypto_sign_ed25519_verify_detached( - signature.data(), msg.data(), msg.size(), ed_pubkey.data()); + to_unsigned(signature.data()), + to_unsigned(msg.data()), + msg.size(), + to_unsigned(ed_pubkey.data())); } bool verify(std::string_view signature, std::string_view curve25519_pubkey, std::string_view msg) { - auto sig = to_span(signature); - if (sig.size() != crypto_sign_ed25519_BYTES) - throw std::invalid_argument{"Invalid signature: expected 64 bytes"}; - - auto pubkey = to_span(curve25519_pubkey); - if (pubkey.size() != 32) - throw std::invalid_argument{"Invalid curve25519_pubkey: expected 32 bytes"}; - - return verify(sig.first<64>(), pubkey.first<32>(), to_span(msg)); + return verify( + require_bytes<64>(signature, "signature"), + require_bytes<32>(curve25519_pubkey, "curve25519_pubkey"), + to_span(msg)); } -std::array pubkey(std::span curve25519_pubkey) { - fe25519 u, y; - crypto_internal_fe25519_frombytes(u, curve25519_pubkey.data()); - fe25519_montx_to_edy(y, u); - - std::array ed_pubkey; - crypto_internal_fe25519_tobytes(ed_pubkey.data(), y); - - return ed_pubkey; -} +// pubkey(...) is in xed25519-tweetnacl.cpp std::string pubkey(std::string_view curve25519_pubkey) { - auto x_pk = to_span(curve25519_pubkey); - if (x_pk.size() != 32) - throw std::invalid_argument{"Invalid curve25519_pubkey: expected 32 bytes"}; - - auto ed_pk = pubkey(x_pk.first<32>()); + auto ed_pk = pubkey(require_bytes<32>(curve25519_pubkey, "curve25519_pubkey")); return std::string{reinterpret_cast(ed_pk.data()), ed_pk.size()}; } @@ -176,7 +157,9 @@ LIBSESSION_C_API bool session_xed25519_sign( assert(signature != NULL); try { auto sig = session::xed25519::sign( - std::span{curve25519_privkey, 32}, {msg, msg_len}); + std::span{ + reinterpret_cast(curve25519_privkey), 32}, + std::span{reinterpret_cast(msg), msg_len}); std::memcpy(signature, sig.data(), sig.size()); return true; } catch (...) { @@ -190,22 +173,17 @@ LIBSESSION_C_API bool session_xed25519_verify( const unsigned char* msg, size_t msg_len) { return session::xed25519::verify( - std::span{signature, 64}, - std::span{pubkey, 32}, - {msg, msg_len}); + std::span{reinterpret_cast(signature), 64}, + std::span{reinterpret_cast(pubkey), 32}, + std::span{reinterpret_cast(msg), msg_len}); } -LIBSESSION_C_API bool session_xed25519_pubkey( +LIBSESSION_C_API void session_xed25519_pubkey( unsigned char* ed25519_pubkey, const unsigned char* curve25519_pubkey) { assert(ed25519_pubkey != NULL); - try { - auto edpk = session::xed25519::pubkey( - std::span{curve25519_pubkey, 32}); - std::memcpy(ed25519_pubkey, edpk.data(), edpk.size()); - return true; - } catch (...) { - return false; - } + auto ed_pk = session::xed25519::pubkey(std::span{ + reinterpret_cast(curve25519_pubkey), 32}); + std::memcpy(ed25519_pubkey, ed_pk.data(), 32); } } // extern "C" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ed0d1af5..18e5fa38d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,6 +5,24 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") endif() set(LIB_SESSION_UTESTS_SOURCES + test_client/attachments.cpp + test_client/configs.cpp + test_client/conversation_api.cpp + test_client/download_cache.cpp + test_client/profile_pictures.cpp + test_client/ids_and_schema.cpp + test_client/interop_and_threading.cpp + test_client/receiving.cpp + test_client/replies.cpp + test_client/requests.cpp + test_client/sending.cpp + test_client/volatile.cpp + test_core_configs.cpp + test_core_devices.cpp + test_core_globals.cpp + test_core_schema.cpp + test_dm_receive.cpp + test_dm_send.cpp test_attachment_encrypt.cpp test_blinding.cpp test_bt_merge.cpp @@ -20,48 +38,58 @@ set(LIB_SESSION_UTESTS_SOURCES test_curve25519.cpp test_ed25519.cpp test_encrypt.cpp + test_format.cpp test_group_keys.cpp test_group_info.cpp test_group_members.cpp test_hash.cpp #test_logging.cpp # Handled separately, see below test_multi_encrypt.cpp + test_mnemonics.cpp test_proto.cpp test_pro_backend.cpp test_random.cpp test_session_encrypt.cpp + test_swarm_retry.cpp test_utils.cpp test_session_protocol.cpp test_xed25519.cpp test_unicode_operations.cpp - test_backend_session_file_server.cpp - test_backed_session_open_group_server.cpp case_logger.cpp ) -if(ENABLE_NETWORKING) - list(APPEND LIB_SESSION_UTESTS_SOURCES test_network_swarm.cpp) - list(APPEND LIB_SESSION_UTESTS_SOURCES test_onionreq.cpp) - list(APPEND LIB_SESSION_UTESTS_SOURCES test_onion_request_router.cpp) - list(APPEND LIB_SESSION_UTESTS_SOURCES test_snode_pool.cpp) -endif() +list(APPEND LIB_SESSION_UTESTS_SOURCES + test_backend_session_file_server.cpp + test_backed_session_open_group_server.cpp + test_ip_country.cpp + test_network_swarm.cpp + test_onionreq.cpp + test_onion_request_router.cpp + test_snode_pool.cpp + test_core_network.cpp + test_poll.cpp + test_pfs_key_cache.cpp + test_sqlite_bind.cpp) add_library(test_libs INTERFACE) target_link_libraries(test_libs INTERFACE + libsession::client libsession::config - libsodium::sodium-internal + libsession::core + libsession::crypto + session::SQLite + sessiondep::libsodium + mlkem_native::mlkem768 nlohmann_json::nlohmann_json oxen::logging) -if (ENABLE_NETWORKING) - target_link_libraries(test_libs INTERFACE libsession::network) -else() - target_compile_definitions(test_libs INTERFACE DISABLE_NETWORKING) -endif() +target_link_libraries(test_libs INTERFACE libsession::network) add_executable(testAll main.cpp ${LIB_SESSION_UTESTS_SOURCES}) +target_include_directories(testAll PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") +add_subdirectory(schema) target_link_libraries(testAll PRIVATE test_libs Catch2::Catch2) @@ -81,10 +109,40 @@ target_link_libraries(testLogging PRIVATE test_libs Catch2::Catch2WithMain) +# Driven by schema_history_check.sh rather than being a unit test: it needs schemas out of git +# history, since a previous full_schema.sql is the only thing a database can be upgraded *from*. +add_executable(schema-upgrade-check schema_upgrade_check.cpp) +target_link_libraries(schema-upgrade-check PRIVATE + libsession::client + test_libs) + if(NOT TARGET check) add_custom_target(check COMMAND testLogging - COMMAND testAll) + COMMAND testAll + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/schema_history_check.sh" $ + DEPENDS schema-upgrade-check) +endif() + +option(BUILD_LIVE_TESTS "Build the live testnet integration tests" ON) + +if(BUILD_LIVE_TESTS) + add_executable(testLive + live/main.cpp + live/test_swarm.cpp + live/test_pubkey_xfer.cpp + live/test_file_transfer.cpp + live/test_attachment_send.cpp) + target_link_libraries(testLive PRIVATE + test_libs + Catch2::Catch2) +endif() + +if(BUILD_LIVE_TESTS) + add_executable(quic-files EXCLUDE_FROM_ALL quic-files.cpp) + target_link_libraries(quic-files PRIVATE test_libs) + target_include_directories(quic-files PRIVATE + ${CMAKE_SOURCE_DIR}/external/session-router/external/CLI11/include) endif() add_executable(swarm-auth-test EXCLUDE_FROM_ALL swarm-auth-test.cpp) @@ -93,6 +151,6 @@ target_link_libraries(swarm-auth-test PRIVATE config) if(STATIC_BUNDLE) add_executable(static-bundle-test static_bundle.cpp) target_include_directories(static-bundle-test PUBLIC ../include) - target_link_libraries(static-bundle-test PRIVATE "${PROJECT_BINARY_DIR}/libsession-util.a" oxenc::oxenc quic) + target_link_libraries(static-bundle-test PRIVATE "${PROJECT_BINARY_DIR}/libsession-util.a" oxenc::oxenc oxen::quic) add_dependencies(static-bundle-test session-util) endif() diff --git a/tests/dns_utils.hpp b/tests/dns_utils.hpp new file mode 100644 index 000000000..be94abdc6 --- /dev/null +++ b/tests/dns_utils.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +extern "C" { +#include +#include +} + +namespace session::test { + +// Resolves a hostname to an IP address string via getaddrinfo. This is needed because libquic +// does not perform DNS resolution. +inline std::string resolve_host(const std::string& host) { + struct addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + + struct addrinfo* res = nullptr; + if (int rc = getaddrinfo(host.c_str(), nullptr, &hints, &res); rc != 0 || !res) + throw std::runtime_error{ + "Failed to resolve '" + host + "': " + (rc ? gai_strerror(rc) : "no results")}; + + char buf[INET6_ADDRSTRLEN]{}; + if (res->ai_family == AF_INET6) + inet_ntop( + AF_INET6, + &reinterpret_cast(res->ai_addr)->sin6_addr, + buf, + sizeof(buf)); + else + inet_ntop( + AF_INET, &reinterpret_cast(res->ai_addr)->sin_addr, buf, sizeof(buf)); + + freeaddrinfo(res); + return buf; +} + +} // namespace session::test diff --git a/tests/live/live_utils.hpp b/tests/live/live_utils.hpp new file mode 100644 index 000000000..2a04ed17a --- /dev/null +++ b/tests/live/live_utils.hpp @@ -0,0 +1,144 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dns_utils.hpp" +#include "../test_helper.hpp" + +using namespace std::literals; + +// Defined in live/main.cpp; consumed here and in all live test files. +extern session::network::opt::router live_router_mode; + +// Testnet QUIC file server direct-connect hostname and Ed25519 pubkey (for --direct mode). +// These are test-only values; the library code uses .sesh addresses for session-router mode. +inline constexpr auto TESTNET_QUIC_FS_HOST = "angus.oxen.io"; +inline constexpr auto TESTNET_QUIC_FS_ED_PUBKEY = + "929e33ded05e653fec04b49645117f51851f102a947e04806791be416ed76602"; + +// Creates a Network instance pointed at testnet using the current live_router_mode. +inline std::unique_ptr make_testnet_network( + std::filesystem::path cache_dir) { + namespace opt = session::network::opt; + + std::vector net_opts; + net_opts.push_back(opt::netid::testnet()); + net_opts.push_back(live_router_mode); + net_opts.push_back(opt::cache_directory{std::move(cache_dir)}); + + // For direct mode, configure the QUIC file server address so DirectRouter uses the + // quic-files protocol instead of the legacy HTTP path. We resolve the hostname here + // because libquic does not do DNS resolution. + if (live_router_mode.type == opt::router::Type::direct) { + net_opts.push_back(opt::quic_file_server_ed_pubkey{TESTNET_QUIC_FS_ED_PUBKEY}); + net_opts.push_back( + opt::quic_file_server_address{session::test::resolve_host(TESTNET_QUIC_FS_HOST)}); + } + + return std::make_unique(net_opts); +} + +// Creates a session::TempCore connected to a fresh testnet Network. A unique temporary directory +// is created for the network's snode-pool cache and stored in session::TempCore::extra_dir so it is +// removed when the session::TempCore is destroyed. All CoreOption arguments are forwarded to the +// session::TempCore constructor (e.g. predefined_seed, encryption options). +template +inline session::TempCore make_live_core(Opts&&... opts) { + static std::atomic n{0}; + auto cache_dir = std::filesystem::temp_directory_path() / fmt::format("live_net_cache_{}", ++n); + std::filesystem::create_directories(cache_dir); + + session::TempCore tc{std::forward(opts)...}; + tc.extra_dir = cache_dir; + tc->set_network(make_testnet_network(std::move(cache_dir))); + return tc; +} + +// Builds the JSON params body for a signed "store" request targeting Core's AccountPubkeys +// namespace. The returned bytes are the raw params (no "method"/"params" wrapper); the network +// routing layer adds the wrapper as required by the transport. +// +// See session-storage-server client_rpc_endpoints.h for the store endpoint spec. +inline std::vector build_account_pubkeys_store_params(session::core::Core& core) { + auto session_id_hex = oxenc::to_hex(core.globals.session_id()); + auto now_ms = session::epoch_ms(session::clock_now_ms()); + constexpr auto ns = static_cast(session::config::Namespace::AccountPubkeys); + + // Signature covers: "store" || namespace (decimal) || sig_timestamp (decimal) + auto to_sign = fmt::format("store{}{}", ns, now_ms); + auto seed = core.globals.account_seed(); + auto sig = session::ed25519::sign(seed.ed25519_secret(), session::to_span(to_sign)); + + auto msg = core.devices.build_account_pubkey_message(); + + nlohmann::json params = { + {"pubkey", session_id_hex}, + {"pubkey_ed25519", core.globals.pubkey_ed25519().hex()}, + {"namespace", ns}, + {"data", + oxenc::to_base64( + std::string_view{reinterpret_cast(msg.data()), msg.size()})}, + {"timestamp", now_ms}, + {"sig_timestamp", now_ms}, + {"signature", oxenc::to_base64(sig)}, + {"ttl", int64_t{2592000000}}, // 30 days in ms + }; + return session::to_vector(params.dump()); +} + +// Pushes Core's AccountPubkeys message to its swarm. Resolves the swarm, sends the signed store +// request, and blocks until the response arrives or the timeout elapses. +// Returns true if the store was accepted by the swarm node. +inline bool store_account_pubkeys( + session::core::Core& core, std::chrono::milliseconds timeout = 30s) { + using namespace session::network; + using namespace std::chrono_literals; + + auto net = core.network(); + if (!net) + throw std::logic_error{"store_account_pubkeys called without a network object"}; + + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto body = build_account_pubkeys_store_params(core); + + net->get_swarm( + core.globals.pubkey_x25519(), + false, + [promise, net, body = std::move(body)]( + swarm_id_t, std::vector swarm) mutable { + if (swarm.empty()) { + promise->set_value(false); + return; + } + net->send_request( + Request{swarm.front(), + "store", + std::move(body), + RequestCategory::standard_small, + 5s}, + [promise](bool success, bool, int16_t, auto, auto) { + promise->set_value(success); + }); + }); + + return future.wait_for(timeout) == std::future_status::ready && future.get(); +} diff --git a/tests/live/main.cpp b/tests/live/main.cpp new file mode 100644 index 000000000..86fcb43d2 --- /dev/null +++ b/tests/live/main.cpp @@ -0,0 +1,52 @@ +#include +#include +#include + +#include "../log_setup.hpp" + +// Router selection: consumed by make_testnet_core() in live_utils.hpp. +session::network::opt::router live_router_mode = +#ifdef ENABLE_NETWORKING_SROUTER + session::network::opt::router::session_router(); +#else + session::network::opt::router::onion_requests(); +#endif + +int main(int argc, char* argv[]) { + Catch::Session session; + + using namespace Catch::Clara; + using session::network::opt::router; + LogSetup log; + log.level = "warning"; + + bool use_srouter = false, use_onionreq = false, use_direct = false; + + auto cli = session.cli() | log.opts() | + Opt(use_srouter)["--srouter"]("route requests via session-router") | + Opt(use_onionreq)["--onionreq"]("route requests via onion requests") | + Opt(use_direct)["--direct"]("route requests directly (no onion routing)"); + + session.cli(cli); + + if (int rc = session.applyCommandLine(argc, argv); rc != 0) + return rc; + + if (int n = use_srouter + use_onionreq + use_direct; n > 1) { + oxen::log::critical( + oxen::log::Cat("live-test"), + "--srouter, --onionreq, and --direct are mutually exclusive"); + return 1; + } + + if (use_direct) + live_router_mode = router::direct(); + else if (use_onionreq) + live_router_mode = router::onion_requests(); + else if (use_srouter) + live_router_mode = router::session_router(); + + log.apply(); + + return session.run(); +} diff --git a/tests/live/test_attachment_send.cpp b/tests/live/test_attachment_send.cpp new file mode 100644 index 000000000..421574d4b --- /dev/null +++ b/tests/live/test_attachment_send.cpp @@ -0,0 +1,157 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "live_utils.hpp" + +using namespace session; +using namespace session::client; +using namespace std::literals; + +namespace { + +struct LiveClient { + std::filesystem::path dir; + std::unique_ptr client; + + LiveClient() : + dir{std::filesystem::temp_directory_path() / + fmt::format("{}", random::unique_id("live_client", 7))} { + std::filesystem::create_directories(dir); + client = std::make_unique(dir / "client.db"); + client->core.set_network(make_testnet_network(dir / "netcache")); + } + + ~LiveClient() { + client.reset(); + std::error_code ec; + std::filesystem::remove_all(dir, ec); + } + + Client* operator->() { return client.get(); } +}; + +// Waits for the message to leave `uploading`, which is the only state the caller is told nothing +// more about: everything after it is reported through send state changes. +bool wait_until_sent(Client& c, int64_t id, std::chrono::seconds limit) { + auto deadline = std::chrono::steady_clock::now() + limit; + while (std::chrono::steady_clock::now() < deadline) { + auto msg = c.message(id, await); + REQUIRE(msg); + if (msg->send_state == SendState::sent) + return true; + if (msg->send_state == SendState::failed || msg->send_state == SendState::unsendable) + return false; + std::this_thread::sleep_for(200ms); + } + return false; +} + +} // namespace + +TEST_CASE( + "Live: a message's attachment is uploaded and named in what is stored", "[live][client]") { + auto file = std::filesystem::temp_directory_path() / "live_attachment_send.bin"; + std::vector contents(64 * 1024); + randombytes_buf(contents.data(), contents.size()); + { + std::ofstream out{file, std::ios::binary}; + out.write(reinterpret_cast(contents.data()), contents.size()); + } + + LiveClient c; + b33 me; + std::ranges::copy(c->core.globals.session_id(), me.begin()); + + std::vector>> reports; + auto id = c->send_message( + ConversationId::dm(me), + {.body = "with an attachment", + .attachments = {OutgoingAttachment{ + .path = file, .content_type = "application/octet-stream"}}}, + [&](size_t idx, int64_t sent, int64_t total, std::optional result) { + reports.emplace_back(idx, sent, total, result); + }, + await); + + REQUIRE(wait_until_sent(*c.client, id, 120s)); + + // The upload reported itself finished, and did so as a result rather than by reaching the + // total: the file server's acceptance is what counts. + REQUIRE(!reports.empty()); + auto [idx, sent, total, result] = reports.back(); + CHECK(idx == 0); + REQUIRE(result.has_value()); + CHECK(*result == 0); + CHECK(total > 0); + + // What was stored has to name the upload, or the recipient has no way to fetch it. + // Both columns are blobs, and the content's length is not fixed -- so it is read as a `blob` + // view from a statement kept alive around it, rather than through a one-shot call that would + // finalize the statement and invalidate the view. + std::vector raw; + int64_t stored_msgid = 0; + { + auto conn = c->core.database().conn(); + auto st = conn.prepared_bind( + "SELECT r.content, m.msgid FROM message_raw_content r" + " JOIN messages m ON m.id = r.message WHERE r.message = ?", + id); + REQUIRE(st->executeStep()); + auto [content, msgid] = sqlite::get(*st); + raw.assign(content.begin(), content.end()); + stored_msgid = msgid; + } + + SessionProtos::Content parsed; + REQUIRE(parsed.ParseFromArray(raw.data(), static_cast(raw.size()))); + REQUIRE(parsed.has_datamessage()); + REQUIRE(parsed.datamessage().attachments_size() == 1); + + const auto& ptr = parsed.datamessage().attachments(0); + CHECK(ptr.has_url()); + CHECK(!ptr.url().empty()); + CHECK(ptr.key().size() == 32); + CHECK(ptr.contenttype() == "application/octet-stream"); + + // The file's own size, exactly -- not the encrypted size the file server reports back, which is + // larger and is what every other client would misread. Pinned to the byte rather than to `> + // 0`, which is what let the two be confused in the first place. + CHECK(ptr.size() == contents.size()); + + // The url has to be one the download path can actually use, rather than merely non-empty. + auto info = network::file_server::parse_download_url(ptr.url()); + REQUIRE(info); + CHECK(!info->file_id.empty()); + + // ...and it has to say how the file is encrypted. upload_file only ever uses the stream + // scheme, so a url without this fragment sends every recipient to the legacy decryptor, which + // cannot open it: an unopenable attachment rather than a failed download. + CHECK(info->wants_stream_decryption); + + // Also asserted on the url itself, since the check above would pass just as happily if both + // sides of the round trip were wrong together. + CHECK(ptr.url().find('#') != std::string::npos); + + // And the deprecated numeric id, which old clients still read, has to agree with it while the + // file server is still issuing numeric ids. + if (std::ranges::all_of(info->file_id, [](char ch) { return ch >= '0' && ch <= '9'; })) + CHECK(std::to_string(ptr.id()) == info->file_id); + + // The one that would corrupt a conversation rather than merely break a download. Sending with + // attachments stores the message twice: once with the body alone, and again once the uploads + // have given it something to point at. The identifier has to survive that rewrite, because it + // is what the copy returning from our own swarm is recognised by -- so the content finally sent + // must carry the same one the row was stored with, rather than a fresh one. + REQUIRE(parsed.has_msgid()); + CHECK(parsed.msgid() == stored_msgid); + + std::filesystem::remove(file); +} diff --git a/tests/live/test_file_transfer.cpp b/tests/live/test_file_transfer.cpp new file mode 100644 index 000000000..bf35aa6f0 --- /dev/null +++ b/tests/live/test_file_transfer.cpp @@ -0,0 +1,206 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "live_utils.hpp" + +using namespace session; +using namespace std::literals; + +// Default timeout for live network operations. +static constexpr auto LIVE_TIMEOUT = 60s; + +// These tests require a QUIC file server and only work under --srouter or --direct mode +// (not --onionreq, which cannot support the QUIC file server protocol). + +TEST_CASE("Live: file upload via QUIC", "[live][file]") { + // Skip if running under onion request mode (no QUIC support) + // Works under all routing modes: --srouter and --direct use the QUIC file server protocol, + // --onionreq falls back to the legacy HTTP proxy path. + + auto core = make_live_core(); + auto net = core->network(); + REQUIRE(net); + + // Generate small test data and encrypt it + std::vector plaintext(4096); + randombytes_buf(plaintext.data(), plaintext.size()); + + auto seed_acc = core->globals.account_seed(); + auto seed = seed_acc.seed(); + auto [encrypted, key] = attachment::encrypt( + std::span{ + reinterpret_cast(seed.data()), seed.size()}, + plaintext, + attachment::Domain::ATTACHMENT, + true); + + // Upload + std::promise> promise; + auto future = promise.get_future(); + + network::UploadRequest req; + req.request_timeout = 30s; + req.overall_timeout = LIVE_TIMEOUT; + + bool consumed = false; + req.next_data = [&]() -> std::vector { + if (consumed) + return {}; + consumed = true; + return {encrypted.begin(), encrypted.end()}; + }; + req.ttl = 1min; + req.on_complete = [&](auto result, bool) { promise.set_value(std::move(result)); }; + + // Intentionally exercising the deprecated upload() path to verify it still works. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + net->upload(std::move(req)); +#pragma GCC diagnostic pop + + REQUIRE(future.wait_for(LIVE_TIMEOUT) == std::future_status::ready); + auto result = future.get(); + REQUIRE(std::holds_alternative(result)); + + auto& meta = std::get(result); + CHECK(!meta.id.empty()); + CHECK(meta.size > 0); +} + +TEST_CASE("Live: streaming file upload via upload_file", "[live][file]") { + auto core = make_live_core(); + auto net = core->network(); + REQUIRE(net); + + // Write test data to a temp file + auto tmp = std::filesystem::temp_directory_path() / "upload_file_test.dat"; + { + std::vector plaintext(8192); + randombytes_buf(plaintext.data(), plaintext.size()); + std::ofstream f{tmp, std::ios::binary}; + REQUIRE(f); + f.write(reinterpret_cast(plaintext.data()), plaintext.size()); + } + + std::array seed; + randombytes_buf(seed.data(), seed.size()); + + std::promise, int16_t>> + promise; + auto future = promise.get_future(); + + network::FileUploadRequest req; + req.file = tmp; + req.domain = attachment::Domain::ATTACHMENT; + req.allow_large = true; + req.ttl = 1min; + req.request_timeout = 30s; + req.overall_timeout = LIVE_TIMEOUT; + req.on_complete = [&](auto result, bool) { promise.set_value(std::move(result)); }; + + net->upload_file(std::move(req), seed); + + REQUIRE(future.wait_for(LIVE_TIMEOUT) == std::future_status::ready); + auto result = future.get(); + + std::filesystem::remove(tmp); + + using pair_t = std::pair; + REQUIRE(std::holds_alternative(result)); + + auto& [meta, key] = std::get(result); + CHECK(!meta.id.empty()); + CHECK(meta.size > 0); + CHECK(!key.empty()); +} + +TEST_CASE("Live: file upload and download round-trip via QUIC", "[live][file]") { + // Works under all routing modes: --srouter and --direct use the QUIC file server protocol, + // --onionreq falls back to the legacy HTTP proxy path. + + auto core = make_live_core(); + auto net = core->network(); + REQUIRE(net); + + // Generate test data + std::vector plaintext(16384); + randombytes_buf(plaintext.data(), plaintext.size()); + + auto seed_acc = core->globals.account_seed(); + auto seed = seed_acc.seed(); + auto [encrypted, key] = attachment::encrypt( + std::span{ + reinterpret_cast(seed.data()), seed.size()}, + plaintext, + attachment::Domain::ATTACHMENT, + true); + + // Upload + std::promise> upload_promise; + auto upload_future = upload_promise.get_future(); + + network::UploadRequest upload_req; + upload_req.request_timeout = 30s; + upload_req.overall_timeout = LIVE_TIMEOUT; + + bool consumed = false; + upload_req.next_data = [&]() -> std::vector { + if (consumed) + return {}; + consumed = true; + return {encrypted.begin(), encrypted.end()}; + }; + upload_req.ttl = 1min; + upload_req.on_complete = [&](auto result, bool) { + upload_promise.set_value(std::move(result)); + }; + + // Intentionally exercising the deprecated upload() path to verify it still works. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + net->upload(std::move(upload_req)); +#pragma GCC diagnostic pop + + REQUIRE(upload_future.wait_for(LIVE_TIMEOUT) == std::future_status::ready); + auto upload_result = upload_future.get(); + REQUIRE(std::holds_alternative(upload_result)); + auto& upload_meta = std::get(upload_result); + + // Download. The bytes went up already encrypted, by the attachment::encrypt call above, so the + // url says stream even though the upload path itself did nothing to them. + auto download_url = network::file_server::generate_download_url( + upload_meta.id, net->file_server_config, /*stream_encrypted=*/true); + + std::promise> download_promise; + auto download_future = download_promise.get_future(); + std::vector downloaded_data; + + network::DownloadRequest download_req; + download_req.download_url = download_url; + download_req.request_timeout = 30s; + download_req.overall_timeout = LIVE_TIMEOUT; + download_req.on_data = [&](auto&, std::span data) { + downloaded_data.insert(downloaded_data.end(), data.begin(), data.end()); + }; + download_req.on_complete = [&](auto result, bool) { + download_promise.set_value(std::move(result)); + }; + + net->download(std::move(download_req)); + + REQUIRE(download_future.wait_for(LIVE_TIMEOUT) == std::future_status::ready); + auto download_result = download_future.get(); + REQUIRE(std::holds_alternative(download_result)); + + // Decrypt and verify + auto decrypted = attachment::decrypt(std::span{downloaded_data}, key); + REQUIRE(decrypted.size() == plaintext.size()); + CHECK(decrypted == plaintext); +} diff --git a/tests/live/test_pubkey_xfer.cpp b/tests/live/test_pubkey_xfer.cpp new file mode 100644 index 000000000..8391ed335 --- /dev/null +++ b/tests/live/test_pubkey_xfer.cpp @@ -0,0 +1,59 @@ +#include + +#include "../utils.hpp" +#include "live_utils.hpp" + +using namespace session; +using namespace std::literals; + +// Default timeout for live network operations. +static constexpr auto LIVE_TIMEOUT = 30s; + +TEST_CASE("Live: PFS key prefetch returns NAK for account with no published keys", "[live][pfs]") { + // Core A is a fresh account that has never published its AccountPubkeys to the swarm. + // Core B fetches keys for Core A's session id and should receive a NAK (empty namespace). + auto core_a = make_live_core(); + auto core_b = make_live_core(); + + b33 sid_a; + std::ranges::copy(core_a->globals.session_id(), sid_a.begin()); + + core_b->prefetch_pfs_keys(sid_a); + + auto entry = wait_for( + [&] { return session::TestHelper::pfs_cache_entry(*core_b, sid_a); }, LIVE_TIMEOUT); + REQUIRE(entry.has_value()); + // NAK: fetch completed but no keys were present. + CHECK_FALSE(entry->fetched_at.has_value()); + CHECK(entry->nak_at.has_value()); +} + +TEST_CASE("Live: PFS key prefetch retrieves keys after store to swarm", "[live][pfs]") { + // Core A stores its AccountPubkeys to the swarm; Core B then fetches them. + auto core_a = make_live_core(); + auto core_b = make_live_core(); + + // Store Core A's account pubkeys to its swarm. + REQUIRE(store_account_pubkeys(*core_a, LIVE_TIMEOUT)); + + // Now fetch from Core B's perspective. + b33 sid_a; + std::ranges::copy(core_a->globals.session_id(), sid_a.begin()); + + core_b->prefetch_pfs_keys(sid_a); + + auto entry = wait_for( + [&] { return session::TestHelper::pfs_cache_entry(*core_b, sid_a); }, LIVE_TIMEOUT); + REQUIRE(entry.has_value()); + // Successful fetch: keys present, no NAK. + REQUIRE(entry->fetched_at.has_value()); + CHECK_FALSE(entry->nak_at.has_value()); + + // The fetched pubkeys must match what Core A has as its active account keys. + auto [expected_x25519, expected_mlkem768] = + session::TestHelper::active_account_pubkeys(*core_a); + REQUIRE(entry->pubkey_x25519.has_value()); + REQUIRE(entry->pubkey_mlkem768.has_value()); + CHECK(*entry->pubkey_x25519 == expected_x25519); + CHECK(*entry->pubkey_mlkem768 == expected_mlkem768); +} diff --git a/tests/live/test_swarm.cpp b/tests/live/test_swarm.cpp new file mode 100644 index 000000000..75c7f1fe0 --- /dev/null +++ b/tests/live/test_swarm.cpp @@ -0,0 +1,35 @@ +#include + +#include "../utils.hpp" +#include "live_utils.hpp" + +using namespace session; +using namespace std::literals; + +// Default timeout for live network operations. +static constexpr auto LIVE_TIMEOUT = 30s; + +TEST_CASE("Live: network bootstraps snode pool from testnet", "[live][swarm]") { + auto core = make_live_core(); + auto& net = *core->network(); + + std::vector result; + callback_waiter waiter{ + [&](std::vector nodes) { result = std::move(nodes); }}; + net.get_random_nodes(5, waiter); + REQUIRE(waiter.wait(LIVE_TIMEOUT)); + CHECK(result.size() >= 1); +} + +TEST_CASE("Live: network resolves swarm for a locally-generated session id", "[live][swarm]") { + auto core = make_live_core(); + auto& net = *core->network(); + + std::vector swarm_result; + callback_waiter waiter{[&](network::swarm_id_t, std::vector swarm) { + swarm_result = std::move(swarm); + }}; + net.get_swarm(core->globals.pubkey_x25519(), false, waiter); + REQUIRE(waiter.wait(LIVE_TIMEOUT)); + CHECK(swarm_result.size() >= 1); +} diff --git a/tests/log_setup.hpp b/tests/log_setup.hpp new file mode 100644 index 000000000..f05ddd307 --- /dev/null +++ b/tests/log_setup.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include + +/// Holds the --log-level / --log-file option state and applies it after argument parsing. +struct LogSetup { + std::string level = "critical"; + std::string file = "stderr"; + + /// Returns a Clara option pipeline for --log-level and --log-file. + auto opts() { + using namespace Catch::Clara; + return Opt(level, + "level")["--log-level"]("oxen-logging log level to apply to the test run") | + Opt(file, "file")["--log-file"]( + "oxen-logging log file to output logs to, or one of " + "stdout/-/stderr/syslog."); + } + + /// Initialises the oxen-logging sink from the parsed level/file values. + void apply() const { + constexpr std::array print_vals = { + "stdout", "-", "", "stderr", "nocolor", "stdout-nocolor", "stderr-nocolor"}; + oxen::log::Type type; + if (std::count(print_vals.begin(), print_vals.end(), file)) + type = oxen::log::Type::Print; + else if (file == "syslog") + type = oxen::log::Type::System; + else + type = oxen::log::Type::File; + + oxen::log::add_sink( + type, file, "[%T.%f] [%*] [\x1b[1m%n\x1b[0m:%^%l%$|\x1b[3m%g:%#\x1b[0m] %v"); + oxen::log::apply_categories(level); + } +}; diff --git a/tests/main.cpp b/tests/main.cpp index 76c4cc49f..4a9d4e5f6 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,21 +1,18 @@ #include #include +#include "log_setup.hpp" + std::string g_test_pro_backend_dev_server_url = "http://127.0.0.1:5000"; int main(int argc, char* argv[]) { Catch::Session session; using namespace Catch::Clara; - std::string log_level = "critical", log_file = "stderr"; + LogSetup log; bool test_case_tracing = false; - auto cli = session.cli() | - Opt(log_level, - "level")["--log-level"]("oxen-logging log level to apply to the test run") | - Opt(log_file, "file")["--log-file"]( - "oxen-logging log file to output logs to, or one of or one of " - "stdout/-/stderr/syslog.") | + auto cli = session.cli() | log.opts() | Opt(test_case_tracing)["-T"]["--test-tracing"]( "enable oxen log tracing of test cases/sections") | Opt(g_test_pro_backend_dev_server_url, "url")["--pro-backend-dev-server-url"]( @@ -27,21 +24,7 @@ int main(int argc, char* argv[]) { if (int rc = session.applyCommandLine(argc, argv); rc != 0) return rc; - auto lvl = oxen::log::level_from_string(log_level); - - constexpr std::array print_vals = { - "stdout", "-", "", "stderr", "nocolor", "stdout-nocolor", "stderr-nocolor"}; - oxen::log::Type type; - if (std::count(print_vals.begin(), print_vals.end(), log_file)) - type = oxen::log::Type::Print; - else if (log_file == "syslog") - type = oxen::log::Type::System; - else - type = oxen::log::Type::File; - - oxen::log::add_sink( - type, log_file, "[%T.%f] [%*] [\x1b[1m%n\x1b[0m:%^%l%$|\x1b[3m%g:%#\x1b[0m] %v"); - oxen::log::reset_level(lvl); + log.apply(); oxen::log::set_level( oxen::log::Cat("testcase"), diff --git a/tests/quic-files.cpp b/tests/quic-files.cpp new file mode 100644 index 000000000..d48a4580e --- /dev/null +++ b/tests/quic-files.cpp @@ -0,0 +1,431 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dns_utils.hpp" + +using namespace std::literals; + +namespace { + +using session::human_size; +using clock = std::chrono::steady_clock; + +namespace net = session::network; +namespace fs = net::file_server; +namespace attachment = session::attachment; + +using upload_result = std::variant, int16_t>; +using download_result = std::variant; +using on_data_t = std::function)>; + +using session::test::resolve_host; + +// --- Generic upload/download that take a transport-initiation callback --- + +int do_upload( + const std::string& filename, + attachment::Domain domain, + std::optional ttl, + std::shared_ptr network, + std::string download_cmd_hint) { + auto path = std::filesystem::path{filename}; + if (!std::filesystem::exists(path)) { + fmt::print(stderr, "File not found: {}\n", path.string()); + return 1; + } + auto file_size = static_cast(std::filesystem::file_size(path)); + fmt::print(stderr, "Uploading {} ({})...\n", path.string(), human_size{file_size}); + + std::array seed; + randombytes_buf(seed.data(), seed.size()); + + auto start = clock::now(); + std::promise promise; + auto future = promise.get_future(); + + net::FileUploadRequest req; + req.file = path; + req.domain = domain; + req.allow_large = true; + req.ttl = ttl; + req.request_timeout = 60s; + req.overall_timeout = 300s; + req.progress_interval = 250ms; + req.on_complete = [&](auto result, bool) { promise.set_value(std::move(result)); }; + + auto last_progress = start; + int64_t last_progress_bytes = 0; + req.on_progress = [&](int64_t acked, int64_t total) { + auto now = clock::now(); + auto since_last = std::chrono::duration(now - last_progress).count(); + auto recent_speed = since_last > 0 ? human_size{static_cast( + (acked - last_progress_bytes) / since_last)} + : human_size{0}; + auto pct = total > 0 ? 100.0 * acked / total : 0.0; + fmt::print( + stderr, + "[{}/{}] {:.1f}% {}/s\n", + human_size{acked}, + human_size{total}, + pct, + recent_speed); + last_progress = now; + last_progress_bytes = acked; + }; + + network->upload_file(std::move(req), seed); + + auto result = future.get(); + auto elapsed_s = std::chrono::duration(clock::now() - start).count(); + + if (auto* pair = std::get_if>(&result)) { + auto& [meta, key] = *pair; + auto key_hex = oxenc::to_hex(key.begin(), key.end()); + auto speed = human_size{static_cast(meta.size / std::max(elapsed_s, 0.001))}; + + fmt::print( + "\nUpload complete!\n" + " File ID: {}\n" + " Key: {}\n" + " Size: {}\n" + " Time: {:.1f}s\n" + " Speed: {}/s\n" + "\n" + "To download:\n" + "{} {} {}\n", + meta.id, + key_hex, + human_size{meta.size}, + elapsed_s, + speed, + download_cmd_hint, + meta.id, + key_hex); + return 0; + } + + fmt::print(stderr, "Upload failed with error {}\n", std::get(result)); + return 1; +} + +int do_download( + const std::string& key_hex, + const std::string& output, + std::function)> initiate) { + if (key_hex.size() != 64 || !oxenc::is_hex(key_hex)) { + fmt::print(stderr, "Invalid key: expected 64 hex characters\n"); + return 1; + } + + std::array key; + oxenc::from_hex(key_hex.begin(), key_hex.end(), reinterpret_cast(key.data())); + + std::ofstream out_file; + std::ostream* out_stream = &std::cout; + if (!output.empty()) { + out_file.open(output, std::ios::binary); + if (!out_file) + throw std::runtime_error{fmt::format("Failed to open {} for writing", output)}; + out_stream = &out_file; + } + + int64_t decrypted_bytes = 0; + attachment::Decryptor decryptor{key, [&](std::span decrypted) { + out_stream->write( + reinterpret_cast(decrypted.data()), + decrypted.size()); + decrypted_bytes += decrypted.size(); + }}; + + auto start = clock::now(); + std::promise promise; + auto future = promise.get_future(); + int64_t received_bytes = 0; + bool first_data = true; + auto last_progress = start; + int64_t last_progress_bytes = 0; + + initiate( + [&](const net::file_metadata& info, std::span data) { + auto now = clock::now(); + + if (first_data) { + first_data = false; + auto latency = std::chrono::duration(now - start); + fmt::print( + stderr, + "Transfer started after {:.0f}ms (file size: {})\n", + latency.count(), + human_size{info.size}); + last_progress = now; + } + + received_bytes += data.size(); + + if (!decryptor.update(data)) + throw std::runtime_error{ + fmt::format("Decryption failed at byte {}", received_bytes)}; + + auto since_last = now - last_progress; + if (since_last >= 2s) { + auto since_last_s = std::chrono::duration(since_last).count(); + auto recent_speed = human_size{static_cast( + (received_bytes - last_progress_bytes) / since_last_s)}; + fmt::print( + stderr, + "[{}/{}] {}/s\n", + human_size{received_bytes}, + human_size{info.size}, + recent_speed); + last_progress = now; + last_progress_bytes = received_bytes; + } + }, + [&](download_result r) { promise.set_value(std::move(r)); }); + + auto result = future.get(); + auto elapsed_s = std::chrono::duration(clock::now() - start).count(); + + if (auto* meta = std::get_if(&result)) { + if (!decryptor.finalize()) { + if (out_file.is_open()) { + out_file.close(); + std::filesystem::remove(output); + } + fmt::print(stderr, "Download succeeded but decryption finalization failed\n"); + return 1; + } + + auto speed = human_size{static_cast(received_bytes / std::max(elapsed_s, 0.001))}; + fmt::print( + "Download complete: {} encrypted, {} decrypted in {:.1f}s ({}/s)\n", + human_size{received_bytes}, + human_size{decrypted_bytes}, + elapsed_s, + speed); + + if (!output.empty()) + fmt::print("Written to {}\n", output); + return 0; + } + + if (out_file.is_open()) { + out_file.close(); + std::filesystem::remove(output); + } + fmt::print(stderr, "Download failed with error {}\n", std::get(result)); + return 1; +} + +// --- Mode-specific runners --- + +struct CliArgs { + // Mode + bool srouter = false; + bool testnet = true; + + // Direct mode + std::string server_pubkey_hex; + std::string server_address = "::1"; + uint16_t server_port = fs::QUIC_DEFAULT_PORT; + size_t max_udp_payload = 0; + + // Upload + std::string upload_filename; + attachment::Domain domain = attachment::Domain::ATTACHMENT; + std::optional ttl{3600s}; + + // Download + std::string dl_source; + std::string dl_key_hex; + std::string dl_output; + + const char* argv0; +}; + +int run(const CliArgs& args, bool is_upload) { + auto netid = args.testnet ? net::opt::netid::testnet() : net::opt::netid::mainnet(); + auto router = args.srouter ? net::opt::router::session_router() : net::opt::router::direct(); + auto cache_dir = std::filesystem::temp_directory_path() / + (args.testnet ? "quic_files_cache_testnet" : "quic_files_cache"); + std::filesystem::create_directories(cache_dir); + + std::vector net_opts; + net_opts.push_back(netid); + net_opts.push_back(router); + net_opts.push_back(net::opt::cache_directory{cache_dir}); + + // For direct mode, pass the QUIC file server address/pubkey/port so that DirectRouter + // uses the QUIC protocol instead of the legacy HTTP path. + if (!args.srouter) { + auto resolved = resolve_host(args.server_address); + if (resolved != args.server_address) + fmt::print(stderr, "Resolved {} -> {}\n", args.server_address, resolved); + + net_opts.push_back(net::opt::quic_file_server_ed_pubkey{args.server_pubkey_hex}); + net_opts.push_back(net::opt::quic_file_server_address{resolved}); + net_opts.push_back(net::opt::quic_file_server_port{args.server_port}); + } + + if (args.max_udp_payload > 0) + net_opts.push_back(net::opt::quic_max_udp_payload{args.max_udp_payload}); + + fmt::print( + stderr, + "Starting network ({}, {})...\n", + args.testnet ? "testnet" : "mainnet", + args.srouter ? "session-router" : "direct"); + + auto network = std::make_shared(net_opts); + + std::string mode_hint = fmt::format( + "{}{}{}", + args.argv0, + args.srouter ? " --srouter" : "", + args.testnet ? "" : " --mainnet"); + + if (is_upload) { + return do_upload( + args.upload_filename, + args.domain, + args.ttl, + network, + fmt::format("{} download", mode_hint)); + } + + // Build download URL from file ID if not already a URL + std::string download_url; + if (args.dl_source.find("://") != std::string::npos) + download_url = args.dl_source; + else + download_url = fs::generate_download_url( + args.dl_source, network->file_server_config, /*stream_encrypted=*/true); + + fmt::print(stderr, "Downloading: {}\n", download_url); + return do_download(args.dl_key_hex, args.dl_output, [&](on_data_t on_data, auto cb) { + net::DownloadRequest req; + req.download_url = download_url; + req.request_timeout = 60s; + req.overall_timeout = 300s; + req.on_data = std::move(on_data); + req.on_complete = [cb = std::move(cb)](auto r, bool) { cb(std::move(r)); }; + network->download(std::move(req)); + }); +} + +} // namespace + +int main(int argc, char* argv[]) { + CLI::App app{"QUIC file server upload/download tool"}; + app.require_subcommand(1); + app.fallthrough(); // Allow global options after subcommand + + CliArgs args; + args.argv0 = argv[0]; + + bool use_direct = false, use_mainnet = false; + app.add_flag("--srouter", args.srouter, "Route via session-router (default: direct)"); + app.add_flag("--direct", use_direct, "Connect directly to the file server"); + app.add_flag("--mainnet", use_mainnet, "Use mainnet (default: testnet)"); + + app.add_option("--server", args.server_pubkey_hex, "Ed25519 pubkey of the file server (hex)"); + app.add_option("--address", args.server_address, "Server address (hostname or IP)"); + app.add_option( + "--port", + args.server_port, + fmt::format("Server port (default: {})", fs::QUIC_DEFAULT_PORT)); + + app.add_option( + "--max-udp-payload", + args.max_udp_payload, + "Cap network-level QUIC UDP payload size (limits path MTU discovery; minimum 1200)"); + + std::string log_level = "warning"; + std::string log_file = "stderr"; + app.add_option( + "--log-level", + log_level, + "Log level/categories (e.g. warning, debug, quic-file-client=trace)"); + app.add_option("--log-file", log_file, "Log output: stderr, stdout, -, or a file path"); + + bool profile_pic = false, max_ttl = false; + int64_t ttl_seconds = 3600; + auto* upload_cmd = app.add_subcommand("upload", "Encrypt and upload a file"); + upload_cmd->add_option("filename", args.upload_filename, "File to upload")->required(); + upload_cmd->add_flag("--profile-pic", profile_pic, "Use PROFILE_PIC encryption domain"); + upload_cmd->add_flag("--max-ttl", max_ttl, "Use server's maximum TTL instead of default 1h"); + upload_cmd->add_option( + "--ttl", ttl_seconds, "TTL in seconds (default: 3600; ignored if --max-ttl)"); + + auto* download_cmd = app.add_subcommand("download", "Download and decrypt a file"); + download_cmd + ->add_option( + "source", + args.dl_source, + "File ID or download URL (e.g. http://host/file/ID#sr=addr.sesh:port)") + ->required(); + download_cmd->add_option("key", args.dl_key_hex, "Decryption key (hex)")->required(); + download_cmd->add_option("output", args.dl_output, "Output filename (default: stdout)"); + + CLI11_PARSE(app, argc, argv); + + if (args.srouter + use_direct > 1) { + fmt::print(stderr, "Error: --srouter and --direct are mutually exclusive\n"); + return 1; + } + if (!args.srouter && !use_direct) + use_direct = true; + if (use_mainnet) + args.testnet = false; + + // For direct mode, default the server pubkey and address from the known file server configs. + if (!args.srouter) { + if (args.server_pubkey_hex.empty()) { + auto& pk = args.testnet ? fs::QUIC_FS_ED_PUBKEY_TESTNET : fs::QUIC_FS_ED_PUBKEY_MAINNET; + args.server_pubkey_hex = oxenc::to_hex(pk.begin(), pk.end()); + } + if (args.server_address == "::1") { + args.server_address = + args.testnet ? "superduperfiles.oxen.io" : "anna.session.foundation"; + } + } + + if (profile_pic) + args.domain = attachment::Domain::PROFILE_PIC; + if (max_ttl) + args.ttl.reset(); + else + args.ttl.emplace(ttl_seconds); + + // Set up logging + { + constexpr std::array print_vals = {"stdout"sv, "-"sv, ""sv, "stderr"sv}; + namespace log = oxen::log; + auto log_type = std::count(print_vals.begin(), print_vals.end(), log_file) + ? log::Type::Print + : log_file == "syslog" ? log::Type::System + : log::Type::File; + log::add_sink(log_type, log_file); + + auto cats = log::extract_categories(log_level); + cats.apply(); + } + + return run(args, upload_cmd->parsed()); +} diff --git a/tests/schema/000_ext_thing.sql b/tests/schema/000_ext_thing.sql new file mode 100644 index 000000000..a6d1fb633 --- /dev/null +++ b/tests/schema/000_ext_thing.sql @@ -0,0 +1,3 @@ +CREATE TABLE ext_thing ( + id INTEGER PRIMARY KEY NOT NULL +) STRICT; diff --git a/tests/schema/000_globals.sql b/tests/schema/000_globals.sql new file mode 100644 index 000000000..c03a480f7 --- /dev/null +++ b/tests/schema/000_globals.sql @@ -0,0 +1,3 @@ +CREATE TABLE ext_globals ( + id INTEGER PRIMARY KEY NOT NULL +) STRICT; diff --git a/tests/schema/001_ordering+002.sql b/tests/schema/001_ordering+002.sql new file mode 100644 index 000000000..58c8970b0 --- /dev/null +++ b/tests/schema/001_ordering+002.sql @@ -0,0 +1,7 @@ +-- Addendum to 001_ordering.sql, exercising the branch workflow: iterate with `+NNN` files, then +-- squash them into the base before merging. +-- +-- The name is deliberately one where '+' (0x2B) sorts below '.' (0x2E), so sorting by *filename* +-- would run this before the table it alters exists and the migration would fail outright. That +-- makes this file the regression test for ordering by migration name instead. +ALTER TABLE ext_ordering ADD COLUMN added_later INTEGER; diff --git a/tests/schema/001_ordering.sql b/tests/schema/001_ordering.sql new file mode 100644 index 000000000..ce7fd65be --- /dev/null +++ b/tests/schema/001_ordering.sql @@ -0,0 +1,4 @@ +-- Base migration for the prefix-ordering check; see 001_ordering+002.sql. +CREATE TABLE ext_ordering ( + id INTEGER PRIMARY KEY NOT NULL +) STRICT; diff --git a/tests/schema/CMakeLists.txt b/tests/schema/CMakeLists.txt new file mode 100644 index 000000000..979da022b --- /dev/null +++ b/tests/schema/CMakeLists.txt @@ -0,0 +1,5 @@ +session_schema_dir( + TARGET testAll + NAMESPACE session::test::schema + DECLARE_HEADER test_schema_registry.hpp +) diff --git a/tests/schema/full_schema.sql b/tests/schema/full_schema.sql new file mode 100644 index 000000000..84c1fd64f --- /dev/null +++ b/tests/schema/full_schema.sql @@ -0,0 +1,16 @@ +-- The schema produced by every migration in this directory, used to build a fresh database in one +-- step. ext_ordering is the interesting one: the chain reaches it as a CREATE plus a later ALTER, +-- so the stored DDL text differs from this even though the schema does not -- which is what the +-- drift test in test_core_schema.cpp compares structurally rather than textually. +CREATE TABLE ext_thing ( + id INTEGER PRIMARY KEY NOT NULL +) STRICT; + +CREATE TABLE ext_globals ( + id INTEGER PRIMARY KEY NOT NULL +) STRICT; + +CREATE TABLE ext_ordering ( + id INTEGER PRIMARY KEY NOT NULL, + added_later INTEGER +) STRICT; diff --git a/tests/schema_fingerprint.hpp b/tests/schema_fingerprint.hpp new file mode 100644 index 000000000..d6e46c018 --- /dev/null +++ b/tests/schema_fingerprint.hpp @@ -0,0 +1,175 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +/// A normalised description of a database's schema, for asserting that two ways of arriving at one +/// schema agree -- in practice, building from full_schema.sql versus replaying the migrations. +/// +/// Comparing sqlite_master.sql text directly does not work: ALTER TABLE ADD COLUMN appends to the +/// *stored* CREATE TABLE text, so a migrated database's DDL differs from a hand-written declaration +/// of the same columns. The pragmas describe the schema as SQLite understands it, which is what we +/// actually care about. +/// +/// The pragmas do not cover everything, though. CHECK constraints have no pragma at all, partial +/// indexes report only that they are partial and not on what, and trigger bodies exist solely as +/// text. Those three are recovered from sqlite_master.sql, normalised for whitespace and comments, +/// which is safe because ALTER TABLE does not rewrite an expression's own text. +namespace session::test { + +namespace detail { + + /// Collapses whitespace runs and strips `--` comments, so formatting differences between a + /// hand-written declaration and one SQLite rewrote do not register as schema differences. + inline std::string normalise_sql(std::string_view sql) { + std::string out; + out.reserve(sql.size()); + bool space_pending = false; + for (size_t i = 0; i < sql.size(); i++) { + if (sql[i] == '-' && i + 1 < sql.size() && sql[i + 1] == '-') { + while (i < sql.size() && sql[i] != '\n') + i++; + space_pending = !out.empty(); + continue; + } + if (std::isspace(static_cast(sql[i]))) { + space_pending = !out.empty(); + continue; + } + if (space_pending) { + out += ' '; + space_pending = false; + } + out += sql[i]; + } + return out; + } + + /// Extracts every CHECK constraint from a CREATE TABLE statement, by balancing parentheses from + /// the one that opens each `CHECK(`. Does not attempt to skip string literals, so a CHECK + /// containing an unbalanced paren inside quotes would confuse it; none does. + inline std::vector extract_checks(std::string_view sql) { + auto norm = normalise_sql(sql); + auto upper = norm; + std::ranges::transform(upper, upper.begin(), [](unsigned char ch) { + return static_cast(std::toupper(ch)); + }); + + std::vector checks; + for (size_t pos = upper.find("CHECK"); pos != std::string::npos; + pos = upper.find("CHECK", pos + 1)) { + auto open = norm.find('(', pos); + if (open == std::string::npos) + continue; + int depth = 0; + for (size_t i = open; i < norm.size(); i++) { + if (norm[i] == '(') + depth++; + else if (norm[i] == ')' && --depth == 0) { + checks.push_back(norm.substr(open, i - open + 1)); + break; + } + } + } + std::ranges::sort(checks); + return checks; + } + +} // namespace detail + +/// Builds the fingerprint. The result is deterministic and diffable: on mismatch, Catch2 prints +/// both, and the differing line names the object. +inline std::string schema_fingerprint(sqlite::Connection& c) { + using namespace session::literals; + std::string out; + + auto objects = [&](std::string_view type) { + std::vector> rows; + for (auto [name, sql] : c.prepared_results>( + "SELECT name, sql FROM sqlite_master WHERE type = ?" + " AND name NOT LIKE 'sqlite_%' ORDER BY name", + type)) + rows.emplace_back(std::move(name), sql.value_or("")); + return rows; + }; + + for (const auto& [table, sql] : objects("table")) { + out += "table {}\n"_format(table); + + for (auto [cid, name, type, notnull, dflt, pk, hidden] : + c.prepared_results< + int64_t, + std::string, + std::string, + int, + std::optional, + int, + int>("PRAGMA table_xinfo({})"_format(table))) + out += " col {} {} notnull={} default={} pk={} hidden={}\n"_format( + name, type, notnull, dflt.value_or("-"), pk, hidden); + + for (const auto& check : detail::extract_checks(sql)) + out += " check {}\n"_format(check); + + for (auto [id, seq, ref_table, from, to, on_update, on_delete, match] : + c.prepared_results< + int64_t, + int64_t, + std::string, + std::string, + std::optional, + std::string, + std::string, + std::string>("PRAGMA foreign_key_list({})"_format(table))) + out += " fk {} -> {}.{} on_update={} on_delete={}\n"_format( + from, ref_table, to.value_or("-"), on_update, on_delete); + + // Indexes are described by their columns rather than their names, because an index implied + // by a UNIQUE constraint is named positionally (sqlite_autoindex__N) and those + // numbers shift if the constraints are declared in a different order. + std::vector indexes; + for (auto [seq, name, uniq, origin, partial] : + c.prepared_results( + "PRAGMA index_list({})"_format(table))) { + std::string desc = " index unique={} origin={} on"_format(uniq, origin); + for (auto [iseq, cid, col, rev, coll, key] : + c.prepared_results< + int64_t, + int64_t, + std::optional, + int, + std::string, + int>("PRAGMA index_xinfo({})"_format(name))) + if (key) + desc += " {}{}/{}"_format(col.value_or(""), rev ? " DESC" : "", coll); + + if (partial) { + // index_list only says *that* it is partial; the predicate is in the DDL. + auto ddl = c.prepared_get>( + "SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", name); + auto norm = detail::normalise_sql(ddl.value_or("")); + if (auto w = norm.find(" WHERE "); w != std::string::npos) + desc += " where{}"_format(norm.substr(w + 6)); + } + indexes.push_back(std::move(desc)); + } + std::ranges::sort(indexes); + for (const auto& i : indexes) + out += i + "\n"; + } + + for (const auto& [name, sql] : objects("trigger")) + out += "trigger {} {}\n"_format(name, detail::normalise_sql(sql)); + + for (const auto& [name, sql] : objects("view")) + out += "view {} {}\n"_format(name, detail::normalise_sql(sql)); + + return out; +} + +} // namespace session::test diff --git a/tests/schema_history_check.sh b/tests/schema_history_check.sh new file mode 100755 index 000000000..2d874c059 --- /dev/null +++ b/tests/schema_history_check.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Checks that a database created by any previously published version upgrades cleanly to the +# current schema. +# +# full_schema.sql is the only thing that creates a schema; the migrations beside it are deltas from +# an older full_schema. There is therefore no way to build a database by replaying migrations from +# nothing, and the only starting points that exist are earlier versions of full_schema.sql — which +# exist solely in git history. That is what this walks. +# +# Starting points are every tag, plus every commit since the most recent tag (so unreleased work in +# progress is covered too, not just published versions), and every revision after SCHEMA_FLOOR. +# Revisions whose schemas are byte-identical to one already checked are skipped: a version that +# changed no schema tells us nothing new, and without this every tag in a quiet period would re-run +# the same check. +# +# Usage: tests/schema_history_check.sh + +set -euo pipefail + +CHECKER=${1:-} +if [[ -z "$CHECKER" || ! -x "$CHECKER" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +cd "$(dirname "$0")/.." + +CORE_DIR=src/core/schema +CLIENT_DIR=src/client/schema + +# Exemptions, one revision per line with a reason, for commits known to carry a broken schema (a +# work-in-progress state that was never released). Kept as a file so removing something from it is +# a reviewable change rather than a silent edit to the walk. +SKIPLIST=tests/schema_history_skip.txt + +# The oldest revision that is *not* a usable starting point, and by extension neither is anything +# behind it. +# +# A schema is described by full_schema.sql alone, so a change to one that ships without a migration +# leaves no way to carry an existing database across it: the database has to be recreated. Every +# revision before such a change therefore describes a database that cannot reach today's schema, and +# checking that it does is checking something we deliberately gave up. +# +# Bump this to the then-current HEAD each time that happens. It is a single revision rather than a +# list because the property is a cutoff: once one database has to be recreated, so does every older +# one. Everything after it is still walked, which is what keeps a development branch honest between +# cutoffs -- consecutive commits do have to upgrade cleanly. +SCHEMA_FLOOR=33812a80035dbfd500f7c7d030699697bfc10cd0 + +revs=$( + git tag + last_tag=$(git describe --tags --abbrev=0 2>/dev/null || true) + if [[ -n "$last_tag" ]]; then + git rev-list "$last_tag..HEAD" + else + git rev-list HEAD + fi +) + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +declare -A seen +checked=0 +skipped=0 + +for rev in $revs; do + if [[ -f "$SKIPLIST" ]] && grep -q "^$(git rev-parse --short "$rev")" "$SKIPLIST" 2>/dev/null; then + continue + fi + + # `--is-ancestor` counts a commit as its own ancestor, so this drops the floor itself too. + if git merge-base --is-ancestor "$rev" "$SCHEMA_FLOOR" 2>/dev/null; then + continue + fi + + core_sql=$(git show "$rev:$CORE_DIR/full_schema.sql" 2>/dev/null || true) + # A revision predating full_schema.sql has no starting point to offer. + [[ -z "$core_sql" ]] && continue + + client_sql=$(git show "$rev:$CLIENT_DIR/full_schema.sql" 2>/dev/null || true) + + # Dedupe on the schemas themselves, not the revision: consecutive releases usually share one. + key=$(printf '%s\0%s' "$core_sql" "$client_sql" | sha256sum | cut -d' ' -f1) + if [[ -n "${seen[$key]:-}" ]]; then + skipped=$((skipped + 1)) + continue + fi + seen[$key]=$rev + + printf '%s' "$core_sql" > "$tmp/core.sql" + args=(--core-schema "$tmp/core.sql") + if [[ -n "$client_sql" ]]; then + printf '%s' "$client_sql" > "$tmp/client.sql" + args+=(--client-schema "$tmp/client.sql") + fi + + # The migrations that existed then are what a database of that era would have recorded; anything + # added since is what must now run. + while read -r f; do + [[ -z "$f" ]] && continue + args+=(--applied "$(basename "$f" | sed -E 's/\.(sql|cpp)$//')") + done < <(git ls-tree --name-only "$rev" "$CORE_DIR/" | grep -E '/[0-9][^/]*\.(sql|cpp)$' || true) + + while read -r f; do + [[ -z "$f" ]] && continue + args+=(--applied "client:$(basename "$f" | sed -E 's/\.(sql|cpp)$//')") + done < <(git ls-tree --name-only "$rev" "$CLIENT_DIR/" | grep -E '/[0-9][^/]*\.(sql|cpp)$' || true) + + echo "checking upgrade from $(git describe --tags --always "$rev")" + if ! "$CHECKER" "${args[@]}"; then + echo "FAILED: a database created at $rev does not upgrade to the current schema" >&2 + exit 1 + fi + checked=$((checked + 1)) +done + +echo "schema history check: $checked starting point(s) verified, $skipped duplicate(s) skipped" +if (( checked == 0 )); then + echo "note: no revision yet carries a full_schema.sql to upgrade from" >&2 +fi diff --git a/tests/schema_history_skip.txt b/tests/schema_history_skip.txt new file mode 100644 index 000000000..d06f126e8 --- /dev/null +++ b/tests/schema_history_skip.txt @@ -0,0 +1,7 @@ +# Revisions to skip in schema_history_check.sh, one short hash per line with a reason. +# +# For commits whose full_schema.sql was broken or mid-rework and was never released, so no database +# can have been created from it. A file rather than a flag so that removing an entry is a +# reviewable change rather than a silent one. +# +# Format: diff --git a/tests/schema_upgrade_check.cpp b/tests/schema_upgrade_check.cpp new file mode 100644 index 000000000..d4de7d6f1 --- /dev/null +++ b/tests/schema_upgrade_check.cpp @@ -0,0 +1,170 @@ +/// Checks that a database created from an *older* full_schema.sql upgrades to the current schema. +/// +/// full_schema.sql is the only thing that creates a schema; the migrations beside it are deltas +/// from an older full_schema. So the migration chain cannot be replayed from nothing, and the only +/// starting points that exist are previous versions of full_schema.sql — which live in git history. +/// schema_history_check.sh digs them out and invokes this for each. +/// +/// Given those historical files, this builds a database as that version would have, opens it with +/// the current code so any migrations since then run, and compares the result against a database +/// freshly created from today's full_schema. Those two must agree, or an upgraded install and a +/// new one are running different schemas. +/// +/// schema-upgrade-check --core-schema FILE [--client-schema FILE] [--applied KEY]... +/// +/// --applied takes fully-qualified names as recorded in migrations_applied ("001_foo" for Core, +/// "client:001_foo" for the extension): the migrations that existed at that revision, which the +/// database of that era would have had recorded. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "schema_fingerprint.hpp" + +namespace { + +std::string read_file(const std::filesystem::path& p) { + std::ifstream f{p, std::ios::binary}; + if (!f) + throw std::runtime_error{fmt::format("cannot read {}", p.string())}; + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +std::filesystem::path temp_db(std::string_view tag) { + auto p = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id(std::string{tag}, 8)); + std::filesystem::remove(p); + return p; +} + +/// Recreates the database an older release would have had: its schema, and the bookkeeping rows +/// recording what it considered applied. Deliberately does not go through Core, since Core at HEAD +/// would create *today's* schema. +void build_historical_db( + const std::filesystem::path& path, + const std::string& core_schema, + const std::optional& client_schema, + const std::vector& applied) { + session::sqlite::Database db{path}; + auto conn = db.conn(); + + session::sqlite::exec_query( + conn.sql, + "CREATE TABLE IF NOT EXISTS migrations_applied (name TEXT PRIMARY KEY NOT NULL) " + "STRICT"); + + conn.sql.exec(core_schema); + conn.prepared_exec("INSERT INTO migrations_applied (name) VALUES (?)", "@created"); + + if (client_schema) { + conn.sql.exec(*client_schema); + conn.prepared_exec("INSERT INTO migrations_applied (name) VALUES (?)", "client:@created"); + } + + for (const auto& key : applied) + conn.prepared_exec("INSERT OR IGNORE INTO migrations_applied (name) VALUES (?)", key); +} + +} // namespace + +int main(int argc, char** argv) { + std::optional core_schema_path, client_schema_path; + std::vector applied; + + for (int i = 1; i < argc; i++) { + std::string_view arg{argv[i]}; + auto next = [&]() -> std::string { + if (++i >= argc) + throw std::runtime_error{fmt::format("{} requires a value", arg)}; + return argv[i]; + }; + if (arg == "--core-schema") + core_schema_path = next(); + else if (arg == "--client-schema") + client_schema_path = next(); + else if (arg == "--applied") + applied.push_back(next()); + else { + std::cerr << "unrecognised argument: " << arg << "\n"; + return 2; + } + } + + if (!core_schema_path) { + std::cerr << "--core-schema is required\n"; + return 2; + } + + try { + auto old_path = temp_db("schema_old"); + auto fresh_path = temp_db("schema_fresh"); + + build_historical_db( + old_path, + read_file(*core_schema_path), + client_schema_path ? std::optional{read_file(*client_schema_path)} : std::nullopt, + applied); + + // Opening with the current code runs whatever migrations that era's database is missing. + std::string upgraded, fresh; + { + session::client::Client c{old_path}; + auto conn = c.core.database().conn(); + upgraded = session::test::schema_fingerprint(conn); + } + { + session::client::Client c{fresh_path}; + auto conn = c.core.database().conn(); + fresh = session::test::schema_fingerprint(conn); + } + + std::error_code ec; + std::filesystem::remove(old_path, ec); + std::filesystem::remove(fresh_path, ec); + + if (upgraded == fresh) + return 0; + + std::cerr << "schema mismatch after upgrade\n"; + + // Reported as a set difference rather than line by line: one missing line would otherwise + // shift everything after it and report a single fault as dozens. Both fingerprints are + // produced by the same function over the same objects, so their line order cannot differ + // except as a consequence of content differing anyway. + auto lines = [](const std::string& s) { + std::vector out; + std::istringstream in{s}; + for (std::string l; std::getline(in, l);) + out.push_back(l); + std::ranges::sort(out); + return out; + }; + auto up = lines(upgraded), fr = lines(fresh); + + std::vector missing, extra; + std::ranges::set_difference(fr, up, std::back_inserter(missing)); + std::ranges::set_difference(up, fr, std::back_inserter(extra)); + + for (const auto& l : missing) + std::cerr << fmt::format(" missing after upgrade: {}\n", l); + for (const auto& l : extra) + std::cerr << fmt::format(" unexpected after upgrade: {}\n", l); + return 1; + } catch (const std::exception& e) { + std::cerr << "schema-upgrade-check failed: " << e.what() << "\n"; + return 1; + } +} diff --git a/tests/static_bundle.cpp b/tests/static_bundle.cpp index a760a178d..b90084b1c 100644 --- a/tests/static_bundle.cpp +++ b/tests/static_bundle.cpp @@ -7,6 +7,6 @@ int main() { if (std::mt19937_64{}() == 123) { auto& k = *reinterpret_cast(12345); - k.encrypt_message(std::span{}); + k.encrypt_message(std::span{}); } } diff --git a/tests/swarm-auth-test.cpp b/tests/swarm-auth-test.cpp index 98c565f4b..05d3bb756 100644 --- a/tests/swarm-auth-test.cpp +++ b/tests/swarm-auth-test.cpp @@ -26,16 +26,16 @@ static constexpr int64_t created_ts = 1680064059; using namespace session::config; -static std::array sk_from_seed(std::span seed) { - std::array ignore; - std::array sk; +static b64 sk_from_seed(std::span seed) { + b32 ignore; + b64 sk; crypto_sign_ed25519_seed_keypair(ignore.data(), sk.data(), seed.data()); return sk; } -static std::string session_id_from_ed(std::span ed_pk) { +static std::string session_id_from_ed(std::span ed_pk) { std::string sid; - std::array xpk; + b32 xpk; int rc = crypto_sign_ed25519_pk_to_curve25519(xpk.data(), ed_pk.data()); assert(rc == 0); sid.reserve(66); @@ -45,8 +45,8 @@ static std::string session_id_from_ed(std::span ed_pk) { } struct pseudo_client { - std::array secret_key; - const std::span public_key{secret_key.data() + 32, 32}; + b64 secret_key; + const std::span public_key{secret_key.data() + 32, 32}; std::string session_id{session_id_from_ed(public_key)}; groups::Info info; @@ -54,23 +54,21 @@ struct pseudo_client { groups::Keys keys; pseudo_client( - std::span seed, + std::span seed, bool admin, const unsigned char* gpk, std::optional gsk) : secret_key{sk_from_seed(seed)}, - info{std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) - : std::nullopt, + info{std::span{gpk, 32}, + admin ? std::make_optional>({*gsk, 64}) : std::nullopt, std::nullopt}, - members{std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) + members{std::span{gpk, 32}, + admin ? std::make_optional>({*gsk, 64}) : std::nullopt, std::nullopt}, keys{to_usv(secret_key), - std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) - : std::nullopt, + std::span{gpk, 32}, + admin ? std::make_optional>({*gsk, 64}) : std::nullopt, std::nullopt, info, members} {} @@ -78,15 +76,15 @@ struct pseudo_client { int main() { - const std::vector group_seed = - "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hexbytes; - const std::vector admin_seed = - "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - const std::vector member_seed = - "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hexbytes; + const std::vector group_seed = + "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hex_b; + const std::vector admin_seed = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + const std::vector member_seed = + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hex_b; - std::array group_pk; - std::array group_sk; + b32 group_pk; + b64 group_sk; crypto_sign_ed25519_seed_keypair(group_pk.data(), group_sk.data(), group_seed.data()); @@ -104,7 +102,7 @@ int main() { session::config::UserGroups member_gr2{member_seed, std::nullopt}; auto [seqno, push, obs] = member_groups.push(); - std::vector>> gr_conf; + std::vector>> gr_conf; gr_conf.emplace_back("fakehash1", push); member_gr2.merge(gr_conf); @@ -114,8 +112,8 @@ int main() { .count(); auto msg = to_usv("hello world"); - std::array store_sig; - std::vector store_to_sign; + b64 store_sig; + std::vector store_to_sign; auto store_vec = session::str_to_vec("store999{}"_format(now)); store_to_sign.insert(store_to_sign.end(), store_vec.begin(), store_vec.end()); @@ -134,7 +132,7 @@ int main() { std::cout << "STORE:\n\n" << store.dump() << "\n\n"; - std::vector retrieve_to_sign; + std::vector retrieve_to_sign; auto retrieve_vec = session::str_to_vec("retrieve999{}"_format(now)); retrieve_to_sign.insert(retrieve_to_sign.end(), retrieve_vec.begin(), retrieve_vec.end()); auto subauth = member.keys.swarm_subaccount_sign(retrieve_to_sign, auth_data); diff --git a/tests/test_attachment_encrypt.cpp b/tests/test_attachment_encrypt.cpp index 684fad8d5..800a5becc 100644 --- a/tests/test_attachment_encrypt.cpp +++ b/tests/test_attachment_encrypt.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "utils.hpp" @@ -255,7 +256,7 @@ TEST_CASE("Attachment file encryption validates its inputs", "[attachments][file attachment::encrypt( std::span{seed}.first<31>(), f.path, attachment::Domain::ATTACHMENT), std::invalid_argument, - Message("attachment::encrypt requires a 32-byte uploader seed")); + Message("attachment::Encryptor requires a 32-byte uploader seed")); std::filesystem::resize_file(f.path, attachment::MAX_REGULAR_SIZE + 1); CHECK_THROWS_MATCHES( @@ -481,3 +482,312 @@ TEST_CASE( CHECK_FALSE(std::filesystem::exists(out.path)); } } + +TEST_CASE("Streaming Encryptor", "[attachments][encryptor]") { + + auto DATA_SIZE = GENERATE(0, 1, 100, 1000, 4053, 8150, 32768, 65536, 100000); + + auto seed = "9123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; + const auto data = make_data(DATA_SIZE); + + SECTION("pull-based encryption with manual source") { + attachment::Encryptor enc{seed, attachment::Domain::ATTACHMENT}; + + // Phase 1: feed data in chunks to derive key + for (size_t pos = 0; pos < data.size();) { + size_t chunk = std::min(1000, data.size() - pos); + enc.update_key(std::span{data}.subspan(pos, chunk)); + pos += chunk; + } + if (data.empty()) + enc.update_key({}); + + // Phase 2: start encryption with a pull source + size_t src_pos = 0; + auto key = enc.start_encryption([&](std::span buf) -> size_t { + size_t avail = std::min(buf.size(), data.size() - src_pos); + std::memcpy(buf.data(), data.data() + src_pos, avail); + src_pos += avail; + return avail; + }); + + // Collect all encrypted output + std::vector encrypted; + while (true) { + auto chunk = enc.next(); + if (chunk.empty()) + break; + encrypted.insert(encrypted.end(), chunk.begin(), chunk.end()); + } + + CHECK(encrypted.size() == attachment::encrypted_size(DATA_SIZE)); + + // Decrypt with the streaming Decryptor and verify round-trip + std::vector decrypted; + attachment::Decryptor dec{key, [&](std::span d) { + decrypted.insert(decrypted.end(), d.begin(), d.end()); + }}; + REQUIRE(dec.update(encrypted)); + REQUIRE(dec.finalize()); + REQUIRE(decrypted.size() == data.size()); + CHECK(!!(decrypted == data)); + } + + SECTION("from_file factory") { + if (DATA_SIZE == 0) + return; // Can't write an empty file for this test + + // Write test data to a temp file + temp_data_file tmp; + { + std::ofstream f{tmp.path, std::ios::binary}; + f.write(reinterpret_cast(data.data()), data.size()); + } + + auto [enc, key] = attachment::Encryptor::from_file( + seed, attachment::Domain::ATTACHMENT, tmp.path, true); + + std::vector encrypted; + while (true) { + auto chunk = enc.next(); + if (chunk.empty()) + break; + encrypted.insert(encrypted.end(), chunk.begin(), chunk.end()); + } + + CHECK(encrypted.size() == attachment::encrypted_size(DATA_SIZE)); + + // Decrypt and verify + auto decrypted = attachment::decrypt(encrypted, key); + REQUIRE(decrypted.size() == data.size()); + CHECK(!!(decrypted == data)); + } +} + +// -- Legacy (AES-CBC + HMAC) attachments --------------------------------------------------------- +// +// The scheme every Session client still *sends*, inherited from libsignal: 32-byte AES key followed +// by a 32-byte HMAC key, and a file laid out as IV || AES-256-CBC(PKCS#7) || HMAC-SHA256(IV||ct), +// with the AttachmentPointer's `digest` being SHA-256 over all three. +// +// The vectors below were produced by an independent implementation (python-cryptography) written +// from session-android's AttachmentCipherInputStream, so this is a known-answer test rather than a +// round trip against ourselves -- which would prove nothing, since libsession deliberately has no +// legacy *encryptor*. + +using namespace oxenc::literals; + +namespace { + +constexpr auto LEGACY_KEY = + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f"_hex_b; + +// "the quick brown fox jumps over the lazy dog, repeatedly and at length." -- 70 bytes, with no +// Session-level zero padding, so the pointer's size is the plaintext length. +constexpr auto LEGACY_BLOB = + "6465666768696a6b6c6d6e6f7071727339c6cadce50e036612224f681bbbe3f3" + "1acc5779dfe5367b18c6272231f4eb139a9b56725e81236c469883304bc53999" + "311ae9c035bf6ed5d2fbd6fb24777de2d1368b650f24d5c454208af7610238a9" + "ff4892a7b4b5e54a9b99e78d73d65f335fc5c5559dd3d4c894401d0c7f7ce95b"_hex_b; +constexpr auto LEGACY_DIGEST = + "68af9ac56f2c7d90a984f9edccdd538765b0e3ba2c1958ce7c0071212c0beccb"_hex_b; +constexpr auto LEGACY_PLAINTEXT = + "the quick brown fox jumps over the lazy dog, repeatedly and at length."sv; + +// 20 real bytes zero-padded out to 200, which is what the pointer's size is actually for. +constexpr auto PADDED_BLOB = + "6465666768696a6b6c6d6e6f70717273e85b4762c96dc9f8ec01d8cce057ce81" + "f52ba3a6d5b7b61214a1000d827532b1c36cc1beb5454eb154e159508cd627f5" + "349d55e9a583df1ad46401d07805608c89ba1ca437e93067b91a18efc3af88e7" + "8247d47fb3562ee73b5e9de5c538caebeeb6e36787f98745414735371311d90a" + "6a169349eb0ecc93b5b45076f95daef493d7663ef1e538e7b28c60aff8e5528c" + "c221d7c1e1ce092184b140a68a9adfe1738ae283e50701d83e370d75f57e6f3f" + "7fcb2a0f410ac34187e414c3bb6d5cf38399b89cafa402e061c675ab409b9eb0" + "849ead65dbccc86863b3b1817d37cd6eb8b50dc259e90fbb4971620dd09eabb9"_hex_b; +constexpr auto PADDED_DIGEST = + "da2680f942213633344c39c0690c652beed7e1c50f6ceef45e113dc0f007fd33"_hex_b; + +auto legacy_key() { + return std::span{LEGACY_KEY}; +} +auto legacy_digest() { + return std::span{LEGACY_DIGEST}; +} +auto padded_digest() { + return std::span{PADDED_DIGEST}; +} + +} // namespace + +TEST_CASE("legacy attachment decryption", "[attachments][legacy]") { + auto out = attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size()); + CHECK(session::to_string_view(out) == LEGACY_PLAINTEXT); + + // A sender too old to set the field leaves the zero padding in place rather than having it + // guessed at, so what comes back is the whole PKCS#7-stripped plaintext. + auto untrimmed = attachment::legacy_decrypt(PADDED_BLOB, legacy_key(), padded_digest(), 0); + CHECK(untrimmed.size() == 200); + + // ...and with the field set, only the real bytes. + auto trimmed = attachment::legacy_decrypt(PADDED_BLOB, legacy_key(), padded_digest(), 20); + REQUIRE(trimmed.size() == 20); + CHECK(session::to_string_view(trimmed) == "twenty bytes exactly"sv); +} + +TEST_CASE("legacy attachment decryption rejects bad input", "[attachments][legacy]") { + auto tampered = [](std::span blob, size_t at) { + std::vector v{blob.begin(), blob.end()}; + v[at] ^= std::byte{0x01}; + return v; + }; + + // A flipped bit anywhere in the file fails the HMAC, whether it lands in the IV, the ciphertext + // or the MAC itself. Nothing is decrypted before that check. + for (size_t at : {size_t{0}, size_t{20}, LEGACY_BLOB.size() - 1}) + CHECK_THROWS(attachment::legacy_decrypt( + tampered(LEGACY_BLOB, at), legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size())); + + // A correct file with a wrong digest is *accepted*, deliberately: we do not verify it. The + // HMAC checked above covers the same bytes under a key only the sender has, so anything a bad + // digest could catch has already failed -- see the reasoning where that check is commented out. + // + // Asserted rather than left implicit because it is a decision, and the natural instinct on + // finding an unverified authenticator is to start verifying it. + auto wrong_digest = tampered(LEGACY_DIGEST, 5); + CHECK(attachment::legacy_decrypt( + LEGACY_BLOB, + legacy_key(), + std::span{wrong_digest}, + LEGACY_PLAINTEXT.size()) == to_vector(LEGACY_PLAINTEXT)); + + // The one a hostile sender controls directly: a size larger than what was decrypted. Refused + // rather than clamped, since the pointer is then lying about its own file. + CHECK_THROWS(attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size() + 1)); + CHECK_THROWS(attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), std::numeric_limits::max())); + + // Too short to hold an IV, a block and a MAC. + CHECK_THROWS(attachment::legacy_decrypt( + LEGACY_BLOB.subspan(0, 40), legacy_key(), legacy_digest(), 1)); + + // Not a whole number of cipher blocks, so it cannot be what a CBC encryptor produced. + CHECK_THROWS(attachment::legacy_decrypt( + LEGACY_BLOB.subspan(0, LEGACY_BLOB.size() - 1), legacy_key(), legacy_digest(), 1)); +} + +TEST_CASE("Attachment encryption -- a key of our own", "[attachments][fixed-key]") { + // Encrypting to our own disk rather than to a file server: the key is ours, kept once and + // reused, so nothing about the content decides it. + cleared_b32 cache_key; + session::random::fill(cache_key); + + std::vector plaintext(70'000); + session::random::fill(plaintext); + + auto encrypt_with = [&](std::span data) { + attachment::Encryptor enc{cache_key}; + size_t pos = 0; + enc.start_encryption( + [&](std::span buf) -> size_t { + auto n = std::min(buf.size(), data.size() - pos); + std::memcpy(buf.data(), data.data() + pos, n); + pos += n; + return n; + }, + false, + data.size()); + + std::vector out; + for (auto chunk = enc.next(); !chunk.empty(); chunk = enc.next()) + out.insert(out.end(), chunk.begin(), chunk.end()); + return out; + }; + + auto encrypted = encrypt_with(plaintext); + + // Reads back with the ordinary decrypt: same format, so there is one decryptor, not two. + CHECK(attachment::decrypt(encrypted, cache_key) == plaintext); + + // Not deterministic, which is the point of the random nonce: the seed-based encryptor + // deliberately repeats itself so a file server can deduplicate, and repeating a keystream + // under one key across every cached file is the failure that would cause here. + auto again = encrypt_with(plaintext); + CHECK(again != encrypted); + CHECK(attachment::decrypt(again, cache_key) == plaintext); + + // Another key does not open it. + cleared_b32 other; + session::random::fill(other); + CHECK_THROWS(attachment::decrypt(encrypted, other)); + + // Padded exactly as the seed-based path is: a local disk ends up in backups and disk images, + // and an exact size identifies a file as well there as on a file server. + CHECK(encrypted.size() == attachment::encrypted_size(plaintext.size())); + + // Phase 1 has no meaning here, and asking for it is a mistake rather than a no-op. + attachment::Encryptor enc{cache_key}; + CHECK_THROWS_AS(enc.update_key(plaintext), std::logic_error); + // ...and without phase 1 nothing knows how much is coming, so the size is required. + CHECK_THROWS_AS( + enc.start_encryption([](std::span) -> size_t { return 0; }), + std::invalid_argument); +} + +TEST_CASE("Display picture decryption -- the legacy GCM scheme", "[attachments][legacy-pic]") { + // Session has three at-rest formats and nothing in the bytes says which is which. This is the + // one display pictures used before the stream scheme: AES-256-GCM, 32-byte key, nonce and tag + // carried in the data -- unrelated to the legacy *attachment* scheme, which is CBC with a + // bolted-on HMAC, a 64-byte key and a digest carried in the protobuf. + // + // The vector is from python-cryptography rather than from our own encryptor, so this cannot + // pass by agreeing with itself. + constexpr auto key = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"_hex_b; + constexpr auto blob = + "000102030405060708090a0b2622b272b695ae7af461e7e2d29d0d1fe6faa7559c173a1b5d0389fc903a" + "65df450ae8c0b3070d5d413b790c"_hex_b; + constexpr auto expected = "a display picture, allegedly"sv; + + auto plain = attachment::legacy_display_pic_decrypt(blob, key); + CHECK(std::string_view{reinterpret_cast(plain.data()), plain.size()} == expected); + + // A flipped bit anywhere fails on the tag rather than yielding rubbish. + std::vector tampered{blob.begin(), blob.end()}; + tampered[20] ^= std::byte{0x01}; + CHECK_THROWS(attachment::legacy_display_pic_decrypt(tampered, key)); + + // As does the wrong key. + constexpr auto wrong = "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"_hex_b; + CHECK_THROWS(attachment::legacy_display_pic_decrypt(blob, wrong)); + + // Too short to hold a nonce and a tag is refused rather than read past. + CHECK_THROWS(attachment::legacy_display_pic_decrypt(blob.subspan(0, 27), key)); +} + +TEST_CASE( + "legacy attachment decryption holds the sender to the size they claimed", + "[attachments][legacy][size]") { + // Over-reporting has always failed: the claim exceeds what came out. + CHECK_THROWS(attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size() + 1)); + + // Under-reporting is *not* caught, and cannot be: the bytes past the claim would have to be + // recognisable as padding, and session-android's padding is whatever its read buffer happened + // to contain (PaddingInputStream reports bulk padding without writing it). So a sender who + // under-reports gets a silently truncated file, and the legacy format offers no way to tell. + CHECK(attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size() - 1) + .size() == LEGACY_PLAINTEXT.size() - 1); + + // The honest claim still works, and the padding that legitimately follows it is dropped. + CHECK(attachment::legacy_decrypt( + LEGACY_BLOB, legacy_key(), legacy_digest(), LEGACY_PLAINTEXT.size()) == + to_vector(LEGACY_PLAINTEXT)); + + // Zero still means "the sender never said", which only clients predating the field do, and + // leaves the padding in place rather than guessing. + CHECK(attachment::legacy_decrypt(LEGACY_BLOB, legacy_key(), legacy_digest(), 0).size() >= + LEGACY_PLAINTEXT.size()); +} diff --git a/tests/test_backend_session_file_server.cpp b/tests/test_backend_session_file_server.cpp index 63fd74eab..6979b2ce4 100644 --- a/tests/test_backend_session_file_server.cpp +++ b/tests/test_backend_session_file_server.cpp @@ -1,8 +1,8 @@ #include #include +#include #include -#include #include "utils.hpp" @@ -38,13 +38,13 @@ TEST_CASE("Download url parsing", "[backend][session_file_server]") { // Extracts the custom pubkey parsed_download_url = file_server::parse_download_url( - "https://example.com/file/abc123#p=0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "https://example.com/file/abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); REQUIRE(parsed_download_url.has_value()); CHECK(parsed_download_url->scheme == "https"sv); CHECK(parsed_download_url->host == "example.com"sv); CHECK(parsed_download_url->file_id == "abc123"sv); CHECK(parsed_download_url->custom_pubkey_hex == - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); CHECK_FALSE(parsed_download_url->wants_stream_decryption); // Ignores the pubkey if it matches the default one @@ -59,24 +59,24 @@ TEST_CASE("Download url parsing", "[backend][session_file_server]") { // Handles both fragments parsed_download_url = file_server::parse_download_url( - "https://example.com/file/abc123#p=0123456789abcdef0123456789abcdef00000000000000000000000000000000&d"sv); + "https://example.com/file/abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29&d"sv); REQUIRE(parsed_download_url.has_value()); CHECK(parsed_download_url->scheme == "https"sv); CHECK(parsed_download_url->host == "example.com"sv); CHECK(parsed_download_url->file_id == "abc123"sv); CHECK(parsed_download_url->custom_pubkey_hex == - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); CHECK(parsed_download_url->wants_stream_decryption); // Handles both fragments in the opposite order parsed_download_url = file_server::parse_download_url( - "https://example.com/file/abc123#d&p=0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "https://example.com/file/abc123#d&p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); REQUIRE(parsed_download_url.has_value()); CHECK(parsed_download_url->scheme == "https"sv); CHECK(parsed_download_url->host == "example.com"sv); CHECK(parsed_download_url->file_id == "abc123"sv); CHECK(parsed_download_url->custom_pubkey_hex == - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); CHECK(parsed_download_url->wants_stream_decryption); // A valueless `d=` is NOT the stream-encryption fragment. Session Desktop builds its fragment @@ -88,10 +88,10 @@ TEST_CASE("Download url parsing", "[backend][session_file_server]") { CHECK_FALSE(parsed_download_url->wants_stream_decryption); parsed_download_url = file_server::parse_download_url( - "https://example.com/file/abc123#p=0123456789abcdef0123456789abcdef00000000000000000000000000000000&d="sv); + "https://example.com/file/abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29&d="sv); REQUIRE(parsed_download_url.has_value()); CHECK(parsed_download_url->custom_pubkey_hex == - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv); CHECK_FALSE(parsed_download_url->wants_stream_decryption); // Doesn't have an issue with a legacy url @@ -104,65 +104,172 @@ TEST_CASE("Download url parsing", "[backend][session_file_server]") { CHECK_FALSE(parsed_download_url.has_value()); } +TEST_CASE("Download url rejects an unusable pubkey", "[backend][session_file_server]") { + // `p=` says which key to encrypt the request to, so a url carrying one we cannot use is not a + // url we can fall back on: ignoring the fragment would keep the url's host but quietly + // substitute our own file server's key, sending the request to a host that cannot read it. + // What makes a key usable is [ed25519][pubkey]'s subject; this only has to show the check is + // reached, alongside the shapes the url parser turns away before the key is even decoded. + auto rejected = GENERATE( + // Well-formed hex of the right length, but not a point on the curve + "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv, + // Too short, too long, not hex, and absent -- `p=` with nothing after it + "abc123"sv, + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da2900"sv, + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"sv, + ""sv); + + INFO("pubkey: " << rejected); + CHECK_FALSE(file_server::parse_download_url( + fmt::format("https://example.com/file/abc123#p={}", rejected)) + .has_value()); + + // A well-formed key is still accepted, so the check above is rejecting the key and not the url + CHECK(file_server::parse_download_url( + "https://example.com/file/abc123#p=" + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv) + .has_value()); +} + TEST_CASE("Download url generation", "[backend][session_file_server]") { auto url = file_server::generate_download_url( "abc123"sv, {"http", "example.com", 123, - "0123456789abcdef0123456789abcdef00000000000000000000000000000000", - 12345, - true}); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29", + 12345}, + true); CHECK(url == "http://example.com:123/file/" - "abc123#p=0123456789abcdef0123456789abcdef00000000000000000000000000000000&d"); + "abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29&d"); - // Omits the stream encryption fragment when disabled + // Omits the stream encryption fragment for a file encrypted the legacy way url = file_server::generate_download_url( "abc123"sv, {"http", "example.com", 123, - "0123456789abcdef0123456789abcdef00000000000000000000000000000000", - 12345, - false}); + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29", + 12345}, + false); CHECK(url == "http://example.com:123/file/" - "abc123#p=0123456789abcdef0123456789abcdef00000000000000000000000000000000"); + "abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"); // Omits the pubkey when it matches the default pubkey url = file_server::generate_download_url( "abc123"sv, - {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345, true}); + {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345}, + true); CHECK(url == "http://example.com:123/file/abc123#d"); - // Omits all fragments when stream encryption is disabled and the default pubkey is used + // Omits all fragments for a legacy-encrypted file on the default server url = file_server::generate_download_url( "abc123"sv, - {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345, false}); + {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345}, + false); CHECK(url == "http://example.com:123/file/abc123"); // Works with other values url = file_server::generate_download_url( "12345678"sv, - {"https", "example2.com", 321, file_server::DEFAULT_CONFIG.pubkey_hex, 54321, false}); + {"https", "example2.com", 321, file_server::DEFAULT_CONFIG.pubkey_hex, 54321}, + false); CHECK(url == "https://example2.com:321/file/12345678"); // Omits the port when the scheme already implies it, so urls for a default-port server are // unchanged from every previous version url = file_server::generate_download_url( "abc123"sv, - {"http", "example.com", 80, file_server::DEFAULT_CONFIG.pubkey_hex, 12345, false}); + {"http", "example.com", 80, file_server::DEFAULT_CONFIG.pubkey_hex, 12345}, + false); CHECK(url == "http://example.com/file/abc123"); url = file_server::generate_download_url( "abc123"sv, - {"https", "example.com", 443, file_server::DEFAULT_CONFIG.pubkey_hex, 12345, false}); + {"https", "example.com", 443, file_server::DEFAULT_CONFIG.pubkey_hex, 12345}, + false); CHECK(url == "https://example.com/file/abc123"); // The default file server is unaffected - url = file_server::generate_download_url("abc123"sv, file_server::DEFAULT_CONFIG); + url = file_server::generate_download_url("abc123"sv, file_server::DEFAULT_CONFIG, false); CHECK(url == fmt::format("http://{}/file/abc123", file_server::DEFAULT_CONFIG.host)); + + // Names a custom server's session router endpoint, leaving out the port when it is the default + // one that whoever parses this will assume anyway + url = file_server::generate_download_url( + "abc123"sv, + {"http", + "example.com", + 123, + file_server::DEFAULT_CONFIG.pubkey_hex, + 12345, + file_server::SRouterTarget{"somewhere.sesh"}}, + false); + CHECK(url == "http://example.com:123/file/abc123#sr=somewhere.sesh"); + + // ... but includes it when it isn't + url = file_server::generate_download_url( + "abc123"sv, + {"http", + "example.com", + 123, + file_server::DEFAULT_CONFIG.pubkey_hex, + 12345, + file_server::SRouterTarget{"somewhere.sesh", 4567}}, + false); + CHECK(url == "http://example.com:123/file/abc123#sr=somewhere.sesh:4567"); + + // Joins with the other fragments rather than replacing them + url = file_server::generate_download_url( + "abc123"sv, + {"http", + "example.com", + 123, + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29", + 12345, + file_server::SRouterTarget{"somewhere.sesh", 4567}}, + true); + CHECK(url == + "http://example.com:123/file/" + "abc123#p=3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29&d" + "&sr=somewhere.sesh:4567"); +} + +TEST_CASE("Download url session router round trip", "[backend][session_file_server]") { + // What we generate has to be what we parse: the generating side had no way to name a session + // router endpoint at all until now, while the parsing side has always understood one, so + // nothing checked that the two agreed. + auto check_round_trip = [](file_server::SRouterTarget target, uint16_t expected_port) { + auto url = file_server::generate_download_url( + "abc123"sv, + {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345, target}, + true); + + auto parsed = file_server::parse_download_url(url); + REQUIRE(parsed.has_value()); + CHECK(parsed->file_id == "abc123"); + CHECK(parsed->wants_stream_decryption); + REQUIRE(parsed->srouter_target.has_value()); + CHECK(parsed->srouter_target->address == target.address); + CHECK(parsed->srouter_target->port == expected_port); + }; + + check_round_trip({"somewhere.sesh"}, file_server::QUIC_DEFAULT_PORT); + check_round_trip( + {"somewhere.sesh", file_server::QUIC_DEFAULT_PORT}, file_server::QUIC_DEFAULT_PORT); + check_round_trip({"somewhere.sesh", 4567}, 4567); + check_round_trip({"name.loki", 1}, 1); + + // Without a target there is no fragment, and nothing to parse back + auto url = file_server::generate_download_url( + "abc123"sv, + {"http", "example.com", 123, file_server::DEFAULT_CONFIG.pubkey_hex, 12345}, + false); + auto parsed = file_server::parse_download_url(url); + REQUIRE(parsed.has_value()); + CHECK_FALSE(parsed->srouter_target.has_value()); } TEST_CASE("Download url port round trip", "[backend][session_file_server]") { @@ -170,9 +277,9 @@ TEST_CASE("Download url port round trip", "[backend][session_file_server]") { // to 80/443, which returns a response rather than an error -- so it surfaced as an // undecryptable attachment, far from its cause. constexpr auto custom_pubkey = - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"sv; + "3b6a27bcceb6a42d62a3a8d02a6f0d73653215771de243a63ac048a18b59da29"sv; auto url = file_server::generate_download_url( - "abc123"sv, {"http", "192.168.1.2", 8000, std::string{custom_pubkey}, 12345, false}); + "abc123"sv, {"http", "192.168.1.2", 8000, std::string{custom_pubkey}, 12345}, false); CHECK(url == fmt::format("http://192.168.1.2:8000/file/abc123#p={}", custom_pubkey)); auto parsed = file_server::parse_download_url(url); @@ -208,13 +315,17 @@ TEST_CASE("Download url port round trip", "[backend][session_file_server]") { CHECK(*parsed->port == 8000); } -TEST_CASE("Default file server onion pubkey", "[backend][session_file_server]") { +TEST_CASE("Built-in file server onion pubkeys", "[backend][session_file_server]") { // A download url carries the file server's ED25519 key, while an onion request needs the X25519 // form, so every request derives one from the other. Pinned here because the two forms are 32 // bytes either way: using the wrong one produces a perfectly well-formed key that simply never // decrypts, and the only symptom is the file server rejecting the request without naming a key. - const auto derived = compute_x25519_pubkey(session::to_span( - oxenc::from_hex(file_server::DEFAULT_CONFIG.pubkey_hex))); + // Storing the derived form in `pubkey_hex` is louder but no easier to spot from the outside: it + // is not a valid Ed25519 point, so the derivation throws and every upload fails before it ever + // reaches the network. + CHECK(compute_x25519_pubkey(ed25519_pubkey::from_hex(file_server::DEFAULT_CONFIG.pubkey_hex)) + .hex() == "09324794aa9c11948189762d198c618148e9136ac9582068180661208927ef34"); - CHECK(derived.hex() == "09324794aa9c11948189762d198c618148e9136ac9582068180661208927ef34"); + CHECK(compute_x25519_pubkey(ed25519_pubkey::from_hex(file_server::TESTNET_CONFIG.pubkey_hex)) + .hex() == "16d6c60aebb0851de7e6f4dc0a4734671dbf80f73664c008596511454cb6576d"); } diff --git a/tests/test_blinding.cpp b/tests/test_blinding.cpp index e275d88c9..ddc082f86 100644 --- a/tests/test_blinding.cpp +++ b/tests/test_blinding.cpp @@ -1,43 +1,29 @@ #include #include -#include #include -#include #include "session/blinding.hpp" +#include "session/hash.hpp" #include "session/util.hpp" #include "utils.hpp" using namespace session; -constexpr std::array seed1{ - 0xfe, 0xcd, 0x9a, 0x60, 0x34, 0xbc, 0x9a, 0xba, 0x27, 0x39, 0x25, 0xde, 0xe7, - 0x06, 0x2b, 0x12, 0x33, 0x34, 0x58, 0x7c, 0x3c, 0x62, 0x57, 0x34, 0x1a, 0xfa, - 0xe2, 0xd7, 0xfe, 0x85, 0xe1, 0x22, 0xf4, 0xef, 0x87, 0x39, 0x08, 0xf6, 0xa5, - 0x37, 0x7b, 0xa3, 0x85, 0x3f, 0x0e, 0x2f, 0xa3, 0x26, 0xee, 0xd9, 0xe7, 0x41, - 0xed, 0xf9, 0xf7, 0xd0, 0x31, 0x1a, 0x3e, 0xcc, 0x66, 0xa5, 0x7b, 0x32}; -constexpr std::array seed2{ - 0x86, 0x59, 0xef, 0xdc, 0xbe, 0x09, 0x49, 0xe0, 0xf8, 0x11, 0x41, 0xe6, 0xd3, - 0x97, 0xe8, 0xbe, 0x75, 0xf4, 0x5d, 0x09, 0x26, 0x2f, 0x20, 0x9d, 0x59, 0x50, - 0xe9, 0x79, 0x89, 0xeb, 0x43, 0xc7, 0x35, 0x70, 0xb6, 0x9a, 0x47, 0xdc, 0x09, - 0x45, 0x44, 0xc1, 0xc5, 0x08, 0x9c, 0x40, 0x41, 0x4b, 0xbd, 0xa1, 0xff, 0xdd, - 0xe8, 0xaa, 0xb2, 0x61, 0x7f, 0xe9, 0x37, 0xee, 0x74, 0xa5, 0xee, 0x81}; - -constexpr std::array xpub1{ - 0xfe, 0x94, 0xb7, 0xad, 0x4b, 0x7f, 0x1c, 0xc1, 0xbb, 0x92, 0x67, - 0x1f, 0x1f, 0x0d, 0x24, 0x3f, 0x22, 0x6e, 0x11, 0x5b, 0x33, 0x77, - 0x04, 0x65, 0xe8, 0x2b, 0x50, 0x3f, 0xc3, 0xe9, 0x6e, 0x1f, -}; -constexpr std::array xpub2{ - 0x05, 0xc9, 0xa9, 0xbf, 0x17, 0x8f, 0xa6, 0x44, 0xd4, 0x4b, 0xeb, - 0xf6, 0x28, 0x71, 0x6d, 0xc7, 0xf2, 0xdf, 0x3d, 0x08, 0x42, 0xe9, - 0x78, 0x81, 0x96, 0x2c, 0x72, 0x36, 0x99, 0x15, 0x20, 0x73, -}; - -const std::string session_id1 = "05" + oxenc::to_hex(xpub1.begin(), xpub1.end()); -const std::string session_id2 = "05" + oxenc::to_hex(xpub2.begin(), xpub2.end()); +constexpr auto seed1 = + "fecd9a6034bc9aba273925dee7062b123334587c3c6257341afae2d7fe85e122" + "f4ef873908f6a5377ba3853f0e2fa326eed9e741edf9f7d0311a3ecc66a57b32"_hex_b; +constexpr auto seed2 = + "8659efdcbe0949e0f81141e6d397e8be75f45d09262f209d5950e97989eb43c7" + "3570b69a47dc094544c1c5089c40414bbda1ffdde8aab2617fe937ee74a5ee81"_hex_b; + +constexpr auto sid1 = "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; +constexpr auto sid2 = "0505c9a9bf178fa644d44bebf628716dc7f2df3d0842e97881962c723699152073"_hex_b; +constexpr auto xpub1 = sid1.last<32>(); +constexpr auto xpub2 = sid2.last<32>(); +const std::string session_id1 = oxenc::to_hex(sid1); +const std::string session_id2 = oxenc::to_hex(sid2); TEST_CASE("Communities 25xxx-blinded pubkey derivation", "[blinding25][pubkey]") { REQUIRE(sodium_init() >= 0); @@ -55,216 +41,158 @@ TEST_CASE("Communities 25xxx-blinded pubkey derivation", "[blinding25][pubkey]") "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789") == "25a69cc6884530bf8498d22892e563716c4742f2845a7eb608de2aecbe7b6b5996"); - std::vector session_id1_raw; - oxenc::from_hex(session_id1.begin(), session_id1.end(), std::back_inserter(session_id1_raw)); CHECK(to_hex(blind25_id( - session_id1_raw, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes)) == + sid1, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b)) == "253b991dcbba44cfdb45d5b38880d95cff723309e3ece6fd01415ad5fa1dccc7ac"); CHECK(to_hex(blind25_id( - {session_id1_raw.begin() + 1, session_id1_raw.end()}, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes)) == + xpub1, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b)) == "253b991dcbba44cfdb45d5b38880d95cff723309e3ece6fd01415ad5fa1dccc7ac"); } TEST_CASE("Communities 25xxx-blinded signing", "[blinding25][sign]") { - - std::array server_pks = { - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "00cdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "999def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "888def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "777def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv}; - auto b25_1 = blind25_id(session_id1, server_pks[0]); - auto b25_2 = blind25_id(session_id1, server_pks[1]); - auto b25_3 = blind25_id(session_id2, server_pks[2]); - auto b25_4 = blind25_id(session_id2, server_pks[3]); - auto b25_5 = blind25_id(session_id2, server_pks[4]); - auto b25_6 = blind25_id(session_id1, server_pks[5]); - - auto sig1 = blind25_sign(to_span(seed1), server_pks[0], to_span("hello")); + constexpr std::array server_pks = { + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "00cdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "999def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "888def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "777def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b}; + auto b25_1 = blind25_id(sid1, server_pks[0]); + auto b25_2 = blind25_id(sid1, server_pks[1]); + auto b25_3 = blind25_id(sid2, server_pks[2]); + auto b25_4 = blind25_id(sid2, server_pks[3]); + auto b25_5 = blind25_id(sid2, server_pks[4]); + auto b25_6 = blind25_id(sid1, server_pks[5]); + + auto sig1 = blind25_sign(seed1, server_pks[0], "hello"_bytes); CHECK(to_hex(sig1) == "e6c57de4ac0cd278abbeef815bd88b163a037085deae789ecaaf4805884c4c3d3db25f3afa856241366cb341" "a3a4c9bbaa2cda81d028079c956fab16a7fe6206"); - CHECK(0 == crypto_sign_verify_detached( - sig1.data(), - to_unsigned("hello"), - 5, - to_unsigned(oxenc::from_hex(b25_1).data()) + 1)); + CHECK(ed25519::verify(sig1, std::span{b25_1}.last<32>(), "hello"_bytes)); - auto sig2 = blind25_sign(to_span(seed1), server_pks[1], to_span("world")); + auto sig2 = blind25_sign(seed1, server_pks[1], "world"_bytes); CHECK(to_hex(sig2) == "4460b606e9f55a7cba0bbe24207fe2859c3422783373788b6b070b2fa62ceba4f2a50749a6cee68e095747a3" "69927f9f4afa86edaf055cad68110e35e8b06607"); - CHECK(0 == crypto_sign_verify_detached( - sig2.data(), - to_unsigned("world"), - 5, - to_unsigned(oxenc::from_hex(b25_2).data()) + 1)); + CHECK(ed25519::verify(sig2, std::span{b25_2}.last<32>(), "world"_bytes)); - auto sig3 = blind25_sign(to_span(seed2), server_pks[2], to_span("this")); + auto sig3 = blind25_sign(seed2, server_pks[2], "this"_bytes); CHECK(to_hex(sig3) == "57bb2f80c88ce2f677902ee58e02cbd83e4e1ec9e06e1c72a34b4ab76d0f5219cfd141ac5ce7016c73c8382d" "b99df9f317f2bc0af6ca68edac2a9a7670938902"); - CHECK(0 == crypto_sign_verify_detached( - sig3.data(), - to_unsigned("this"), - 4, - to_unsigned(oxenc::from_hex(b25_3).data()) + 1)); + CHECK(ed25519::verify(sig3, std::span{b25_3}.last<32>(), "this"_bytes)); - auto sig4 = blind25_sign(to_span(seed2), server_pks[3], to_span("is")); + auto sig4 = blind25_sign(seed2, server_pks[3], "is"_bytes); CHECK(to_hex(sig4) == "ecce032b27b09d2d3d6df4ebab8cae86656c64fd1e3e70d6f020cd7e1a8058c57e3df7b6b01e90ccd592ac4a" "845dde7a2fdceb1a328a6690686851583133ea0c"); - CHECK(0 == crypto_sign_verify_detached( - sig4.data(), - to_unsigned("is"), - 2, - to_unsigned(oxenc::from_hex(b25_4).data()) + 1)); + CHECK(ed25519::verify(sig4, std::span{b25_4}.last<32>(), "is"_bytes)); - auto sig5 = blind25_sign(to_span(seed2), server_pks[4], to_span("")); + auto sig5 = blind25_sign(seed2, server_pks[4], ""_bytes); CHECK(to_hex(sig5) == "bf2fb9a511adbf5827e2e3bcf09f0a1cff80f85556fb76d8001aa8483b5f22e14539b170eaa0dbfa1489d1b8" "618ce8b48d7512cb5602c7eb8a05ce330a68350b"); - CHECK(0 == - crypto_sign_verify_detached( - sig5.data(), to_unsigned(""), 0, to_unsigned(oxenc::from_hex(b25_5).data()) + 1)); + CHECK(ed25519::verify(sig5, std::span{b25_5}.last<32>(), ""_bytes)); - auto sig6 = blind25_sign(to_span(seed1), server_pks[5], to_span("omg!")); + auto sig6 = blind25_sign(seed1, server_pks[5], "omg!"_bytes); CHECK(to_hex(sig6) == "322e280fbc3547c6b6512dbea4d60563d32acaa2df10d665c40a336c99fc3b8e4b13a7109dfdeadab2ab58b2" "cb314eb0510b947f43e5dfb6e0ce5bf1499d240f"); - CHECK(0 == crypto_sign_verify_detached( - sig6.data(), - to_unsigned("omg!"), - 4, - to_unsigned(oxenc::from_hex(b25_6).data()) + 1)); + CHECK(ed25519::verify(sig6, std::span{b25_6}.last<32>(), "omg!"_bytes)); // Test that it works when given just the seed instead of the whole sk: - auto sig6b = blind25_sign(to_span(seed1).subspan(0, 32), server_pks[5], to_span("omg!")); + auto sig6b = blind25_sign(seed1.first<32>(), server_pks[5], "omg!"_bytes); CHECK(to_hex(sig6b) == "322e280fbc3547c6b6512dbea4d60563d32acaa2df10d665c40a336c99fc3b8e4b13a7109dfdeadab2ab58b2" "cb314eb0510b947f43e5dfb6e0ce5bf1499d240f"); - CHECK(0 == crypto_sign_verify_detached( - sig6b.data(), - to_unsigned("omg!"), - 4, - to_unsigned(oxenc::from_hex(b25_6).data()) + 1)); + CHECK(ed25519::verify(sig6b, std::span{b25_6}.last<32>(), "omg!"_bytes)); } TEST_CASE("Communities 15xxx-blinded pubkey derivation", "[blinding15][pubkey]") { REQUIRE(sodium_init() >= 0); - std::vector session_id1_raw, session_id2_raw; - oxenc::from_hex(session_id1.begin(), session_id1.end(), std::back_inserter(session_id1_raw)); - oxenc::from_hex(session_id2.begin(), session_id2.end(), std::back_inserter(session_id2_raw)); CHECK(to_hex(blind15_id( - session_id1_raw, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes)) == + sid1, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b)) == "15b74ed205f1f931e1bb1291183778a9456b835937d923b0f2e248aa3a44c07844"); CHECK(to_hex(blind15_id( - session_id2_raw, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes)) == + sid2, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b)) == "1561e070286ff7a71f167e92b18c709882b148d8238c8872caf414b301ba0564fd"); CHECK(to_hex(blind15_id( - {session_id1_raw.begin() + 1, session_id1_raw.end()}, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes)) == + xpub1, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b)) == "15b74ed205f1f931e1bb1291183778a9456b835937d923b0f2e248aa3a44c07844"); } TEST_CASE("Communities 15xxx-blinded signing", "[blinding15][sign]") { REQUIRE(sodium_init() >= 0); - std::array server_pks = { - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "00cdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "999def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "888def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv, - "777def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"sv}; - auto b15_1 = blind15_id(session_id1, server_pks[0])[0]; - auto b15_2 = blind15_id(session_id1, server_pks[1])[0]; - // session_id2 has a negative pubkey, so these next three need the negative [1] instead: - auto b15_3 = blind15_id(session_id2, server_pks[2])[1]; - auto b15_4 = blind15_id(session_id2, server_pks[3])[1]; - auto b15_5 = blind15_id(session_id2, server_pks[4])[1]; - auto b15_6 = blind15_id(session_id1, server_pks[5])[0]; - - auto sig1 = blind15_sign(to_span(seed1), server_pks[0], to_span("hello")); + constexpr std::array server_pks = { + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "00cdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "999def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "888def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b, + "777def0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b}; + // Use blind15_key_pair for pubkeys: avoids the sign ambiguity of blind15_id + auto [b15_1_pk, _1] = blind15_key_pair(seed1, server_pks[0]); + auto [b15_2_pk, _2] = blind15_key_pair(seed1, server_pks[1]); + auto [b15_3_pk, _3] = blind15_key_pair(seed2, server_pks[2]); + auto [b15_4_pk, _4] = blind15_key_pair(seed2, server_pks[3]); + auto [b15_5_pk, _5] = blind15_key_pair(seed2, server_pks[4]); + auto [b15_6_pk, _6] = blind15_key_pair(seed1, server_pks[5]); + + auto sig1 = blind15_sign(seed1, server_pks[0], "hello"_bytes); CHECK(to_hex(sig1) == "1a5ade20b43af0e16b3e591d6f86303938d7557c0ac54469dd4f5aea759f82d22cafa42587251756e133acdd" "dd8cbec2f707a9ce09a49f2193f46a91502c5006"); - CHECK(0 == crypto_sign_verify_detached( - sig1.data(), - to_unsigned("hello"), - 5, - to_unsigned(oxenc::from_hex(b15_1).data()) + 1)); + CHECK(ed25519::verify(sig1, b15_1_pk, "hello"_bytes)); - auto sig2 = blind15_sign(to_span(seed1), server_pks[1], to_span("world")); + auto sig2 = blind15_sign(seed1, server_pks[1], "world"_bytes); CHECK(to_hex(sig2) == "d357f74c5ec5536840aec575051f71fdb22d70f35ef31db1715f5f694842de3b39aa647c84aa8e28ec56eb76" "2d237c9e030639c83f429826d419ac719cd4df03"); - CHECK(0 == crypto_sign_verify_detached( - sig2.data(), - to_unsigned("world"), - 5, - to_unsigned(oxenc::from_hex(b15_2).data()) + 1)); + CHECK(ed25519::verify(sig2, b15_2_pk, "world"_bytes)); - auto sig3 = blind15_sign(to_span(seed2), server_pks[2], to_span("this")); + auto sig3 = blind15_sign(seed2, server_pks[2], "this"_bytes); CHECK(to_hex(sig3) == "dacf91dfb411e99cd8ef4cb07b195b49289cf1a724fef122c73462818560bc29832a98d870ec4feb79dedca5" "b59aba6a466d3ce8f3e35adf25a1813f6989fd0a"); - CHECK(0 == crypto_sign_verify_detached( - sig3.data(), - to_unsigned("this"), - 4, - to_unsigned(oxenc::from_hex(b15_3).data()) + 1)); + CHECK(ed25519::verify(sig3, b15_3_pk, "this"_bytes)); - auto sig4 = blind15_sign(to_span(seed2), server_pks[3], to_span("is")); + auto sig4 = blind15_sign(seed2, server_pks[3], "is"_bytes); CHECK(to_hex(sig4) == "8339ea9887d3e44131e33403df160539cdc7a0a8107772172c311e95773660a0d39ed0a6c2b2c794dde1fdc6" "40943e403497aa02c4d1a21a7d9030742beabb05"); - CHECK(0 == crypto_sign_verify_detached( - sig4.data(), - to_unsigned("is"), - 2, - to_unsigned(oxenc::from_hex(b15_4).data()) + 1)); + CHECK(ed25519::verify(sig4, b15_4_pk, "is"_bytes)); - auto sig5 = blind15_sign(to_span(seed2), server_pks[4], to_span("")); + auto sig5 = blind15_sign(seed2, server_pks[4], ""_bytes); CHECK(to_hex(sig5) == "8b0d6447decff3a21ec1809141580139c4a51e24977b0605fe7984439993f5377ebc9681e4962593108d03cc" "8b6873c5c5ba8c30287188137d2dee9ab10afd0f"); - CHECK(0 == - crypto_sign_verify_detached( - sig5.data(), to_unsigned(""), 0, to_unsigned(oxenc::from_hex(b15_5).data()) + 1)); + CHECK(ed25519::verify(sig5, b15_5_pk, ""_bytes)); - auto sig6 = blind15_sign(to_span(seed1), server_pks[5], to_span("omg!")); + auto sig6 = blind15_sign(seed1, server_pks[5], "omg!"_bytes); CHECK(to_hex(sig6) == "946725055399376ecebb605c79f845fbf689a47f98507c2a1f239516fd9c9104e19fe533631c27ba4e744457" "4f0e4f0f0d422b7256ed63681a3ab2fe7e040601"); - CHECK(0 == crypto_sign_verify_detached( - sig6.data(), - to_unsigned("omg!"), - 4, - to_unsigned(oxenc::from_hex(b15_6).data()) + 1)); + CHECK(ed25519::verify(sig6, b15_6_pk, "omg!"_bytes)); // Test that it works when given just the seed instead of the whole sk: - auto sig6b = blind15_sign(to_span(seed1).subspan(0, 32), server_pks[5], to_span("omg!")); + auto sig6b = blind15_sign(seed1.first<32>(), server_pks[5], "omg!"_bytes); CHECK(to_hex(sig6b) == "946725055399376ecebb605c79f845fbf689a47f98507c2a1f239516fd9c9104e19fe533631c27ba4e744457" "4f0e4f0f0d422b7256ed63681a3ab2fe7e040601"); - CHECK(0 == crypto_sign_verify_detached( - sig6b.data(), - to_unsigned("omg!"), - 4, - to_unsigned(oxenc::from_hex(b15_6).data()) + 1)); + CHECK(ed25519::verify(sig6b, b15_6_pk, "omg!"_bytes)); } TEST_CASE("Version 07xxx-blinded pubkey derivation", "[blinding07][key_pair]") { REQUIRE(sodium_init() >= 0); - auto [pubkey, seckey] = blind_version_key_pair(to_span(seed1)); + auto [pubkey, seckey] = blind_version_key_pair(seed1); CHECK(oxenc::to_hex(pubkey.begin(), pubkey.end()) == "88e8adb27e7b8ce776fcc25bc1501fb2888fcac0308e52fb10044f789ae1a8fa"); @@ -272,10 +200,8 @@ TEST_CASE("Version 07xxx-blinded pubkey derivation", "[blinding07][key_pair]") { oxenc::to_hex(pubkey.begin(), pubkey.end())); // Hash ourselves just to make sure we get what we expect for the seed part of the secret key: - cleared_uc32 expect_seed; - static const auto hash_key = to_span("VersionCheckKey_sig"sv); - crypto_generichash_blake2b( - expect_seed.data(), 32, seed1.data(), 32, hash_key.data(), hash_key.size()); + cleared_b32 expect_seed; + hash::blake2b_key(expect_seed, "VersionCheckKey_sig"sv, seed1.first<32>()); CHECK(oxenc::to_hex(seckey.begin(), seckey.begin() + 32) == oxenc::to_hex(expect_seed.begin(), expect_seed.end())); @@ -288,31 +214,25 @@ TEST_CASE("Version 07xxx-blinded pubkey derivation", "[blinding07][key_pair]") { TEST_CASE("Version 07xxx-blinded signing", "[blinding07][sign]") { REQUIRE(sodium_init() >= 0); - auto signature = blind_version_sign(to_span(seed1), Platform::desktop, 1234567890); + auto signature = blind_version_sign(seed1, Platform::desktop, 1234567890); CHECK(oxenc::to_hex(signature.begin(), signature.end()) == "143c2c9828f7680ee81e6247bc7aa4777c4991add87cd724149b00452bed4e92" "0fa57daf4627c68f43fcbddb2d465d5ea11def523f3befb2bbee39c769676305"); - auto [pk, sk] = blind_version_key_pair(to_span(seed1)); + auto [pk, sk] = blind_version_key_pair(seed1); auto method = "GET"sv; - auto method_span = to_span(method); auto path = "/path/to/somewhere"sv; - auto path_span = to_span(path); auto body = to_span("some body (once told me)"); uint64_t timestamp = 1234567890; - std::vector full_message = to_vector("{}{}{}"_format(timestamp, method, path)); + std::vector full_message = to_vector("{}{}{}"_format(timestamp, method, path)); - auto req_sig_no_body = - blind_version_sign_request(to_span(seed1), timestamp, method, path, std::nullopt); - CHECK(crypto_sign_verify_detached( - req_sig_no_body.data(), full_message.data(), full_message.size(), pk.data()) == - 0); + auto req_sig_no_body = blind_version_sign_request(seed1, timestamp, method, path, std::nullopt); + CHECK(ed25519::verify(req_sig_no_body, pk, full_message)); full_message.insert(full_message.end(), body.begin(), body.end()); - auto req_sig = blind_version_sign_request(to_span(seed1), timestamp, method, path, body); - CHECK(crypto_sign_verify_detached( - req_sig.data(), full_message.data(), full_message.size(), pk.data()) == 0); + auto req_sig = blind_version_sign_request(seed1, timestamp, method, path, body); + CHECK(ed25519::verify(req_sig, pk, full_message)); } TEST_CASE("Communities session id blinded id matching", "[blinding][matching]") { diff --git a/tests/test_bugs.cpp b/tests/test_bugs.cpp index 088962547..edbd435c8 100644 --- a/tests/test_bugs.cpp +++ b/tests/test_bugs.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -10,13 +9,9 @@ using namespace session::config; TEST_CASE("Dirty/Mutable test case", "[config][dirty]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -25,14 +20,14 @@ TEST_CASE("Dirty/Mutable test case", "[config][dirty]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Contacts c1{session::to_span(seed), std::nullopt}; + session::config::Contacts c1{seed, std::nullopt}; c1.set_name("050000000000000000000000000000000000000000000000000000000000000000", "alfonso"); auto [seqno, data, obsolete] = c1.push(); CHECK(obsolete == std::vector{}); c1.confirm_pushed(seqno, {"fakehash1"}); - session::config::Contacts c2{session::to_span(seed), c1.dump()}; - session::config::Contacts c3{session::to_span(seed), c1.dump()}; + session::config::Contacts c2{seed, c1.dump()}; + session::config::Contacts c3{seed, c1.dump()}; CHECK_FALSE(c2.needs_dump()); CHECK_FALSE(c2.needs_push()); @@ -53,7 +48,7 @@ TEST_CASE("Dirty/Mutable test case", "[config][dirty]") { REQUIRE(seqno3 == 2); CHECK(as_set(obs3) == make_set("fakehash1"s)); - auto r = c1.merge(std::vector>>{ + auto r = c1.merge(std::vector>>{ {{"fakehash2", data2[0]}, {"fakehash3", data3[0]}}}); CHECK(r == std::unordered_set{{"fakehash2"s, "fakehash3"s}}); CHECK(c1.needs_dump()); @@ -77,13 +72,9 @@ TEST_CASE("Dirty/Mutable test case", "[config][dirty]") { // included in the old_hashes (which would result in clients deleting the current config from the // swarm) TEST_CASE("Merge existing config into clean state", "[config][merge_existing]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -92,7 +83,7 @@ TEST_CASE("Merge existing config into clean state", "[config][merge_existing]") CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Contacts c1{std::span{seed}, std::nullopt}; + session::config::Contacts c1{seed, std::nullopt}; c1.set_name("050000000000000000000000000000000000000000000000000000000000000000", "alfonso"); auto [seqno, data, obsolete] = c1.push(); CHECK(obsolete == std::vector{}); @@ -101,7 +92,7 @@ TEST_CASE("Merge existing config into clean state", "[config][merge_existing]") CHECK(!c1.needs_dump()); CHECK(!c1.needs_push()); - auto r = c1.merge(std::vector>>{ + auto r = c1.merge(std::vector>>{ {{"fakehash1"s, session::to_span(data[0])}}}); CHECK(as_set(r) == make_set("fakehash1"s)); @@ -114,13 +105,9 @@ TEST_CASE("Merge existing config into clean state", "[config][merge_existing]") // in old_hashes (which ends up being the same hash the dirty config gets after pushing, resulting // in the current config getting deleted from the swarm) TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -129,13 +116,13 @@ TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty] CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Contacts c1{std::span{seed}, std::nullopt}; + session::config::Contacts c1{seed, std::nullopt}; c1.set_name("050000000000000000000000000000000000000000000000000000000000000000", "alfonso"); auto [seqno, data, obsolete] = c1.push(); CHECK(obsolete == std::vector{}); c1.confirm_pushed(seqno, {"fakehash1"s}); - session::config::Contacts c2{std::span{seed}, c1.dump()}; + session::config::Contacts c2{seed, c1.dump()}; CHECK_FALSE(c2.needs_dump()); CHECK_FALSE(c2.needs_push()); @@ -151,7 +138,7 @@ TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty] c2.confirm_pushed(seqno2, {"fakehash2"s}); CHECK(c1.is_dirty()); // already dirty before the merge - auto r = c1.merge(std::vector>>{ + auto r = c1.merge(std::vector>>{ {{"fakehash2"s, session::to_span(data2[0])}}}); CHECK(r == std::unordered_set{{"fakehash2"s}}); CHECK(c1.needs_dump()); @@ -171,7 +158,7 @@ TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty] c2.confirm_pushed(seqno3, {"fakehash3"s}); CHECK(c1.is_dirty()); // already dirty before the merge - auto r2 = c1.merge(std::vector>>{ + auto r2 = c1.merge(std::vector>>{ {{"fakehash3", session::to_span(data3[0])}}}); CHECK(r2 == std::unordered_set{{"fakehash3"s}}); CHECK(c1.needs_dump()); @@ -205,7 +192,7 @@ TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty] c1.set_name("051111111111111111111111111111111111111111111111111111111111111140", "barney40"); auto size_before_merge = c1.size(); // retrieve size before trying to merge CHECK(c1.is_dirty()); // already dirty before the merge - auto r4 = c1.merge(std::vector>>{ + auto r4 = c1.merge(std::vector>>{ {{"fakehash21", session::to_span(data4[0])}}}); CHECK(r4 == std::unordered_set{{"fakehash21"s}}); CHECK(c1.needs_dump()); @@ -222,20 +209,16 @@ TEST_CASE("Merge config matching local changse", "[config][merge_matching_dirty] // *next* ordinary change (correctly landing on the value we had burned) looked like a conflict: // merging it produced a pointless conflict-resolution push instead of a clean adoption. TEST_CASE("Merge matching local changes adopts without consuming a seqno", "[config][merge]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; - session::config::Contacts c1{session::to_span(seed), std::nullopt}; + session::config::Contacts c1{seed, std::nullopt}; c1.set_name("050000000000000000000000000000000000000000000000000000000000000000", "alfonso"); auto [seqno1, data1, obs1] = c1.push(); REQUIRE(seqno1 == 1); c1.confirm_pushed(seqno1, {"fakehash1"}); auto dump1 = c1.dump(); - session::config::Contacts c2{session::to_span(seed), dump1}; + session::config::Contacts c2{seed, dump1}; // Both devices make the same change; c2 gets its push in first. c1.set_name("051111111111111111111111111111111111111111111111111111111111111111", "barney"); @@ -246,7 +229,7 @@ TEST_CASE("Merge matching local changes adopts without consuming a seqno", "[con c2.confirm_pushed(seqno2, {"fakehash2"}); REQUIRE(c1.is_dirty()); - auto r = c1.merge(std::vector>>{ + auto r = c1.merge(std::vector>>{ {{"fakehash2"s, session::to_span(data2[0])}}}); CHECK(r == std::unordered_set{{"fakehash2"s}}); CHECK(c1.is_clean()); @@ -260,7 +243,7 @@ TEST_CASE("Merge matching local changes adopts without consuming a seqno", "[con REQUIRE(seqno3 == 3); c2.confirm_pushed(seqno3, {"fakehash3"}); - r = c1.merge(std::vector>>{ + r = c1.merge(std::vector>>{ {{"fakehash3"s, session::to_span(data3[0])}}}); CHECK(r == std::unordered_set{{"fakehash3"s}}); CHECK(c1.is_clean()); @@ -279,10 +262,10 @@ TEST_CASE("Merge matching local changes adopts without consuming a seqno", "[con // Same identical-change situation, but the peer had already pushed a further change on top by // the time we merge: both messages arrive together and we adopt the newest at its own seqno. - session::config::Contacts c3{session::to_span(seed), dump1}; + session::config::Contacts c3{seed, dump1}; c3.set_name("051111111111111111111111111111111111111111111111111111111111111111", "barney"); REQUIRE(c3.is_dirty()); - r = c3.merge(std::vector>>{ + r = c3.merge(std::vector>>{ {{"fakehash2"s, session::to_span(data2[0])}, {"fakehash3"s, session::to_span(data3[0])}}}); CHECK(r == std::unordered_set{{"fakehash2"s, "fakehash3"s}}); diff --git a/tests/test_client/attachments.cpp b/tests/test_client/attachments.cpp new file mode 100644 index 000000000..fe3d553d3 --- /dev/null +++ b/tests/test_client/attachments.cpp @@ -0,0 +1,1631 @@ +#include "../../src/client/download_cache.hpp" +#include "../utils.hpp" +#include "common.hpp" + +namespace cache = session::client::cache; + +TEST_CASE("Client: an arriving message records the files it names", "[client][attachments]") { + TempClient c; + SenderKeys peer; + + // `id` is deprecated in favour of `url` but is still `required` by the protobuf, so every + // pointer carries one whether or not anything reads it. + uint64_t next_id = 111; + auto add = [&next_id]( + SessionProtos::DataMessage& data, + std::string_view url, + size_t key_len, + auto&& fill) { + auto* a = data.add_attachments(); + a->set_id(next_id++); + a->set_url(std::string{url}); + a->set_key(std::string(key_len, 'k')); + fill(a); + }; + + // Three attachments and no body at all: the case that used to be discarded outright, since a + // message was only history if it had text. + deliver(*c, + peer, + "", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + // Stream-encrypted, as anything we send is: 32-byte key, `d` in the url. + add(data, "http://fs.example/file/111#d", 32, [](auto* a) { + a->set_size(4321); + a->set_contenttype("image/png"); + a->set_filename("kitten.png"); + a->set_width(640); + a->set_height(480); + }); + // Legacy, which is what every current client actually sends: 64-byte key and a + // digest, and no fragment on the url. + add(data, "http://fs.example/file/222", 64, [](auto* a) { + a->set_digest(std::string(32, 'd')); + a->set_size(99); + a->set_contenttype("application/pdf"); + a->set_filename("invoice.pdf"); + a->set_caption("last month"); + a->set_flags(1); + }); + // A pointer with no url at all: unfetchable, but still one of three files the + // sender said were here, so it is not silently dropped. + add(data, "", 32, [](auto* a) { a->clear_url(); }); + }); + sync(*c); + + auto msgs = c->conversation(ConversationId::dm(peer.session_id), await)->messages(await); + REQUIRE(msgs.size() == 1); + const auto& m = msgs[0]; + CHECK(m.body.empty()); + CHECK_FALSE(m.outgoing); + REQUIRE(m.attachments.size() == 3); + + CHECK(m.attachments[0].index == 0); + CHECK(m.attachments[0].content_type == "image/png"); + CHECK(m.attachments[0].filename == "kitten.png"); + CHECK(m.attachments[0].size == 4321); + CHECK(m.attachments[0].width == 640); + CHECK(m.attachments[0].height == 480); + CHECK_FALSE(m.attachments[0].voice_message); + // Always true on an incoming attachment: the file server is where it came from. + CHECK(m.attachments[0].uploaded); + + CHECK(m.attachments[1].caption == "last month"); + CHECK(m.attachments[1].voice_message); + CHECK(m.attachments[1].uploaded); + + // The unusable one still occupies its position, so the indices keep meaning what the sender + // meant by them. + CHECK(m.attachments[2].index == 2); + CHECK_FALSE(m.attachments[2].uploaded); +} + +TEST_CASE("Client: a message reports the attachments it carries", "[client][send][attachments]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_attach", 7); + std::filesystem::create_directories(dir); + auto write = [&](std::string_view name) { + auto p = dir / name; + std::ofstream{p, std::ios::binary} << "not really a file"; + return p; + }; + // Names chosen for the ways extension parsing goes wrong: several dots, and a dotfile whose + // leading dot must not be read as an extension. + auto photo = write("holiday.snap.PNG"); + auto doc = write("notes.pdf"); + auto mystery = write(".hidden"); + + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto id = c->send_message( + ConversationId::dm(me), + {.attachments = + {OutgoingAttachment{.path = photo, .caption = "on the beach"}, + OutgoingAttachment{ + .path = doc, .content_type = "application/x-my-own", .width = 4}, + OutgoingAttachment{.path = mystery, .voice_message = true}}}, + await); + + auto msg = c->message(id, await); + REQUIRE(msg.has_value()); + + // An attachments-only message: nothing to show but the files, which is exactly the case that + // used to be indistinguishable from an empty message. + CHECK(msg->body.empty()); + REQUIRE(msg->attachments.size() == 3); + + // Ordered by position, and that position is what an upload report names. + CHECK(msg->attachments[0].index == 0); + CHECK(msg->attachments[1].index == 1); + CHECK(msg->attachments[2].index == 2); + + // Inferred from the last extension, case-insensitively, when the caller named none... + CHECK(msg->attachments[0].content_type == "image/png"); + CHECK(msg->attachments[0].filename == "holiday.snap.PNG"); + CHECK(msg->attachments[0].caption == "on the beach"); + CHECK_FALSE(msg->attachments[0].voice_message); + + // ...and never overriding one the caller did name. + CHECK(msg->attachments[1].content_type == "application/x-my-own"); + CHECK(msg->attachments[1].width == 4); + CHECK_FALSE(msg->attachments[1].height.has_value()); + + // A dotfile has no extension -- "hidden" is the name, not the type -- so it falls back rather + // than being given a type invented out of the filename. + CHECK(msg->attachments[2].content_type == "application/octet-stream"); + CHECK(msg->attachments[2].filename == ".hidden"); + CHECK(msg->attachments[2].voice_message); + + // Each file reached the server, and the size recorded is the file's own -- read at upload time, + // not the padded ciphertext's length that the server reports back. + sync(*c); + msg = c->message(id, await); + REQUIRE(msg.has_value()); + for (const auto& a : msg->attachments) { + CHECK(a.uploaded); + CHECK(a.size == static_cast(std::string_view{"not really a file"}.size())); + } + + // The same list reaches a paged read, not only the single-message one. + auto page = c->conversation(ConversationId::dm(me), await)->messages(await); + REQUIRE(page.size() == 1); + CHECK(page[0].attachments.size() == 3); + CHECK(page[0].attachments[0].content_type == "image/png"); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: saving an attachment fetches, decrypts and reports it", "[client][attachments]") { + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + // So the notification below goes out as a v1 send rather than queueing behind a key fetch. + TestHelper::seed_pfs_nak(c->core, peer.session_id); + + // A real attachment: encrypted exactly as a sender would, so what the download serves is what + // the decryptor has to cope with, chunk boundaries and padding included. + std::vector plaintext(9000); + for (size_t i = 0; i < plaintext.size(); i++) + plaintext[i] = static_cast(i * 31 % 256); + auto seed = random::random(32); + auto [ciphertext, key] = attachment::encrypt(seed, plaintext, attachment::Domain::ATTACHMENT); + + deliver( + *c, + peer, + "", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url("http://fs.example/file/1#d"); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(plaintext.size()); + a->set_filename("payload.bin"); + }, + 42); + sync(*c); + + auto msgs = c->conversation(ConversationId::dm(peer.session_id), await)->messages(await); + REQUIRE(msgs.size() == 1); + auto msg_id = msgs[0].id; + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_save", 7); + std::filesystem::create_directories(dir); + auto dest = dir / "saved.bin"; + + std::vector reports; + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + msg_id, + 0, + dest, + [&](const AttachmentProgress& p) { reports.push_back(p); }, + [&](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + + // Nothing is fetched until asked, and asking produces exactly one download. + sync(*c); + REQUIRE(net->downloads.size() == 1); + CHECK(net->downloads[0].download_url == "http://fs.example/file/1#d"); + + REQUIRE(serve_downloads(*net, ciphertext) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + + // The file is the file, byte for byte, with the padding that hid its length gone. + REQUIRE(std::filesystem::exists(dest)); + CHECK(std::filesystem::file_size(dest) == plaintext.size()); + { + std::ifstream in{dest, std::ios::binary}; + std::vector got(plaintext.size()); + in.read(reinterpret_cast(got.data()), got.size()); + CHECK(!!(got == plaintext)); + } + // ...and nothing is left behind that could be mistaken for it. + CHECK_FALSE(std::filesystem::exists(dest.string() + ".part")); + + // Progress reported as a send does: an opening 0/0 that says it has begun, then exactly one + // terminal result. Each report says which attachment of which message it is about, since a + // caller may be watching several. + REQUIRE(reports.size() >= 2); + CHECK_FALSE(reports.front().result.has_value()); + CHECK(reports.front().done == 0); + CHECK(reports.front().total == 0); + CHECK(reports.back().result == 0); + for (const auto& r : reports) { + CHECK(r.message_id == msg_id); + CHECK(r.index == 0); + } + + // And the sender is told, since nothing said otherwise. + sync(*c); + CHECK(stores(*net).size() == 1); + + // We also remember that we saved it, which is what stops a client offering "save" forever and + // writing a second copy. Recorded whether or not the sender was told. + auto saved = c->message(msg_id, await); + REQUIRE(saved.has_value()); + REQUIRE(saved->attachments.size() == 1); + REQUIRE(saved->attachments[0].saved_at.has_value()); + CHECK(*saved->attachments[0].saved_at > from_epoch_ms(0)); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: a save can be kept to ourselves, and a bad one writes nothing", + "[client][attachments]") { + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + TestHelper::seed_pfs_nak(c->core, peer.session_id); + + std::vector plaintext(500, std::byte{7}); + auto seed = random::random(32); + auto [ciphertext, key] = attachment::encrypt(seed, plaintext, attachment::Domain::ATTACHMENT); + + auto add = [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url("http://fs.example/file/2#d"); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(plaintext.size()); + }; + deliver(*c, peer, "", from_epoch_ms(2000), "h2", "", std::nullopt, add, 43); + sync(*c); + auto msg_id = + c->conversation(ConversationId::dm(peer.session_id), await)->messages(await)[0].id; + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_save", 7); + std::filesystem::create_directories(dir); + + // The promise is shared rather than captured by reference: save_attachment's callback outlives + // this scope, and a reference to a local here would dangle by the time the download is served. + auto save = [&](const std::filesystem::path& dest, bool notify) { + auto done = std::make_shared>>(); + auto waiter = done->get_future(); + c->Client::save_attachment( + msg_id, + 0, + dest, + nullptr, + [done](std::optional err, std::filesystem::path) { + done->set_value(std::move(err)); + }, + notify); + sync(*c); + return waiter; + }; + + // Asked not to tell them, we do not -- the file still lands. + { + auto quiet = dir / "quiet.bin"; + auto waiter = save(quiet, false); + REQUIRE(serve_downloads(*net, ciphertext) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + CHECK(std::filesystem::exists(quiet)); + sync(*c); + CHECK(stores(*net).empty()); + + // Telling them and remembering it ourselves are separate: a private save is still a save. + CHECK(c->message(msg_id, await)->attachments[0].saved_at.has_value()); + } + + // The account's own answer refuses the notification even when the caller asked for it, so a + // client that never grew a setting for this still honours one made on another device. + { + c->core.configs.user_profile().set_notify_media_saved(false); + auto loud = dir / "still-quiet.bin"; + auto waiter = save(loud, true); + REQUIRE(serve_downloads(*net, ciphertext) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + CHECK(std::filesystem::exists(loud)); + sync(*c); + CHECK(stores(*net).empty()); + c->core.configs.user_profile().set_notify_media_saved(true); + } + + // A file that fails to authenticate is a failure, not a corrupt file on disk: the ciphertext is + // written to a temporary name and only renamed once it has been decrypted whole. + { + auto bad = dir / "bad.bin"; + auto corrupt = ciphertext; + corrupt[corrupt.size() / 2] ^= std::byte{0xff}; + auto waiter = save(bad, true); + REQUIRE(serve_downloads(*net, corrupt) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK(waiter.get().has_value()); + CHECK_FALSE(std::filesystem::exists(bad)); + CHECK_FALSE(std::filesystem::exists(bad.string() + ".part")); + // Nothing was saved, so nobody is told one was. + sync(*c); + CHECK(stores(*net).empty()); + } + + std::filesystem::remove_all(dir); +} + +TEST_CASE("Client: an attachment we sent can be saved back", "[client][attachments]") { + // The whole path in one go: a file is encrypted and uploaded by sending it, and then fetched, + // decrypted and written by saving it. Nothing here knows what the other half did except + // through what was stored -- the url, its `d` fragment, the key and the size -- so a + // disagreement between the two shows up as bytes that do not match. + TempClient c; + auto* net = attach_mock_network(c->core); + + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_roundtrip", 7); + std::filesystem::create_directories(dir); + auto source = dir / "original.bin"; + + // Larger than one encryption chunk, so the streaming path is what runs rather than a single + // block that would hide a chunk-boundary bug. + std::vector contents(70 * 1024); + for (size_t i = 0; i < contents.size(); i++) + contents[i] = static_cast((i * 7 + i / 251) % 256); + { + std::ofstream out{source, std::ios::binary}; + out.write(reinterpret_cast(contents.data()), contents.size()); + } + + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here it is", .attachments = {OutgoingAttachment{.path = source}}}, + await); + sync(*c); + REQUIRE(accept_stores(*net) == 1); + + auto msg = c->message(id, await); + REQUIRE(msg.has_value()); + REQUIRE(msg->attachments.size() == 1); + CHECK(msg->attachments[0].uploaded); + // What the pointer advertises is the file's own length, not the padded ciphertext's. + CHECK(msg->attachments[0].size == static_cast(contents.size())); + + auto dest = dir / "saved.bin"; + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + msg->id, + 0, + dest, + nullptr, + [&done](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + sync(*c); + + // Served from what the upload left behind, found by the id in the url the send generated. + REQUIRE(serve_downloads(*net) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + + REQUIRE(std::filesystem::exists(dest)); + REQUIRE(std::filesystem::file_size(dest) == contents.size()); + std::ifstream in{dest, std::ios::binary}; + std::vector got(contents.size()); + in.read(reinterpret_cast(got.data()), got.size()); + CHECK(!!(got == contents)); + + // Stamped, even though the message is outgoing: this conversation is with ourselves, so the + // recipient saving it and us saving it are the same event. + CHECK(c->message(id, await)->attachments[0].saved_at.has_value()); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: a save does not destroy files it was not asked to touch", + "[client][attachments]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_clobber", 7); + std::filesystem::create_directories(dir); + auto source = dir / "sent.bin"; + std::vector contents(1024, std::byte{0x11}); + { + std::ofstream out{source, std::ios::binary}; + out.write(reinterpret_cast(contents.data()), contents.size()); + } + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here", .attachments = {OutgoingAttachment{.path = source}}}, + await); + sync(*c); + REQUIRE(accept_stores(*net) == 1); + + auto dest = dir / "report.pdf"; + + // Two files the caller never mentioned: somebody's own scratch file from another downloader, + // and something that appeared at the destination after the caller checked it was free. + auto their_part = dir / "report.pdf.part"; + { + std::ofstream out{their_part}; + out << "theirs"; + } + { + std::ofstream out{dest}; + out << "appeared since"; + } + + std::promise, std::filesystem::path>> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + id, + 0, + dest, + nullptr, + [&done](std::optional err, std::filesystem::path where) { + done.set_value({std::move(err), std::move(where)}); + }); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + auto [err, saved_to] = waiter.get(); + REQUIRE_FALSE(err.has_value()); + + // Neither file was touched, and the caller is told where its own went. + CHECK(std::filesystem::exists(their_part)); + CHECK(std::filesystem::file_size(their_part) == 6); + CHECK(std::filesystem::file_size(dest) == 14); + CHECK(saved_to == dir / "report (1).pdf"); + CHECK(std::filesystem::file_size(saved_to) == contents.size()); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: an approved replacement is not renamed out of the way", "[client][attachments]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_replace", 7); + std::filesystem::create_directories(dir); + auto source = dir / "sent.bin"; + std::vector contents(1024, std::byte{0x22}); + { + std::ofstream out{source, std::ios::binary}; + out.write(reinterpret_cast(contents.data()), contents.size()); + } + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here", .attachments = {OutgoingAttachment{.path = source}}}, + await); + sync(*c); + REQUIRE(accept_stores(*net) == 1); + + auto dest = dir / "report.pdf"; + { + std::ofstream out{dest}; + out << "the one the user chose to replace"; + } + + std::promise done; + auto waiter = done.get_future(); + c->Client::save_attachment( + id, + 0, + dest, + nullptr, + [&done](std::optional, std::filesystem::path where) { + done.set_value(std::move(where)); + }, + /*notify_sender=*/true, + /*replace=*/true); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + + // Renaming here would be worse than clobbering: the file the user agreed to replace would still + // be sitting there, and their answer would have been thrown away. + CHECK(waiter.get() == dest); + CHECK(std::filesystem::file_size(dest) == contents.size()); + CHECK_FALSE(std::filesystem::exists(dir / "report (1).pdf")); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: saving what we sent someone else does not claim they saved it", + "[client][attachments]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + SenderKeys peer; + TestHelper::seed_pfs_nak(c->core, peer.session_id); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_theirs", 7); + std::filesystem::create_directories(dir); + auto source = dir / "sent.bin"; + std::vector contents(2048, std::byte{0x5a}); + { + std::ofstream out{source, std::ios::binary}; + out.write(reinterpret_cast(contents.data()), contents.size()); + } + + auto id = c->send_message( + ConversationId::dm(peer.session_id), + {.body = "for you", .attachments = {OutgoingAttachment{.path = source}}}, + await); + sync(*c); + REQUIRE(accept_stores(*net) >= 1); + + auto dest = dir / "my-copy.bin"; + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + id, 0, dest, nullptr, [&done](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + REQUIRE_FALSE(waiter.get().has_value()); + REQUIRE(std::filesystem::exists(dest)); + + // `saved_at` on an outgoing attachment means *they* saved it, which is what lets a sender read + // it as "the file reached a person rather than a file server". Our own copy says nothing about + // that, so it must leave the field alone -- otherwise a UI reports "they have it" about someone + // who may never have opened the conversation. + auto msg = c->message(id, await); + REQUIRE(msg); + REQUIRE(msg->attachments.size() == 1); + CHECK_FALSE(msg->attachments[0].saved_at.has_value()); + + std::filesystem::remove_all(dir); +} + +TEST_CASE("Client: a peer can tell us they saved what we sent", "[client][attachments]") { + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + + TestHelper::seed_pfs_nak(c->core, peer.session_id); + TestHelper::seed_pfs_nak(c->core, own_sid(*c)); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_notified", 7); + std::filesystem::create_directories(dir); + auto one = dir / "one.bin"; + auto two = dir / "two.bin"; + std::ofstream{one, std::ios::binary} << "first file"; + std::ofstream{two, std::ios::binary} << "second file"; + + auto id = c->send_message( + ConversationId::dm(peer.session_id), + {.body = "two of them", + .attachments = {OutgoingAttachment{.path = one}, OutgoingAttachment{.path = two}}}, + await); + sync(*c); + REQUIRE(accept_stores(*net) == 2); + + auto sent = c->message(id, await); + REQUIRE(sent.has_value()); + REQUIRE(sent->attachments.size() == 2); + // Nobody has said anything yet, and an upload reaching the file server is not someone saving + // it. + CHECK_FALSE(sent->attachments[0].saved_at.has_value()); + CHECK_FALSE(sent->attachments[1].saved_at.has_value()); + + // What the peer's client sends when its user saves one file out of the message. + auto msgid = c->core.loop().call_get([&] { + return c->core.database().conn().prepared_get( + "SELECT msgid FROM messages WHERE id = ?", id); + }); + auto notify = [&](auto&& fill, sys_ms at) { + SessionProtos::Content content; + content.set_sigtimestamp(static_cast(epoch_ms(at))); + auto* note = content.mutable_dataextractionnotification(); + note->set_type(SessionProtos::DataExtractionNotification::MEDIA_SAVED); + fill(note); + + auto plaintext = content.SerializeAsString(); + auto encoded = encode_dm_v1( + std::as_bytes(std::span{plaintext}), peer.ed_sk, at, own_sid(*c), std::nullopt); + core::SwarmMessage sm{ + encoded, random::unique_id("h", 8), at, from_epoch_ms(1'000'000'000'000)}; + c->core.loop().call_get([&] { + c->core.receive_messages({&sm, 1}, config::Namespace::Default, true); + return 0; + }); + sync(*c); + }; + + auto saved_at = from_epoch_ms(9'000'000); + notify( + [&](auto* note) { + note->set_msgtimestamp(static_cast(epoch_ms(sent->timestamp))); + note->set_msgid(msgid); + note->set_attindex(1); + }, + saved_at); + + auto after = c->message(id, await); + REQUIRE(after.has_value()); + // Only the one they named, and stamped with when *they* saved it -- not the message's own + // timestamp, which is what identifies it and is generally older. + CHECK_FALSE(after->attachments[0].saved_at.has_value()); + REQUIRE(after->attachments[1].saved_at.has_value()); + CHECK(*after->attachments[1].saved_at == saved_at); + + // -1 is "all of them, together", which is what saving from a gallery view reports. + auto all_at = from_epoch_ms(9'500'000); + notify( + [&](auto* note) { + note->set_msgtimestamp(static_cast(epoch_ms(sent->timestamp))); + note->set_msgid(msgid); + note->set_attindex(-1); + }, + all_at); + + auto all = c->message(id, await); + REQUIRE(all->attachments[0].saved_at == all_at); + // The later save overwrites the earlier one: what this answers is "is there any point offering + // save again", not a history of every time they did. + CHECK(all->attachments[1].saved_at == all_at); + + // A notification naming a message we do not have changes nothing, and neither does one that + // names no message at all -- which is every notification the other clients send today, since + // their `timestamp` field means something different in each of them. + notify( + [&](auto* note) { + note->set_msgtimestamp(static_cast(epoch_ms(sent->timestamp))); + note->set_msgid(msgid + 1); + note->set_attindex(0); + }, + from_epoch_ms(9'900'000)); + notify([&](auto* note) { note->set_timestamp(12345); }, from_epoch_ms(9'900'000)); + + auto unchanged = c->message(id, await); + CHECK(unchanged->attachments[0].saved_at == all_at); + CHECK(unchanged->attachments[1].saved_at == all_at); + + std::filesystem::remove_all(dir); +} + +TEST_CASE("Client: a legacy attachment is saved", "[client][attachments][legacy]") { + // Every Session client still sends attachments encrypted the old way, so this is the path most + // received attachments actually take. The blob is fixed rather than generated: libsession has + // no legacy encryptor -- deliberately, since we never send these -- so producing one here would + // mean writing the very thing we chose not to have, and testing it against itself. It came + // from an independent implementation written against session-android's + // AttachmentCipherInputStream. + // + // 56 bytes of text, zero-padded to 128, then AES-256-CBC with an HMAC and a digest over it. + constexpr auto LEGACY_KEY = + "101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f"_hex_b; + constexpr auto LEGACY_BLOB = + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf22bf91f23b4781cc75fcba799b05fa6d" + "f93931dc76588ba849c27514c2e21560130db54a94a65303ea60adc0166ff90c" + "e07d033f2107b1ed38ddc006b1c71c3bb796d591ebbb2f9877027962dcb6ab13" + "b10b97cb736fddd7e2edd7b0908cd2b0ba84be5def8e67316556917af6faf793" + "56695fbd811fad5de80b9f70b15eb6987e18eab948150964e6309b24c3367b59" + "7a75cd3520a42061ce5ef6a0d647b1eb310fc355214c1f3eab964a9e7df62c65"_hex_b; + constexpr auto LEGACY_DIGEST = + "f75f0a8286252a371f131c711ddc5b04cc69092c8b8397b0460f1bc6946907c4"_hex_b; + constexpr auto LEGACY_TEXT = "a legacy attachment, from a client that has not moved on"sv; + + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + TestHelper::seed_pfs_nak(c->core, peer.session_id); + + // No `d` fragment on the url, a 64-byte key and a digest: that combination is what tells the + // save which of the two schemes to use, and nothing else does. + deliver( + *c, + peer, + "", + from_epoch_ms(3000), + "legacy_hash", + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(9); + a->set_url("http://fs.example/file/legacy1"); + a->set_key(std::string{ + reinterpret_cast(LEGACY_KEY.data()), LEGACY_KEY.size()}); + a->set_digest(std::string{ + reinterpret_cast(LEGACY_DIGEST.data()), LEGACY_DIGEST.size()}); + a->set_size(LEGACY_TEXT.size()); + a->set_filename("legacy.txt"); + }, + 44); + sync(*c); + + auto msgs = c->conversation(ConversationId::dm(peer.session_id), await)->messages(await); + REQUIRE(msgs.size() == 1); + REQUIRE(msgs[0].attachments.size() == 1); + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_legacy", 7); + std::filesystem::create_directories(dir); + auto dest = dir / "legacy.txt"; + + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + msgs[0].id, + 0, + dest, + nullptr, + [&done](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + sync(*c); + + REQUIRE(serve_downloads(*net, LEGACY_BLOB) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + + // Trimmed to the length the pointer claimed, with the zero padding that hid it gone. + REQUIRE(std::filesystem::exists(dest)); + REQUIRE(std::filesystem::file_size(dest) == LEGACY_TEXT.size()); + std::ifstream in{dest, std::ios::binary}; + std::string got{std::istreambuf_iterator{in}, {}}; + CHECK(got == LEGACY_TEXT); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: a message with attachments needs readable files", "[client][send][attachments]") { + TempClient c{}; + auto me = own_sid(*c); + + CHECK_THROWS_AS( + c->send_message( + ConversationId::dm(me), + {.body = "here you go", + .attachments = {OutgoingAttachment{.path = "/nonexistent/nope.png"}}}, + await), + std::invalid_argument); + + // Rejected before anything was stored, rather than leaving a message that can never be sent -- + // and not even a conversation, which is a stronger thing to be able to say than that it has no + // messages in it. + CHECK_FALSE(c->conversation(ConversationId::dm(me), await).has_value()); +} + +TEST_CASE( + "Client: attachments that cannot be uploaded fail the message", + "[client][send][attachments]") { + auto file = std::filesystem::temp_directory_path() / "libsession_attachment_test.bin"; + { + std::ofstream out{file, std::ios::binary}; + out << "some file contents"; + } + + Recorder r; + TempClient c{r.handlers()}; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + std::vector>> reports; + + // No network is attached, so the upload cannot even be attempted. The message must still end + // up somewhere final: the failure is what a caller waits on, and a message left in `uploading` + // would wait forever. + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here you go", .attachments = {OutgoingAttachment{.path = file}}}, + [&](size_t idx, int64_t sent, int64_t total, std::optional result) { + reports.emplace_back(idx, sent, total, result); + }, + await); + sync(*c); + + auto msg = c->message(id, await); + REQUIRE(msg); + CHECK(msg->body == "here you go"); + CHECK(msg->send_state == SendState::failed); + + REQUIRE(reports.size() == 1); + auto [idx, sent, total, result] = reports.front(); + CHECK(idx == 0); + REQUIRE(result.has_value()); + CHECK(*result != 0); + + std::filesystem::remove(file); +} + +TEST_CASE( + "Client: throttling never squelches what a caller must hear", + "[client][send][attachments]") { + auto file = std::filesystem::temp_directory_path() / "libsession_throttle_test.bin"; + { + std::ofstream out{file, std::ios::binary}; + out << "some file contents"; + } + + TempClient c; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + // Long enough that anything passing through the throttle would be dropped: what arrives is + // exactly what is exempt from it. + c->set_high_freq_dispatch_interval(1h); + + std::vector> results; + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here you go", .attachments = {OutgoingAttachment{.path = file}}}, + [&](size_t, int64_t, int64_t, std::optional result) { results.push_back(result); }, + await); + sync(*c); + + // With no network the upload cannot start, so the one report is its failure -- which is the + // point: an outcome is never a thing the throttle may drop. + CHECK(c->message(id, await)->send_state == SendState::failed); + REQUIRE(results.size() == 1); + REQUIRE(results.front().has_value()); + CHECK(*results.front() != 0); + + std::filesystem::remove(file); +} + +TEST_CASE("Client: retrying a send that cannot work", "[client][send][attachments]") { + auto file = std::filesystem::temp_directory_path() / "libsession_retry_test.bin"; + { + std::ofstream out{file, std::ios::binary}; + out << "some file contents"; + } + + Recorder r; + TempClient c{r.handlers()}; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto id = c->send_message( + ConversationId::dm(me), + {.body = "here you go", .attachments = {OutgoingAttachment{.path = file}}}, + await); + sync(*c); + REQUIRE(c->message(id, await)->send_state == SendState::failed); + + // Retrying is allowed while the failure is one that might not recur, and reports itself as + // started rather than as succeeded -- the outcome arrives through the message's state. + std::vector> results; + CHECK(c->retry_send( + id, + [&](size_t, int64_t, int64_t, std::optional result) { results.push_back(result); }, + await)); + sync(*c); + REQUIRE(results.size() == 1); + CHECK(c->message(id, await)->send_state == SendState::failed); + + // With the file gone the retry can only ever fail the same way, so the message becomes + // terminal rather than staying something an application would offer to try again. + std::filesystem::remove(file); + + results.clear(); + CHECK(c->retry_send( + id, + [&](size_t, int64_t, int64_t, std::optional result) { results.push_back(result); }, + await)); + sync(*c); + REQUIRE(results.size() == 1); + CHECK(results.front() == ATTACHMENT_FILE_MISSING); + CHECK(c->message(id, await)->send_state == SendState::unsendable); + + // ... and being terminal, it is refused rather than attempted again. + CHECK_FALSE(c->retry_send(id, await)); +} + +TEST_CASE( + "Client: a stream attachment must be the size its sender claimed", + "[client][attachments][size]") { + // The pointer's size is exact -- the file server is told the byte count up front and refuses + // anything else -- so a file that decrypts to a different length is not a file we asked for. + // Nothing else catches this for a stream attachment: the format strips its own padding and + // never consults the pointer, so before this the claim was simply ignored. + // One short, one long: over-reporting and under-reporting are both lies. + int64_t claimed = GENERATE(8999, 9001); + + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + + std::vector plaintext(9000); + random::fill(plaintext); + auto seed = random::random(32); + auto [ciphertext, key] = attachment::encrypt(seed, plaintext, attachment::Domain::ATTACHMENT); + + deliver( + *c, + peer, + "", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + [&, claimed](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url("http://fs.example/file/1#d"); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + // A lie in one direction or the other; the truth is 9000. + a->set_size(static_cast(claimed)); + a->set_filename("payload.bin"); + }, + 42); + sync(*c); + + auto msg_id = + c->conversation(ConversationId::dm(peer.session_id), await)->messages(await)[0].id; + + auto dir = std::filesystem::temp_directory_path() / random::unique_id("test_size", 7); + std::filesystem::create_directories(dir); + auto dest = dir / "saved.bin"; + + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + msg_id, 0, dest, nullptr, [&](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + + sync(*c); + REQUIRE(serve_downloads(*net, ciphertext) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + + // Reported as a failure rather than saved short or saved long. + auto err = waiter.get(); + REQUIRE(err.has_value()); + CHECK(err->find("sender said") != std::string::npos); + + // And nothing is left on disk that could be mistaken for the file. + CHECK_FALSE(std::filesystem::exists(dest)); + CHECK_FALSE(std::filesystem::exists(dest.string() + ".part")); + + std::filesystem::remove_all(dir); +} + +TEST_CASE( + "Client: two askers for one attachment share one download", "[client][attachments][join]") { + // A conversation opening while its attachments are being fetched asks for bytes that are not in + // the cache yet. Without joining, that starts a second download of the same file: the cache is + // still empty, so a display sees a miss and fetches it again. + TempCacheDir dir; + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + std::vector plaintext(9000); + random::fill(plaintext); + auto seed = random::random(32); + auto [ciphertext, key] = attachment::encrypt(seed, plaintext, attachment::Domain::ATTACHMENT); + net->served["shared"] = ciphertext; + + deliver( + *c, + peer, + "", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url(network::file_server::generate_download_url("shared", {}, true)); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(plaintext.size()); + a->set_contenttype("image/png"); + }, + 42); + sync(*c); + auto msg_id = + c->conversation(ConversationId::dm(peer.session_id), await)->messages(await)[0].id; + + std::vector>> got(2); + std::vector> seen(2); + for (int i = 0; i < 2; i++) + c->attachment_data( + msg_id, + 0, + [&, i](const AttachmentProgress& p) { seen[i].push_back(p); }, + [&, i](std::optional err, std::vector d) { + REQUIRE_FALSE(err.has_value()); + got[i] = std::move(d); + }); + sync(*c); + + // One transfer, not two. + REQUIRE(net->downloads.size() == 1); + + REQUIRE(serve_downloads(*net, ciphertext) == 1); + sync(*c); + + // Both askers get the whole file. + for (int i = 0; i < 2; i++) { + REQUIRE(got[i]); + CHECK(*got[i] == plaintext); + // ...and both were told how it was going, not only the one that started it. + CHECK_FALSE(seen[i].empty()); + CHECK(seen[i].back().result == 0); + CHECK(seen[i].back().message_id == msg_id); + } + + // The second asker joined midway and was told where it had got to straight away, rather than + // being left with nothing until the next chunk. + CHECK_FALSE(seen[1].empty()); + + // And having finished, a third ask is served from the cache with no download at all. + net->downloads.clear(); + std::optional> third; + c->attachment_data( + msg_id, 0, nullptr, [&](std::optional err, std::vector d) { + REQUIRE_FALSE(err.has_value()); + third = std::move(d); + }); + sync(*c); + CHECK(net->downloads.empty()); + REQUIRE(third); + CHECK(*third == plaintext); +} + +TEST_CASE("Client: saving joins a fetch already under way", "[client][attachments][join]") { + // The direction that *can* combine: something is being accumulated for the cache -- a gallery, + // or the auto-downloader -- and a save asks for the same file. Waiting on it costs nothing, + // since that buffer is committed either way, and fetching it twice would be two transfers of + // one file. + // + // (The reverse cannot: a save streams to disk and keeps nothing, so a display arriving midway + // has no way to be given the half already written.) + TempCacheDir dir; + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + TestHelper::seed_pfs_nak(c->core, peer.session_id); + + std::vector plaintext(9000); + random::fill(plaintext); + auto seed = random::random(32); + auto [ciphertext, key] = attachment::encrypt(seed, plaintext, attachment::Domain::ATTACHMENT); + net->served["both"] = ciphertext; + + deliver( + *c, + peer, + "", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url(network::file_server::generate_download_url("both", {}, true)); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(plaintext.size()); + a->set_contenttype("image/png"); + }, + 42); + sync(*c); + auto msg_id = + c->conversation(ConversationId::dm(peer.session_id), await)->messages(await)[0].id; + + // A display asks first, so the file is being accumulated. + std::optional> shown; + c->attachment_data( + msg_id, 0, nullptr, [&](std::optional err, std::vector d) { + REQUIRE_FALSE(err.has_value()); + shown = std::move(d); + }); + sync(*c); + REQUIRE(net->downloads.size() == 1); + + // Now a save of the same attachment, while that is still in flight. + auto dest = dir.path / "saved.png"; + std::vector saw; + std::promise> done; + auto waiter = done.get_future(); + c->Client::save_attachment( + msg_id, + 0, + dest, + [&](const AttachmentProgress& p) { saw.push_back(p); }, + [&](std::optional err, std::filesystem::path) { + done.set_value(std::move(err)); + }); + sync(*c); + + // Still one transfer: the save waited rather than asking for the same bytes again. + CHECK(net->downloads.size() == 1); + + REQUIRE(serve_downloads(*net, ciphertext) == 1); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + CHECK_FALSE(waiter.get().has_value()); + sync(*c); + + // Both callers got what they asked for: the display its bytes, the save its file. + REQUIRE(shown); + CHECK(*shown == plaintext); + REQUIRE(std::filesystem::exists(dest)); + CHECK(std::filesystem::file_size(dest) == plaintext.size()); + { + std::ifstream in{dest, std::ios::binary}; + std::vector got(plaintext.size()); + in.read(reinterpret_cast(got.data()), got.size()); + CHECK(!!(got == plaintext)); + } + + // The save was told how the transfer it joined was going, not left silent until it finished. + CHECK_FALSE(saw.empty()); + CHECK(saw.back().result == 0); +} + +TEST_CASE("Client: a conversation set to auto-download fetches on arrival", "[client][auto]") { + TempCacheDir dir; + std::vector> progress; + callbacks cbs; + cbs.attachment_progress = [&](const ConversationId& id, const AttachmentProgress& p) { + progress.emplace_back(id, p); + }; + TempClient c{cbs}; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + auto convo = ConversationId::dm(peer.session_id); + c->open_dm(convo, await); + + std::vector image(4000), doc(4000); + random::fill(image); + random::fill(doc); + auto seed = random::random(32); + auto [image_ct, image_key] = attachment::encrypt(seed, image, attachment::Domain::ATTACHMENT); + auto [doc_ct, doc_key] = attachment::encrypt(seed, doc, attachment::Domain::ATTACHMENT); + net->served["img"] = image_ct; + net->served["doc"] = doc_ct; + + auto arrive = [&](std::string hash, bool with_doc) { + deliver(*c, + peer, + "", + from_epoch_ms(1000), + hash, + "", + std::nullopt, + [&](SessionProtos::DataMessage& data) { + auto* a = data.add_attachments(); + a->set_id(1); + a->set_url(network::file_server::generate_download_url("img", {}, true)); + a->set_key(std::string{ + reinterpret_cast(image_key.data()), image_key.size()}); + a->set_size(image.size()); + a->set_contenttype("image/png"); + if (with_doc) { + auto* b = data.add_attachments(); + b->set_id(2); + b->set_url(network::file_server::generate_download_url("doc", {}, true)); + b->set_key(std::string{ + reinterpret_cast(doc_key.data()), doc_key.size()}); + b->set_size(doc.size()); + b->set_contenttype("application/pdf"); + } + }); + sync(*c); + }; + + SECTION("unasked means nothing is fetched") { + // Never having been asked is not consent, and is what a client prompts on. + REQUIRE_FALSE(c->conversation(convo, await)->auto_download().has_value()); + arrive("h1", false); + CHECK(net->downloads.empty()); + CHECK(progress.empty()); + CHECK_FALSE(c->conversation(convo, await)->messages(await)[0].gallery); + } + + SECTION("images only fetches the image and leaves the document") { + c->conversation(convo, await)->set_auto_download(AutoDownload::image_attachments, await); + arrive("h2", true); + REQUIRE(net->downloads.size() == 1); + CHECK(net->downloads[0].download_url.find("img") != std::string::npos); + + // Not a gallery: one of its attachments is not an image, so it cannot be shown as one. + auto m = c->conversation(convo, await)->messages(await)[0]; + CHECK_FALSE(m.gallery_viewable); + CHECK_FALSE(m.gallery); + } + + SECTION("all fetches both, and an all-image message opens as a gallery") { + c->conversation(convo, await)->set_auto_download(AutoDownload::all, await); + arrive("h3", true); + CHECK(net->downloads.size() == 2); + + // Two attachments, one of them a pdf: still not gallery viewable even though both were + // fetched. What is downloaded and what can be displayed as a gallery are different + // questions. + CHECK_FALSE(c->conversation(convo, await)->messages(await)[0].gallery); + + arrive("h4", false); + auto m = c->conversation(convo, await)->messages(await)[0]; + CHECK(m.gallery_viewable); + CHECK(m.gallery); + } + + SECTION("a size limit refuses what is too big, before fetching it") { + c->conversation(convo, await)->set_auto_download(AutoDownload::all, await); + c->set_auto_download_max_size(1000, await); + arrive("h5", false); + CHECK(net->downloads.empty()); + + // Raising it lets the next one through, so the limit is read per message rather than + // remembered from startup. + c->set_auto_download_max_size(std::nullopt, await); + arrive("h6", false); + CHECK(net->downloads.size() == 1); + } + + SECTION("the fetch is reported, cached, and never told to the sender") { + c->conversation(convo, await)->set_auto_download(AutoDownload::all, await); + arrive("h7", false); + REQUIRE(net->downloads.size() == 1); + REQUIRE(serve_downloads(*net, image_ct) == 1); + sync(*c); + + // Broadcast, since nobody asked for it and there is no caller to hand a report to. + REQUIRE_FALSE(progress.empty()); + CHECK(progress.back().first == convo); + CHECK(progress.back().second.result == 0); + + // In the cache, so opening the conversation costs nothing... + CHECK(std::filesystem::exists(cache::path_for( + dir.path, + cache::ATTACHMENT_DIR, + network::file_server::generate_download_url("img", {}, true)))); + + // ...and the sender is *not* told, because nobody has saved anything. That notification + // belongs to a save, whether or not the bytes came from the cache. + CHECK(stores(*net).empty()); + } +} + +TEST_CASE("Client: the cache evicts least recently used", "[client][auto][evict]") { + TempCacheDir dir; + TempClient c; + SenderKeys peer; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + auto convo = ConversationId::dm(peer.session_id); + c->open_dm(convo, await); + c->conversation(convo, await)->set_auto_download(AutoDownload::all, await); + + // `last_used` is a millisecond timestamp, and the whole of this test would otherwise run inside + // one of them, leaving every row tied and the eviction order arbitrary. Real uses are spread + // out; these have to be spread out by hand. + ScopedClockOffset clock{0s}; + auto later = [t = 0s]() mutable { AdjustedClock::set_offset(t += 1s); }; + + auto seed = random::random(32); + // Three files, fetched in order, each about the same size on disk. + std::vector urls; + std::vector ids; + for (int i = 0; i < 3; i++) { + later(); + std::vector data(3000); + random::fill(data); + auto [ct, key] = attachment::encrypt(seed, data, attachment::Domain::ATTACHMENT); + auto file_id = "f{}"_format(i); + net->served[file_id] = ct; + auto url = network::file_server::generate_download_url(file_id, {}, true); + urls.push_back(url); + + deliver(*c, + peer, + "", + from_epoch_ms(1000 + i), + "h{}"_format(i), + "", + std::nullopt, + [&, url](SessionProtos::DataMessage& d) { + auto* a = d.add_attachments(); + a->set_id(static_cast(i + 1)); + a->set_url(url); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(data.size()); + a->set_contenttype("image/png"); + }); + sync(*c); + REQUIRE(serve_downloads(*net, ct) == 1); + sync(*c); + ids.push_back(c->conversation(convo, await)->messages(await)[0].id); + } + + auto cached = [&](const std::string& url) { + return std::filesystem::exists(cache::path_for(dir.path, cache::ATTACHMENT_DIR, url)); + }; + for (const auto& u : urls) + REQUIRE(cached(u)); + + // Reach for the *oldest* one, which makes it the most recently used. Under oldest-first + // eviction it would still be first to go; under least-recently-used it is last. + later(); + c->attachment_data(ids[0], 0, nullptr, [](auto, auto) {}); + sync(*c); + + // Now a limit that only two of the three fit under. + auto one = + std::filesystem::file_size(cache::path_for(dir.path, cache::ATTACHMENT_DIR, urls[0])); + c->set_attachment_cache_limit(static_cast(one * 2 + one / 2), await); + + // Nothing happens until something is added, which is the only moment the total can grow. + CHECK(cached(urls[1])); + + // A fourth arrival pushes it over and evicts. + later(); + std::vector more(3000); + random::fill(more); + auto [more_ct, more_key] = attachment::encrypt(seed, more, attachment::Domain::ATTACHMENT); + net->served["f3"] = more_ct; + auto more_url = network::file_server::generate_download_url("f3", {}, true); + deliver(*c, + peer, + "", + from_epoch_ms(2000), + "h3", + "", + std::nullopt, + [&](SessionProtos::DataMessage& d) { + auto* a = d.add_attachments(); + a->set_id(9); + a->set_url(more_url); + a->set_key(std::string{ + reinterpret_cast(more_key.data()), more_key.size()}); + a->set_size(more.size()); + a->set_contenttype("image/png"); + }); + sync(*c); + REQUIRE(serve_downloads(*net, more_ct) == 1); + sync(*c); + + // The one just fetched is kept -- a download that completed and immediately vanished would read + // as a failure. + CHECK(cached(more_url)); + // The one that was read most recently is kept, though it is the oldest by arrival. + CHECK(cached(urls[0])); + // The one nobody has touched since it arrived is gone. + CHECK_FALSE(cached(urls[1])); + + // The index agrees with the directory rather than describing files that are no longer there. + auto rows = c->core.database().conn().prepared_get( + "SELECT count(*) FROM attachment_cache"); + size_t on_disk = 0; + for (const auto& e : std::filesystem::directory_iterator{dir.path / cache::ATTACHMENT_DIR}) + if (!e.path().filename().string().ends_with(cache::PARTIAL_SUFFIX)) + on_disk++; + CHECK(static_cast(rows) == on_disk); +} + +TEST_CASE("Client: the sweep reconciles the cache with what the database says", "[client][evict]") { + TempCacheDir dir; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + auto seed = random::random(32); + + TempClient c; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + c->open_dm(convo, await); + c->conversation(convo, await)->set_auto_download(AutoDownload::all, await); + + std::vector data(2000); + random::fill(data); + auto [ct, key] = attachment::encrypt(seed, data, attachment::Domain::ATTACHMENT); + net->served["real"] = ct; + auto url = network::file_server::generate_download_url("real", {}, true); + + deliver(*c, + peer, + "", + from_epoch_ms(1000), + "hh", + "", + std::nullopt, + [&](SessionProtos::DataMessage& d) { + auto* a = d.add_attachments(); + a->set_id(1); + a->set_url(url); + a->set_key(std::string{reinterpret_cast(key.data()), key.size()}); + a->set_size(data.size()); + a->set_contenttype("image/png"); + }); + sync(*c); + REQUIRE(serve_downloads(*net, ct) == 1); + sync(*c); + + auto real_name = cache::path_for(dir.path, cache::ATTACHMENT_DIR, url).filename().string(); + REQUIRE(std::filesystem::exists(dir.path / cache::ATTACHMENT_DIR / real_name)); + + // What a crash between writing a file and recording it leaves: a file no row names. + auto orphan_name = + cache::path_for(dir.path, cache::ATTACHMENT_DIR, "http://fs.example/file/ghost") + .filename() + .string(); + std::ofstream{dir.path / cache::ATTACHMENT_DIR / orphan_name, std::ios::binary} + << "no row names this"; + + // An in-progress write, which is not garbage but unfinished. + auto partial = "{}{}"_format(orphan_name, cache::PARTIAL_SUFFIX); + std::ofstream{dir.path / cache::ATTACHMENT_DIR / partial, std::ios::binary} << "half"; + + // And the other direction: a row naming a file that is not there. Not cosmetic -- eviction + // totals `size` over the rows, so this one makes the cache look 10 MB fuller than it is. + std::string stale_name = "deadbeef"; + c->core.loop().call_get([&] { + c->core.database().conn().prepared_exec( + "INSERT INTO attachment_cache (name, size, last_used) VALUES (?, ?, ?)", + stale_name, + int64_t{10'000'000}, + int64_t{1}); + return 0; + }); + + // Reopened, which is when a client sweeps: the leaks above are exactly what survives a restart. + c.reopen(); + c->set_cache_dir(dir.path); + + // Reopened *again* rather than waited on, because destruction is what a sweep is guaranteed + // against: the destructor joins the sweeper, and the sweeper does not finish until the + // reconcile it posted has run. A `sync` here would only prove the loop was idle, which it is + // well before the listing is done. This one is given no cache directory, so it does not sweep + // in turn. + c.reopen(); + + CHECK(std::filesystem::exists(dir.path / cache::ATTACHMENT_DIR / real_name)); + CHECK_FALSE(std::filesystem::exists(dir.path / cache::ATTACHMENT_DIR / orphan_name)); + CHECK(std::filesystem::exists(dir.path / cache::ATTACHMENT_DIR / partial)); + + auto rows = c->core.loop().call_get([&] { + return c->core.database().conn().prepared_get( + "SELECT count(*) FROM attachment_cache WHERE name = ?", stale_name); + }); + CHECK(rows == 0); + + // The tracked file kept its row: reconciling is not an excuse to rebuild the index. + auto kept = c->core.loop().call_get([&] { + return c->core.database().conn().prepared_get( + "SELECT count(*) FROM attachment_cache WHERE name = ?", real_name); + }); + CHECK(kept == 1); +} + +TEST_CASE("Client: the list preview describes a message's attachments", "[client][attachments]") { + TempClient c; + SenderKeys gallery, mixed, voice, wordy; + // Approved, or these would be message requests and `conversations()` would not list them. + for (const auto* who : {&gallery, &mixed, &voice, &wordy}) + approve(*c, who->session_id); + + uint64_t next_id = 500; + auto add = [&next_id]( + SessionProtos::DataMessage& data, + std::string_view ctype, + std::string_view name, + int flags = 0) { + auto* a = data.add_attachments(); + a->set_id(next_id); + a->set_url("http://fs.example/file/{}#d"_format(next_id++)); + a->set_key(std::string(32, 'k')); + a->set_size(10); + if (!ctype.empty()) + a->set_contenttype(std::string{ctype}); + if (!name.empty()) + a->set_filename(std::string{name}); + if (flags) + a->set_flags(flags); + }; + + // Attachments and no body at all: the case that rendered as a blank row, because an empty + // preview string could not say whether there was a message. + deliver(*c, gallery, "", from_epoch_ms(1000), "h1", "", std::nullopt, [&](auto& data) { + add(data, "image/png", "kitten.png"); + add(data, "image/jpeg", "puppy.jpg"); + }); + // Not all images, so a row must not offer to show them as one. The second file carries no name + // at all, which a sender is free to omit. + deliver(*c, mixed, "", from_epoch_ms(2000), "h2", "", std::nullopt, [&](auto& data) { + add(data, "image/png", "chart.png"); + add(data, "application/pdf", ""); + }); + // A voice message, which a row names rather than counts. + deliver(*c, voice, "", from_epoch_ms(3000), "h3", "", std::nullopt, [&](auto& data) { + add(data, "audio/ogg", "clip.ogg", 1); + }); + // Body *and* attachments together -- the combination that rules out describing a preview with a + // single kind enum, since a row wants both halves. + deliver(*c, + wordy, + "look at this", + from_epoch_ms(4000), + "h4", + "", + std::nullopt, + [&](auto& data) { add(data, "image/png", "photo.png"); }); + sync(*c); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 4); + + auto preview_of = [&](const SenderKeys& who) { + auto id = ConversationId::dm(who.session_id); + auto found = std::ranges::find_if(convos, [&](const auto& c) { return c.id() == id; }); + REQUIRE(found != convos.end()); + REQUIRE(found->last_preview()); + return *found->last_preview(); + }; + + auto g = preview_of(gallery); + CHECK(g.body.empty()); + // Named, and in the order the sender listed them rather than whatever the table hands back. + CHECK(g.filenames == std::vector{"kitten.png", "puppy.jpg"}); + CHECK(g.all_images); + CHECK_FALSE(g.voice_message); + CHECK_FALSE(g.outgoing); + + auto m = preview_of(mixed); + // The unnamed file still occupies its place, so the count stays right and the entries stay + // aligned with the attachment indices -- a row draws its own fallback for the empty one. + CHECK(m.filenames == std::vector{"chart.png", ""}); + CHECK_FALSE(m.all_images); + + auto v = preview_of(voice); + CHECK(v.filenames == std::vector{"clip.ogg"}); + CHECK(v.voice_message); + CHECK_FALSE(v.all_images); + + auto w = preview_of(wordy); + CHECK(w.body == "look at this"); + CHECK(w.filenames == std::vector{"photo.png"}); + CHECK(w.all_images); +} + +TEST_CASE("Client: a text-only message previews no attachments", "[client][attachments]") { + TempClient c; + SenderKeys peer; + approve(*c, peer.session_id); + + deliver(*c, peer, "just words", from_epoch_ms(1000), "h1"); + sync(*c); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + REQUIRE(convos[0].last_preview()); + // A message with no attachments does not come back from the aggregate query at all, so this is + // what says the defaults it was left with are the right ones. + CHECK(convos[0].last_preview()->body == "just words"); + CHECK(convos[0].last_preview()->filenames.empty()); + CHECK_FALSE(convos[0].last_preview()->all_images); + CHECK_FALSE(convos[0].last_preview()->voice_message); +} diff --git a/tests/test_client/common.hpp b/tests/test_client/common.hpp new file mode 100644 index 000000000..2c2e4d37a --- /dev/null +++ b/tests/test_client/common.hpp @@ -0,0 +1,246 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../test_helper.hpp" + +using namespace session; +using namespace session::client; +using namespace std::literals; +using namespace oxenc::literals; + +/// Named rather than anonymous, and `inline` rather than `static`: nine translation units include +/// this, and an anonymous namespace would give each its own copy of every helper and a warning for +/// each one it happens not to use. The using-directive at the bottom is what keeps the test bodies +/// reading as they did when they were all one file. +namespace client_test { + +struct SenderKeys { + b32 ed_pk; + b64 ed_sk; + b33 session_id; + + SenderKeys() { + ed25519::keypair(ed_pk, ed_sk); + ed25519::pk_to_session_id(session_id, ed_pk); + } +}; + +/// RAII Client over a unique temporary database, mirroring TempCore. Unlike TempCore this can +/// close and reopen the same file, which is how the restart behaviour is exercised. +struct TempClient { + std::filesystem::path path; + std::unique_ptr client; + + template + explicit TempClient(Opts&&... opts) : + path{std::filesystem::temp_directory_path() / + fmt::format("{}.db", random::unique_id("test_client", 7))}, + client{std::make_unique(path, std::forward(opts)...)} {} + + template + explicit TempClient(callbacks cbs, Opts&&... opts) : + path{std::filesystem::temp_directory_path() / + fmt::format("{}.db", random::unique_id("test_client", 7))}, + client{std::make_unique(path, std::move(cbs), std::forward(opts)...)} {} + + template + void reopen(Opts&&... opts) { + client.reset(); + client = std::make_unique(path, std::forward(opts)...); + } + + ~TempClient() { + client.reset(); + std::error_code ec; + std::filesystem::remove(path, ec); + } + + Client* operator->() { return client.get(); } + Client& operator*() { return *client; } +}; + +/// A cache directory that removes itself, so a failing assertion cannot leave one behind. +struct TempCacheDir { + std::filesystem::path path{ + std::filesystem::temp_directory_path() / + fmt::format("{}", random::unique_id("test_cache", 8))}; + + TempCacheDir() { std::filesystem::create_directories(path); } + ~TempCacheDir() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } +}; + +inline b33 own_sid(Client& c) { + b33 out; + std::ranges::copy(c.core.globals.session_id(), out.begin()); + return out; +} + +/// `c`'s own sending keys, for building the copy of an outgoing message that Session stores on the +/// sender's own swarm. +inline SenderKeys self_keys(Client& c) { + SenderKeys k; + auto seed = c.core.globals.account_seed(); + std::ranges::copy(seed.ed25519_secret(), k.ed_sk.begin()); + std::ranges::copy(seed.ed25519_secret().last<32>(), k.ed_pk.begin()); + std::ranges::copy(c.core.globals.session_id(), k.session_id.begin()); + return k; +} + +/// Marks an account as approved, which is what having written to them would have done. +/// +/// A stranger's first message is a message request, so a test that is about anything else -- the +/// ordering of the list, what a priority does, what a handler is told -- has to say that this is an +/// ordinary conversation, or the list it is asking about is empty. +inline void approve(Client& c, const b33& sid) { + c.core.loop().call_get([&] { + auto conn = c.core.database().conn(); + conn.prepared_exec("INSERT OR IGNORE INTO accounts (session_id) VALUES (?)", sid); + conn.prepared_exec( + R"( + INSERT INTO contacts (account, approved) + VALUES ((SELECT id FROM accounts WHERE session_id = ?), 1) + ON CONFLICT (account) DO UPDATE SET approved = 1 + )", + sid); + return 0; + }); +} + +/// Builds, encrypts and delivers a v1 DM into `to` as if it had arrived from the swarm. +inline void deliver( + Client& to, + const SenderKeys& from, + std::string_view body, + sys_ms ts, + std::string hash, + std::string_view display_name = "", + std::optional sync_target = std::nullopt, + const std::function& decorate = nullptr, + std::optional msgid = std::nullopt) { + SessionProtos::Content content; + content.set_sigtimestamp(static_cast(ts.time_since_epoch().count())); + if (msgid) + content.set_msgid(*msgid); + auto* data = content.mutable_datamessage(); + data->set_body(std::string{body}); + if (!display_name.empty()) + data->mutable_profile()->set_displayname(std::string{display_name}); + if (sync_target) + data->set_synctarget(oxenc::to_hex(sync_target->begin(), sync_target->end())); + if (decorate) + decorate(*data); + + auto plaintext = content.SerializeAsString(); + auto encoded = encode_dm_v1( + std::as_bytes(std::span{plaintext}), from.ed_sk, ts, own_sid(to), std::nullopt); + + core::SwarmMessage sm{encoded, std::move(hash), ts, from_epoch_ms(1'000'000'000'000)}; + + // Core delivers arriving messages from its event loop, so do the same here rather than writing + // the database from the test thread: the connection pool is single-threaded by design. + to.core.loop().call_get([&] { + to.core.receive_messages({&sm, 1}, config::Namespace::Default, true); + return 0; + }); +} + +/// Records every callback so a test can assert on what a subscriber was told, and in what order. +struct Recorder { + std::vector order; + std::vector added, updated; + std::vector removed; + std::vector> replaced, requests_replaced; + std::vector> msg_added, msg_updated; + + callbacks handlers() { + return { + .conversation_added = + [this](AnyConversation&& c) { + order.push_back("added"); + added.push_back(std::move(c)); + }, + .conversation_updated = + [this](AnyConversation&& c) { + order.push_back("updated"); + updated.push_back(std::move(c)); + }, + .conversation_removed = + [this](const ConversationId& id) { + order.push_back("removed"); + removed.push_back(id); + }, + .conversation_list_replaced = + [this](std::vector&& l) { + order.push_back("replaced"); + replaced.push_back(std::move(l)); + }, + .request_list_replaced = + [this](std::vector&& l) { + order.push_back("requests"); + requests_replaced.push_back(std::move(l)); + }, + .message_added = + [this](const ConversationId& id, Message&& m) { + order.push_back("message"); + msg_added.emplace_back(id, std::move(m)); + }, + .message_updated = + [this](const ConversationId& id, Message&& m) { + order.push_back("message_updated"); + msg_updated.emplace_back(id, std::move(m)); + }, + }; + } +}; + +/// Waits for work Client deferred onto the loop -- the coalesced conversation_updated -- to have +/// run, by queueing a job behind it and waiting on that. +/// +/// **This does not settle everything.** `process_job_queue` swaps the queue out and drains only +/// what was in it, so a `call_soon` issued from *within* a job lands in the next batch and needs +/// another `sync` to run. Worse, a job that a transaction queued may run before that transaction +/// commits, since the commit happens further up the stack than the job knows about -- so a second +/// connection can see the row as it was, or not at all. A test that waits on state written that +/// way is testing the scheduler. Wait on the observable outcome instead. +inline void sync(Client& c) { + c.core.loop().call_get([] { return 0; }); +} + +/// The body of a conversation's last-message preview, or "" if it has no preview at all. +/// +/// So that an assertion about the body reads as one: dereferencing the optional in the CHECK itself +/// would make "there is no preview" undefined behaviour rather than a failure, and a test that +/// crashes instead of failing tells you nothing about which of the two went wrong. A test that +/// cares about the difference asserts on `last_preview()` directly. +inline std::string preview_body(const AnyConversation& c) { + const auto& p = c.last_preview(); + return p ? p->body : ""; +} + +} // namespace client_test + +using namespace client_test; diff --git a/tests/test_client/config_helpers.hpp b/tests/test_client/config_helpers.hpp new file mode 100644 index 000000000..5ca98d491 --- /dev/null +++ b/tests/test_client/config_helpers.hpp @@ -0,0 +1,142 @@ +#pragma once + +#include "common.hpp" + +/// Building what "another device on this account" pushed, and feeding it back in as a poll would. +/// Shared by every test whose subject is reconciliation between the configs and the database. +namespace client_test { + +/// What another device on this account pushed to its UserProfile. Another device is exactly this: +/// a second config object holding the same account key. +inline std::vector> profile_from_another_device( + Client& c, const std::function& change) { + // The other device has seen what we published, rather than being invented alongside us: it is + // built from our own dump once ours has gone out. Starting it from nothing would make a rival + // at the same seqno, which is a different scenario entirely -- and one that resolves by merging + // the two sets of changes rather than by taking theirs. + auto& ours = c.core.configs.user_profile(); + auto [seqno, messages, obsolete] = ours.push(); + ours.confirm_pushed(seqno, {"ourprofile"}); + + auto seed = c.core.globals.account_seed(); + config::UserProfile theirs{seed.ed25519_secret(), ours.make_dump()}; + change(theirs); + auto [their_seqno, their_messages, their_obsolete] = theirs.push(); + return their_messages; +} + +/// Feeds them in as a poll would. A SwarmMessage points at its data rather than owning it, so +/// `messages` has to outlive this call. +inline void merge_profile(Client& c, const std::vector>& messages) { + std::vector incoming; + for (size_t i = 0; i < messages.size(); i++) { + core::SwarmMessage m; + m.hash = fmt::format("profilehash{}", i); + m.data = messages[i]; + incoming.push_back(std::move(m)); + } + c.core.receive_messages(incoming, config::Namespace::UserProfile, true); +} + +inline ConversationId self_convo(Client& c) { + return ConversationId::dm(c.core.globals.session_id()); +} + +inline bool listed(Client& c, const ConversationId& id) { + auto all = c.conversations(await); + return std::ranges::any_of(all, [&](const auto& x) { return x.id() == id; }); +} + +/// What another device pushed to its Contacts config, having set up one contact however the caller +/// says. +inline std::vector> contacts_from_another_device( + Client& c, + std::string_view session_id, + const std::function& change) { + auto seed = c.core.globals.account_seed(); + config::Contacts theirs{seed.ed25519_secret(), std::nullopt}; + auto entry = theirs.get_or_construct(std::string{session_id}); + change(entry); + theirs.set(entry); + auto [seqno, messages, obsolete] = theirs.push(); + return messages; +} + +/// A further push from a device that has seen ours: built from our own dump once ours has gone out, +/// so it descends from our history rather than being a rival at the same seqno. A rival merges to +/// the union of the two, which is right but is never what a test about *removal* wants. +inline std::vector> contacts_update_from_another_device( + Client& c, const std::function& change) { + auto& ours = c.core.configs.contacts(); + auto [seqno, messages, obsolete] = ours.push(); + ours.confirm_pushed(seqno, {"ourcontacts"}); + + auto seed = c.core.globals.account_seed(); + config::Contacts theirs{seed.ed25519_secret(), ours.make_dump()}; + change(theirs); + auto [their_seqno, their_messages, their_obsolete] = theirs.push(); + return their_messages; +} + +inline ConversationId dm_from_hex(std::string_view hex) { + auto raw = oxenc::from_hex(hex); + b33 sid; + std::memcpy(sid.data(), raw.data(), sid.size()); + return ConversationId::dm(sid); +} + +/// Puts a message into a DM at a chosen moment, which is what a test about deleting by timestamp +/// needs and what send_message cannot give it. +inline void insert_message( + Client& c, const ConversationId& id, int64_t timestamp, std::string body) { + auto conn = c.core.database().conn(); + conn.prepared_exec( + R"( + INSERT INTO messages (conversation, sender, outgoing, timestamp, body) + VALUES ((SELECT c.id FROM conversations c JOIN accounts a ON a.id = c.dm + WHERE a.session_id = ?1), + (SELECT id FROM accounts WHERE session_id = ?1), 0, ?2, ?3) + )", + id.session_id(), + timestamp, + body); +} + +/// A ConvoInfoVolatile update from a device that has seen ours, built the same way and for the same +/// reason as `contacts_update_from_another_device`. +inline std::vector> volatile_from_another_device( + Client& c, const std::function& change) { + auto& ours = c.core.configs.convo_info_volatile(); + auto [seqno, messages, obsolete] = ours.push(); + ours.confirm_pushed(seqno, {"ourvolatile"}); + + auto seed = c.core.globals.account_seed(); + config::ConvoInfoVolatile theirs{seed.ed25519_secret(), ours.make_dump()}; + change(theirs); + auto [their_seqno, their_messages, their_obsolete] = theirs.push(); + return their_messages; +} + +inline void merge_volatile(Client& c, const std::vector>& messages) { + std::vector incoming; + for (size_t i = 0; i < messages.size(); i++) { + core::SwarmMessage m; + m.hash = fmt::format("volatilehash{}", i); + m.data = messages[i]; + incoming.push_back(std::move(m)); + } + c.core.receive_messages(incoming, config::Namespace::ConvoInfoVolatile, true); +} + +inline void merge_contacts(Client& c, const std::vector>& messages) { + std::vector incoming; + for (size_t i = 0; i < messages.size(); i++) { + core::SwarmMessage m; + m.hash = fmt::format("contacthash{}", i); + m.data = messages[i]; + incoming.push_back(std::move(m)); + } + c.core.receive_messages(incoming, config::Namespace::Contacts, true); +} + +} // namespace client_test diff --git a/tests/test_client/configs.cpp b/tests/test_client/configs.cpp new file mode 100644 index 000000000..9e9f4331d --- /dev/null +++ b/tests/test_client/configs.cpp @@ -0,0 +1,571 @@ +#include "config_helpers.hpp" + +TEST_CASE("Client: a merged contact reaches all three tables", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, 'a'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { + e.set_name("Padmé"); + e.set_nickname("Pad"); + e.approved = true; + e.approved_me = true; + e.priority = 3; + e.exp_mode = config::expiration_mode::after_read; + e.exp_timer = std::chrono::seconds{86400}; + }); + merge_contacts(*c.client, pushed); + + // The conversation exists because the config says the contact does, not because anything has + // been said in it. + auto convo = c->conversation(id, await); + REQUIRE(convo); + CHECK(convo->priority() == 3); + + // Nickname wins over name for display, which is what the split is for. + CHECK(convo->display_name() == "Pad"); + + auto conn = c->core.database().conn(); + CHECK(conn.prepared_get( + "SELECT name FROM accounts WHERE session_id = ?", id.session_id()) == "Padmé"); + auto [approved, approved_me, blocked] = conn.prepared_get( + R"(SELECT approved, approved_me, blocked FROM contacts + WHERE account = (SELECT id FROM accounts WHERE session_id = ?))", + id.session_id()); + CHECK(approved == 1); + CHECK(approved_me == 1); + CHECK(blocked == 0); +} + +TEST_CASE("Client: re-deriving a contact changes nothing", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, 'b'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { + e.set_name("Leia"); + e.set_nickname("Lei"); + e.approved = true; + e.approved_me = true; + e.blocked = false; + e.priority = 7; + e.notifications = config::notify_mode::disabled; + e.mute_until = 1700000000; + e.exp_mode = config::expiration_mode::after_send; + e.exp_timer = std::chrono::seconds{600}; + e.created = 1690000000; + e.profile_updated = std::chrono::sys_seconds{std::chrono::seconds{1695000000}}; + }); + merge_contacts(*c.client, pushed); + + // The property that makes the mapping trustworthy: applying a config to the tables and then + // deriving a config back from those tables is the identity. Anything lost, rounded or + // defaulted on the way through shows up here as a config that went dirty -- and a mapping that + // dirties on every pass would push a pointless update after every merge, forever. + auto& contacts = c->core.configs.contacts(); + REQUIRE_FALSE(contacts.needs_push()); + TestHelper::sync_contact(*c.client, id); + CHECK_FALSE(contacts.needs_push()); + CHECK_FALSE(contacts.needs_dump()); +} + +TEST_CASE("Client: a contact removed elsewhere takes its history", "[client][configs]") { + std::vector gone; + callbacks cbs; + cbs.conversation_removed = [&](const ConversationId& id) { gone.push_back(id); }; + TempClient c{cbs}; + + auto them = "05" + std::string(64, 'c'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { + e.set_name("Anakin"); + e.approved = true; + }); + merge_contacts(*c.client, pushed); + REQUIRE(c->conversation(id, await)); + + auto conn = c->core.database().conn(); + auto account = conn.prepared_get( + "SELECT id FROM accounts WHERE session_id = ?", id.session_id()); + conn.prepared_exec( + R"(INSERT INTO messages (conversation, sender, outgoing, timestamp, body) + VALUES ((SELECT id FROM conversations WHERE dm = ?1), ?1, 0, 1000, 'hi'))", + account); + REQUIRE(c->conversation(id, await)->messages(await).size() == 1); + + // Now the other device removes them entirely. An absent entry can only mean the stronger + // thing, since hiding arrives as a negative priority instead. + auto emptied = contacts_update_from_another_device( + *c.client, [&](config::Contacts& theirs) { theirs.erase(them); }); + merge_contacts(*c.client, emptied); + + // Conversation and history both gone, and reported. + CHECK_FALSE(c->conversation(id, await)); + CHECK(conn.prepared_get("SELECT count(*) FROM messages WHERE sender = ?", account) == + 0); + CHECK(std::ranges::find(gone, id) != gone.end()); + + // But not the account: we may have seen them in a group, and their profile renders that. + CHECK(conn.prepared_get( + "SELECT count(*) FROM accounts WHERE session_id = ?", id.session_id()) == 1); + CHECK(conn.prepared_get("SELECT count(*) FROM contacts WHERE account = ?", account) == + 0); +} + +TEST_CASE( + "Client: a contact whose dump was lost is published, not destroyed", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, 'e'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { + e.set_name("Rey"); + e.approved = true; + }); + merge_contacts(*c.client, pushed); + REQUIRE(c->conversation(id, await)); + + // Stand in for a crash between committing the row and writing the dump: the tables hold a + // contact the config has never heard of. Reconciled inward first, that is indistinguishable + // from one deleted elsewhere and would be destroyed with its history. + REQUIRE(c->core.configs.contacts().erase(them)); + + c.reopen(); + + // Startup derives outward before reconciling inward, so it is published rather than deleted. + CHECK(c->conversation(id, await)); + CHECK(c->core.configs.contacts().get(them).has_value()); +} + +TEST_CASE("Client: a new account starts with note to self hidden", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + // Seeded at account creation rather than left at the default, because nts_priority is carried + // in the shared UserProfile config: a default of 0 would not merely show the conversation here, + // it would make it appear on every other device on the account once they synced. + CHECK(c->core.configs.user_profile().get_nts_priority() == -1); + CHECK_FALSE(listed(*c.client, me)); +} + +TEST_CASE("Client: writing a note to self reveals it", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + REQUIRE_FALSE(listed(*c.client, me)); + + c->send_message(me, {.body = "a reminder"}, await); + + // Both halves: it is in our own list, and UserProfile says so, which is what stops the other + // devices on the account from carrying on hiding it. + CHECK(listed(*c.client, me)); + CHECK(c->core.configs.user_profile().get_nts_priority() == 0); + CHECK(c->core.configs.user_profile().needs_push()); +} + +TEST_CASE("Client: revealing note to self keeps a pin it already had", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + auto pinned = profile_from_another_device(*c.client, [](auto& p) { p.set_nts_priority(7); }); + merge_profile(*c.client, pinned); + REQUIRE(c->conversation(me, await)->priority() == 7); + + c->send_message(me, {.body = "a reminder"}, await); + + // Already visible, so there is nothing to reveal and the pin is left where the user put it. + CHECK(c->core.configs.user_profile().get_nts_priority() == 7); + CHECK(c->conversation(me, await)->priority() == 7); +} + +TEST_CASE("Client: our own profile reaches the conversation", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + auto pushed = profile_from_another_device(*c.client, [](auto& p) { + p.set_name("Leia"); + p.set_nts_priority(0); // another device unhid it + }); + merge_profile(*c.client, pushed); + + auto convo = c->conversation(me, await); + REQUIRE(convo); + CHECK(convo->display_name() == "Leia"); + CHECK(convo->dm()->note_to_self); + CHECK(listed(*c.client, me)); +} + +TEST_CASE("Client: hiding note to self elsewhere keeps it out of the list", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + // Visible first, so the hide is a change rather than the initial state. + auto shown = profile_from_another_device(*c.client, [](auto& p) { + p.set_name("Leia"); + p.set_nts_priority(0); + }); + merge_profile(*c.client, shown); + REQUIRE(listed(*c.client, me)); + + auto hidden = profile_from_another_device(*c.client, [](auto& p) { + p.set_name("Leia"); + p.set_nts_priority(-1); + }); + merge_profile(*c.client, hidden); + + // Still reachable by name -- hiding is a statement about the list, not about existence -- but + // gone from it. + auto convo = c->conversation(me, await); + REQUIRE(convo); + CHECK(convo->priority() == -1); + CHECK_FALSE(listed(*c.client, me)); +} + +TEST_CASE("Client: a note-to-self timer waits for the conversation", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + // A timer set on another device while we have no note-to-self conversation. There is nothing + // to attach it to yet, and that is not a loss: the config is where it lives until there is. + auto pushed = profile_from_another_device( + *c.client, [](auto& p) { p.set_nts_expiry(std::chrono::seconds{600}); }); + merge_profile(*c.client, pushed); + REQUIRE_FALSE(c->conversation(me, await)); + + // Writing a note brings the conversation into being, and everything the config was holding for + // it lands at that moment rather than being lost. + c->send_message(me, {.body = "a reminder"}, await); + REQUIRE(c->conversation(me, await)); + + auto [mode, timer] = c->core.database().conn().prepared_get( + "SELECT exp_mode, exp_timer FROM conversations" + " WHERE dm = (SELECT id FROM accounts WHERE session_id = ?)", + c->core.globals.session_id()); + + // The config carries a duration and no mode, because only one mode means anything when the + // reader is also the writer: there is no moment at which someone else reads it. + CHECK(mode == static_cast(config::expiration_mode::after_send)); + CHECK(timer == 600); +} + +TEST_CASE("Client: a restart reconciles what nothing announced", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + auto pushed = profile_from_another_device(*c.client, [](auto& p) { + p.set_name("Leia"); + p.set_nts_priority(0); + }); + merge_profile(*c.client, pushed); + REQUIRE(c->conversation(me, await)->display_name() == "Leia"); + + // Put the database behind the config behind its back, which is what a crash between merging and + // reconciling leaves -- or a config merged by a version that could not yet reconcile it. In + // neither case is a further notification owed, so nothing would ever come back for it. + c->core.database().conn().prepared_exec( + "UPDATE accounts SET name = NULL WHERE session_id = ?", c->core.globals.session_id()); + REQUIRE(c->conversation(me, await)->display_name().empty()); + + c.reopen(); + + // Starting up reconciles regardless of whether anything changed, so it is repaired. + CHECK(c->conversation(me, await)->display_name() == "Leia"); +} + +TEST_CASE("Client: reconciling twice does not disturb the list", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + + auto pushed = profile_from_another_device(*c.client, [](auto& p) { + p.set_name("Leia"); + p.set_nts_priority(0); + }); + merge_profile(*c.client, pushed); + + auto before = c->conversation(me, await); + REQUIRE(before); + + // The same profile again reconciles again, since the seqno detector overfires on an identical + // config, so reconciliation has to be idempotent. The specific trap is ensure_conversation, + // which bumps last_activity on a row that already exists -- called unguarded it would shuffle + // note to self to the top of the list on every single config merge. + merge_profile(*c.client, pushed); + + auto after = c->conversation(me, await); + REQUIRE(after); + CHECK(after->last_activity() == before->last_activity()); + CHECK(after->display_name() == before->display_name()); +} + +TEST_CASE("Client: blocking someone makes them a contact", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, '1'); + auto id = dm_from_hex(them); + + // Someone we have merely seen: an account row and nothing else, which is what an unanswered + // message request looks like. + { + auto conn = c->core.database().conn(); + conn.prepared_exec("INSERT INTO accounts (session_id) VALUES (?)", id.session_id()); + } + REQUIRE_FALSE(c->core.configs.contacts().get(them)); + + // Through Client, not through a DM: there is no conversation here, which is exactly the case + // that carve-out exists for. + c->set_blocked(id, true, await); + + // The block has to be synced and the entry is the only place it can live, so blocking makes + // one. It does not approve them: refusing someone's messages is not accepting them. + auto entry = c->core.configs.contacts().get(them); + REQUIRE(entry); + CHECK(entry->blocked); + CHECK_FALSE(entry->approved); + + c->set_blocked(id, false, await); + REQUIRE(c->core.configs.contacts().get(them)); + CHECK_FALSE(c->core.configs.contacts().get(them)->blocked); +} + +TEST_CASE("Client: clearing a conversation says when it was cleared", "[client][configs]") { + std::vector reloaded; + callbacks cbs; + cbs.history_replaced = [&](const ConversationId& id) { reloaded.push_back(id); }; + TempClient c{cbs}; + + auto them = "05" + std::string(64, '2'); + auto id = dm_from_hex(them); + c->open_dm(id, await); + insert_message(*c.client, id, 1000, "hi"); + REQUIRE(c->conversation(id, await)->messages(await).size() == 1); + + auto before = std::chrono::floor(clock_now_ms()); + c->conversation(id, await)->clear_messages(await); + + CHECK(c->conversation(id, await)->messages(await).empty()); + CHECK(c->conversation(id, await)); // The conversation stays; only its history went. + CHECK(std::ranges::find(reloaded, id) != reloaded.end()); + + // And the moment is recorded rather than the deletion being local, so a device that has been + // offline through all of this deletes the same messages when it catches up. + auto entry = c->core.configs.contacts().get(them); + REQUIRE(entry); + CHECK(entry->delete_before >= before); +} + +TEST_CASE("Client: deleting a conversation keeps the contact", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, '3'); + auto id = dm_from_hex(them); + c->open_dm(id, await); + c->conversation(id, await)->set_priority(5, await); + insert_message(*c.client, id, 1000, "hi"); + REQUIRE(listed(*c.client, id)); + + c->conversation(id, await)->delete_conversation(await); + + CHECK_FALSE(listed(*c.client, id)); + CHECK(c->conversation(id, await)->messages(await).empty()); + + auto entry = c->core.configs.contacts().get(them); + REQUIRE(entry); // Still a contact, so a message from them brings the conversation back. + CHECK(entry->approved); + CHECK(entry->priority == -1); // The pin it had is not among the things kept. + CHECK(entry->delete_before > std::chrono::sys_seconds{}); +} + +TEST_CASE("Client: hiding note to self keeps what is in it", "[client][configs]") { + TempClient c; + auto me = self_convo(*c.client); + c->open_dm(me, await); + insert_message(*c.client, me, 1000, "note"); + REQUIRE(listed(*c.client, me)); + + c->conversation(me, await)->delete_conversation(/*keep_messages=*/true, await); + + CHECK_FALSE(listed(*c.client, me)); + CHECK(c->conversation(me, await)->messages(await).size() == 1); + CHECK(c->core.configs.user_profile().get_nts_priority() == -1); + + // No instruction to destroy anything, which is the whole difference between hiding a + // conversation and deleting one. + CHECK(c->core.configs.user_profile().get_nts_delete_before() == std::chrono::sys_seconds{}); +} + +TEST_CASE("Client: deleting a contact takes the entry that held the block", "[client][configs]") { + std::vector gone; + callbacks cbs; + cbs.conversation_removed = [&](const ConversationId& id) { gone.push_back(id); }; + TempClient c{cbs}; + + auto them = "05" + std::string(64, '4'); + auto id = dm_from_hex(them); + c->open_dm(id, await); + c->dm(id, await)->set_blocked(true, await); + insert_message(*c.client, id, 1000, "hi"); + + c->dm(id, await)->delete_contact(await); + + CHECK_FALSE(c->conversation(id, await)); + CHECK(std::ranges::find(gone, id) != gone.end()); + + // No entry means no delete-before instruction is owed: another device merging this drops the + // conversation and its history because the contact is gone, not because it was told to. It + // also means the block is gone, since the entry was the only thing holding it. + CHECK_FALSE(c->core.configs.contacts().get(them)); + + auto conn = c->core.database().conn(); + CHECK(conn.prepared_get("SELECT count(*) FROM messages") == 0); + CHECK(conn.prepared_get( + "SELECT count(*) FROM accounts WHERE session_id = ?", id.session_id()) == 1); +} + +TEST_CASE("Client: a delete-before from another device destroys history", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, '5'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { e.approved = true; }); + merge_contacts(*c.client, pushed); + REQUIRE(c->conversation(id, await)); + + insert_message(*c.client, id, 1'000'000, "old"); + insert_message(*c.client, id, 3'000'000, "new"); + REQUIRE(c->conversation(id, await)->messages(await).size() == 2); + + // Retroactive: what the instruction is about is the history that was there when someone chose + // to destroy it, not merely what arrives after it. + auto cleared = contacts_update_from_another_device(*c.client, [&](config::Contacts& theirs) { + auto e = theirs.get_or_construct(them); + e.delete_before = std::chrono::sys_seconds{2000s}; + theirs.set(e); + }); + merge_contacts(*c.client, cleared); + + auto left = c->conversation(id, await)->messages(await); + REQUIRE(left.size() == 1); + CHECK(left[0].body == "new"); +} + +TEST_CASE("Client: approval is not walked back by a merge", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, '7'); + auto id = dm_from_hex(them); + + c->open_dm(id, await); + REQUIRE(c->core.configs.contacts().get(them)); + REQUIRE(c->core.configs.contacts().get(them)->approved); + + // Another client clearing both flags on its way to deleting the contact, merged without the + // deletion that was to follow. Copied verbatim this would file the conversation back under + // message requests -- and there is no message anyone could send to put it back. + auto unapproved = contacts_update_from_another_device(*c.client, [&](config::Contacts& theirs) { + auto e = theirs.get_or_construct(them); + e.approved = false; + e.approved_me = false; + theirs.set(e); + }); + merge_contacts(*c.client, unapproved); + + CHECK(c->message_requests(await).empty()); + REQUIRE(c->conversation(id, await)); + CHECK_FALSE(c->conversation(id, await)->dm()->request); +} + +TEST_CASE("Client: a delete-before is not walked back", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, '6'); + auto id = dm_from_hex(them); + c->open_dm(id, await); + + // Another device cleared at a moment this one has not reached yet -- clock skew is enough for + // that. Publishing our own, smaller value would tell it to un-delete what it destroyed. + auto& contacts = c->core.configs.contacts(); + auto later = std::chrono::floor(clock_now_ms()) + 1h; + auto entry = contacts.get_or_construct(them); + entry.delete_before = later; + contacts.set(entry); + + c->conversation(id, await)->clear_messages(await); + + REQUIRE(contacts.get(them)); + CHECK(contacts.get(them)->delete_before == later); +} + +TEST_CASE("Client: our own profile is account state, not a conversation", "[client][configs]") { + TempClient c; + + // Nothing set yet, and -- the point of this being on Client -- no note-to-self conversation + // needed for the question to have an answer. + CHECK(c->conversations(await).empty()); + CHECK(c->display_name(await).empty()); + + c->set_display_name("Leia", await); + CHECK(c->display_name(await) == "Leia"); + CHECK(c->core.configs.user_profile().get_name() == "Leia"); + + // Still no conversation: setting a name is not writing to yourself. + CHECK(c->conversations(await).empty()); +} + +TEST_CASE("Client: the save-notification preference follows the account", "[client][configs]") { + TempClient c; + + // Session's default is to tell people when you save their files. + CHECK(c->notify_media_saved(await)); + + c->set_notify_media_saved(false, await); + CHECK_FALSE(c->notify_media_saved(await)); + CHECK_FALSE(c->core.configs.user_profile().get_notify_media_saved()); + + // What another device set reaches us through a merge, like any other profile field. + auto pushed = profile_from_another_device( + *c.client, [](config::UserProfile& p) { p.set_notify_media_saved(true); }); + merge_profile(*c.client, pushed); + CHECK(c->notify_media_saved(await)); +} + +TEST_CASE( + "Client: a conversation reports the picture it has been told about", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, 'b'); + auto id = dm_from_hex(them); + + std::vector key(32, std::byte{0x7}); + auto pushed = contacts_from_another_device(*c.client, them, [&](auto& e) { + e.set_name("Padmé"); + e.profile_picture = config::profile_pic{"http://fs.example/file/99#pubkey=aa", key}; + }); + merge_contacts(*c.client, pushed); + + auto convo = c->conversation(id, await); + REQUIRE(convo); + CHECK(convo->picture().url == "http://fs.example/file/99#pubkey=aa"); + CHECK(convo->picture().key == key); + + // Somebody we know nothing about has none, which is not the same as an error. + auto stranger = dm_from_hex("05" + std::string(64, 'c')); + c->open_dm(stranger, await); + CHECK(c->conversation(stranger, await)->picture().url.empty()); +} + +TEST_CASE("Client: no picture is nullopt rather than a failure", "[client][configs]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + c->open_dm(id, await); + + std::optional> got; + std::optional err; + bool called = false; + c->profile_picture(id, [&](std::optional e, auto pic) { + err = std::move(e); + got = std::move(pic); + called = true; + }); + sync(*c); + + REQUIRE(called); + CHECK_FALSE(err.has_value()); + CHECK_FALSE(got.has_value()); +} diff --git a/tests/test_client/conversation_api.cpp b/tests/test_client/conversation_api.cpp new file mode 100644 index 000000000..04ad48add --- /dev/null +++ b/tests/test_client/conversation_api.cpp @@ -0,0 +1,216 @@ +#include "config_helpers.hpp" + +TEST_CASE("Client: a conversation reports the settings it carries", "[client][convos]") { + TempClient c; + auto them = "05" + std::string(64, 'a'); + auto id = dm_from_hex(them); + c->open_dm(id, await); + + auto convo = [&] { return *c->conversation(id, await); }; + + // Defaults, and the ones a screen needs in order to draw a toggle rather than a button. + CHECK(convo().notifications() == config::notify_mode::defaulted); + CHECK(convo().mute_until() == std::chrono::sys_seconds{}); + CHECK(convo().exp_mode() == config::expiration_mode::none); + CHECK_FALSE(convo().dm()->blocked); + + c->conversation(id, await)->set_notifications(config::notify_mode::disabled, await); + c->conversation(id, await)->set_mute_until(std::chrono::sys_seconds{1700000000s}, await); + c->conversation(id, await)->set_expiry(config::expiration_mode::after_read, 86400s, await); + c->set_blocked(id, true, await); + c->dm(id, await)->set_nickname("Bilbo", await); + + CHECK(convo().notifications() == config::notify_mode::disabled); + CHECK(convo().mute_until() == std::chrono::sys_seconds{1700000000s}); + CHECK(convo().exp_mode() == config::expiration_mode::after_read); + CHECK(convo().exp_timer() == 86400s); + CHECK(convo().dm()->blocked); + + // The two halves display_name merges. A row wants the merge; a screen that edits the nickname + // has to show both, and cannot when only the resolved answer arrives. + CHECK(convo().dm()->nickname == "Bilbo"); + CHECK(convo().display_name() == "Bilbo"); + CHECK(convo().dm()->name.empty()); + + // All of it reaches the Contacts config, which is what makes it follow the account rather than + // the device. + auto entry = c->core.configs.contacts().get(them); + REQUIRE(entry); + CHECK(entry->notifications == config::notify_mode::disabled); + CHECK(entry->mute_until == 1700000000); + CHECK(entry->exp_mode == config::expiration_mode::after_read); + CHECK(entry->exp_timer == 86400s); + CHECK(entry->blocked); + CHECK(entry->nickname == "Bilbo"); + + // Clearing the nickname falls back to what they call themselves. + c->dm(id, await)->set_nickname("", await); + CHECK(convo().dm()->nickname.empty()); + CHECK_FALSE(c->core.configs.contacts().get(them)->nickname == "Bilbo"); + + // A timer without a mode expires nothing, so it is not stored as though it were a setting. + c->conversation(id, await)->set_expiry(config::expiration_mode::none, 3600s, await); + CHECK(convo().exp_mode() == config::expiration_mode::none); + CHECK(convo().exp_timer() == 0s); +} + +TEST_CASE("Client: settings from another device reach the conversation", "[client][configs]") { + TempClient c; + auto them = "05" + std::string(64, 'b'); + auto id = dm_from_hex(them); + + auto pushed = contacts_from_another_device(*c.client, them, [](auto& e) { + e.set_name("Frodo"); + e.set_nickname("Mr Underhill"); + e.approved = true; + e.blocked = true; + e.notifications = config::notify_mode::disabled; + e.mute_until = 1700000000; + e.exp_mode = config::expiration_mode::after_send; + e.exp_timer = 600s; + }); + merge_contacts(*c.client, pushed); + + auto convo = c->conversation(id, await); + REQUIRE(convo); + CHECK(convo->notifications() == config::notify_mode::disabled); + CHECK(convo->mute_until() == std::chrono::sys_seconds{1700000000s}); + CHECK(convo->exp_mode() == config::expiration_mode::after_send); + CHECK(convo->exp_timer() == 600s); + REQUIRE(convo->dm()); + CHECK(convo->dm()->blocked); + CHECK(convo->dm()->name == "Frodo"); + CHECK(convo->dm()->nickname == "Mr Underhill"); + CHECK(convo->display_name() == "Mr Underhill"); +} + +TEST_CASE("Client: a page size has to be a page", "[client][convos]") { + TempClient c; + auto id = dm_from_hex("05" + std::string(64, 'a')); + c->open_dm(id, await); + c->send_message(id, {.body = "hi"}, await); + + // Unchecked, these reach SQLite as `LIMIT ?`, where a negative is no limit at all: asking for + // one message would load every message in the conversation. + for (int limit : {0, -1, -50}) { + CHECK_THROWS_AS(c->conversation(id, await)->messages(limit, await), std::invalid_argument); + CHECK_THROWS_AS( + c->conversation(id, await)->messages(limit, std::nullopt, true, await), + std::invalid_argument); + } + + // The handler form refuses on the calling thread too, rather than reporting it: the caller is + // still there to catch, and a bad page size is its bug rather than a runtime condition. + auto ignore = [](std::optional, std::vector) {}; + CHECK_THROWS_AS(c->conversation(id, await)->messages(0, ignore), std::invalid_argument); + + CHECK(c->conversation(id, await)->messages(1, await).size() == 1); +} + +TEST_CASE("Client: a conversation knows which kind it is", "[client][convos]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + c->open_dm(id, await); + + auto convo = c->conversation(id, await); + REQUIRE(convo); + + // The kind is what makes a question askable: `request` is a thing only a DM can be, and asking + // it of a community should not compile rather than quietly answering false. + REQUIRE(convo->dm()); + CHECK_FALSE(convo->dm()->request); + CHECK_FALSE(convo->group()); + CHECK_FALSE(convo->community()); + + // Common fields reach through whichever kind it is. + CHECK(convo->id() == id); + CHECK(convo->unread() == 0); +} + +TEST_CASE( + "Client: a handler-form operation outlives the conversation it came from", + "[client][convos]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + approve(*c, them.session_id); + deliver(*c, them, "hi", from_epoch_ms(5000), "h1"); + REQUIRE(c->conversation(id, await)->unread() == 1); + + // The natural way to write any of these is on something that has already gone by the time the + // work runs: a temporary, a handler's parameter, or a list element whose list got replaced. So + // the operation must not reach back into the object -- and only the handler form can get this + // wrong, since the waiting form runs before it returns. + std::optional error = "not called"; + { + auto convo = c->conversation(id, await); + REQUIRE(convo); + convo->mark_read([&](auto err) { error = std::move(err); }); + } // convo destroyed here, before the loop has run the work + sync(*c); + + CHECK_FALSE(error.has_value()); + CHECK(c->conversation(id, await)->unread() == 0); + + // And on an outright temporary, which is how it reads at a call site. + std::optional paged = "not called"; + size_t got = 0; + c->conversation(id, await)->messages(50, [&](auto err, auto msgs) { + paged = std::move(err); + got = msgs.size(); + }); + sync(*c); + CHECK_FALSE(paged.has_value()); + CHECK(got == 1); +} + +TEST_CASE("Client: every conversation kind starts with its base", "[client][convos]") { + // What makes reading a common field off AnyConversation free: each alternative holds its + // `Conversation` at offset zero, so every arm of the variant's dispatch computes the same + // address and the compiler folds it away. Put another base ahead of `Conversation` and that + // silently becomes a runtime selection instead. + // + TempClient c; + Conversation base{*c.client, dm_from_hex("05" + std::string(64, '9'))}; + DM dm{base}; + Group group{base}; + Community community{base}; + CHECK(static_cast(&dm) == static_cast(&dm)); + CHECK(static_cast(&group) == static_cast(&group)); + CHECK(static_cast(&community) == static_cast(&community)); +} + +TEST_CASE("Client: auto-download is per conversation and stays here", "[client][convos]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + c->open_dm(id, await); + + // Unset means nobody has been asked, which is what a client prompts on. It is not `none`. + CHECK_FALSE(c->conversation(id, await)->auto_download().has_value()); + + c->conversation(id, await)->set_auto_download(AutoDownload::image_attachments, await); + CHECK(c->conversation(id, await)->auto_download() == AutoDownload::image_attachments); + + // Answering "none" is an answer: it records that the question was asked, so a client that + // prompts when unset does not prompt again. + c->conversation(id, await)->set_auto_download(AutoDownload::none, await); + auto after = c->conversation(id, await)->auto_download(); + REQUIRE(after.has_value()); + CHECK(*after == AutoDownload::none); + + // Device-local: nothing about it reaches the config that follows the account. Checked by + // deriving the contact outward and finding the config unchanged -- if this were synced, the + // setting above would have dirtied it. + auto& contacts = c->core.configs.contacts(); + TestHelper::sync_contact(*c.client, id); + auto before_push = contacts.needs_push(); + c->conversation(id, await)->set_auto_download(AutoDownload::all, await); + TestHelper::sync_contact(*c.client, id); + CHECK(contacts.needs_push() == before_push); + + // And it survives a restart, being a stored property rather than a session's opinion. + c.reopen(); + CHECK(c->conversation(id, await)->auto_download() == AutoDownload::all); +} diff --git a/tests/test_client/download_cache.cpp b/tests/test_client/download_cache.cpp new file mode 100644 index 000000000..f9a54c48b --- /dev/null +++ b/tests/test_client/download_cache.cpp @@ -0,0 +1,151 @@ +#include "../../src/client/download_cache.hpp" + +#include +#include + +#include "common.hpp" + +namespace cache = session::client::cache; + +namespace { + +// A temporary directory that removes itself, so a failing assertion cannot leave one behind. +struct TempDir { + std::filesystem::path path{ + std::filesystem::temp_directory_path() / + fmt::format("{}", session::random::unique_id("test_cache", 8))}; + + TempDir() { std::filesystem::create_directories(path); } + ~TempDir() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } +}; + +b32 a_key() { + b32 k; + session::random::fill(k); + return k; +} + +} // namespace + +TEST_CASE("Cache: a url names one file, whatever is hung off it", "[client][cache]") { + TempDir dir; + + auto base = cache::path_for(dir.path, cache::PROFILE_DIR, "http://fs.example/file/1234"); + auto fragment = cache::path_for( + dir.path, cache::PROFILE_DIR, "http://fs.example/file/1234#pubkey=abcdef"); + auto query = cache::path_for(dir.path, cache::PROFILE_DIR, "http://fs.example/file/1234?v=2"); + + // The bytes at the base url are the bytes; a fragment says how to reach and unpack them, and a + // query string is not part of which file this is. + CHECK(base == fragment); + CHECK(base == query); + + // A different file is a different entry. + CHECK(base != cache::path_for(dir.path, cache::PROFILE_DIR, "http://fs.example/file/5678")); + + // And the two kinds do not share a directory, so a sweep of one cannot see the other's files. + CHECK(base != cache::path_for(dir.path, cache::ATTACHMENT_DIR, "http://fs.example/file/1234")); + + // The name is a hash, not the url: usable as a filename whatever the url looked like. + CHECK(base.filename().string().size() == 64); + CHECK(base.filename().string().find('/') == std::string::npos); +} + +TEST_CASE("Cache: what goes in comes back out", "[client][cache]") { + TempDir dir; + auto key = a_key(); + auto file = cache::path_for(dir.path, cache::PROFILE_DIR, "http://fs.example/file/1"); + + CHECK_FALSE(cache::read(file, key).has_value()); + + std::vector data(5000); + session::random::fill(data); + cache::write(file, key, data); + + REQUIRE(std::filesystem::exists(file)); + auto got = cache::read(file, key); + REQUIRE(got); + CHECK(*got == data); + + // On disk it is not the plaintext: the file is bigger than what went in (header, macs, padding) + // and does not contain it. + auto on_disk = std::filesystem::file_size(file); + CHECK(on_disk > data.size()); + + // Another key does not open it, and the unreadable entry is dropped rather than left to fail + // forever. + auto other = a_key(); + CHECK_FALSE(cache::read(file, other).has_value()); + CHECK_FALSE(std::filesystem::exists(file)); +} + +TEST_CASE("Cache: a corrupted entry is a miss, not a throw", "[client][cache]") { + TempDir dir; + auto key = a_key(); + auto file = cache::path_for(dir.path, cache::ATTACHMENT_DIR, "http://fs.example/file/2"); + + std::vector data(100); + session::random::fill(data); + cache::write(file, key, data); + + { + std::ofstream out{file, std::ios::binary | std::ios::app}; + out << "rubbish"; + } + + CHECK_FALSE(cache::read(file, key).has_value()); + CHECK_FALSE(std::filesystem::exists(file)); +} + +TEST_CASE("Cache: listing offers up what a sweep may consider", "[client][cache]") { + TempDir dir; + auto key = a_key(); + + std::vector data(64); + session::random::fill(data); + + std::string kept = "http://fs.example/file/keep"; + std::string dropped = "http://fs.example/file/drop"; + cache::write(cache::path_for(dir.path, cache::PROFILE_DIR, kept), key, data); + cache::write(cache::path_for(dir.path, cache::PROFILE_DIR, dropped), key, data); + + // A download still running: no url references it yet, and unlinking it would fail the fetch for + // a reason nothing could explain. + auto partial = cache::path_for(dir.path, cache::PROFILE_DIR, "http://fs.example/file/busy"); + partial += "-abcdefgh"; + partial += std::string{cache::PARTIAL_SUFFIX}; + { + std::ofstream out{partial, std::ios::binary}; + out << "half a file"; + } + + auto listed = cache::list(dir.path, cache::PROFILE_DIR); + std::set names{listed.begin(), listed.end()}; + + // Two finished files and not the third: what is offered up is only what a sweep may act on. + CHECK(names.size() == 2); + CHECK(names.contains(cache::path_for(dir.path, cache::PROFILE_DIR, kept).filename().string())); + CHECK_FALSE(names.contains(partial.filename().string())); + + // The referencing url carries a fragment, as a stored one may, and still names the same file -- + // which is what lets a caller decide by url what to keep by name. + CHECK(names.contains(cache::path_for(dir.path, cache::PROFILE_DIR, kept + "#pubkey=aa") + .filename() + .string())); + + auto drop_name = cache::path_for(dir.path, cache::PROFILE_DIR, dropped).filename().string(); + CHECK(cache::remove(dir.path, cache::PROFILE_DIR, drop_name)); + CHECK_FALSE(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, dropped))); + CHECK(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, kept))); + CHECK(std::filesystem::exists(partial)); + + // Removing what is already gone is the ordinary outcome of two sweeps racing, not an error. + CHECK_FALSE(cache::remove(dir.path, cache::PROFILE_DIR, drop_name)); + + // A directory that was never created lists as empty rather than throwing: a client that has + // cached no attachments has no attachments directory. + CHECK(cache::list(dir.path, cache::ATTACHMENT_DIR).empty()); +} diff --git a/tests/test_client/ids_and_schema.cpp b/tests/test_client/ids_and_schema.cpp new file mode 100644 index 000000000..6606ecd08 --- /dev/null +++ b/tests/test_client/ids_and_schema.cpp @@ -0,0 +1,86 @@ +#include "common.hpp" + +// ── ConversationId ────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("ConversationId: round-trips through its string form", "[client][convo_id]") { + constexpr auto sid = "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + constexpr auto gid = "03fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + + auto dm = ConversationId::dm(sid); + CHECK(dm.type() == ConversationId::Type::dm); + CHECK(dm.to_string() == oxenc::to_hex(sid)); + CHECK(ConversationId::parse(dm.to_string()) == dm); + CHECK(std::ranges::equal(dm.session_id(), sid)); + + auto group = ConversationId::group(gid); + CHECK(group.type() == ConversationId::Type::group); + CHECK(ConversationId::parse(group.to_string()) == group); + CHECK(std::ranges::equal(group.group_id(), gid)); + + // Same 32-byte body, different prefix: distinct conversations. + CHECK(dm != group); + + auto com = ConversationId::community("http://example.com", "room"); + CHECK(com.type() == ConversationId::Type::community); + CHECK(com.to_string() == "community:http://example.com/room"); + CHECK(ConversationId::parse(com.to_string()) == com); + auto [url, room] = com.community(); + CHECK(url == "http://example.com"); + CHECK(room == "room"); +} + +TEST_CASE("ConversationId: normalises community URLs and rooms", "[client][convo_id]") { + auto a = ConversationId::community("http://Example.COM/", "Room"); + auto b = ConversationId::community("http://example.com", "room"); + CHECK(a == b); + + CHECK_THROWS_AS(ConversationId::community("", "room"), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::community("http://x.com", ""), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::community("http://x.com", "a/b"), std::invalid_argument); +} + +TEST_CASE("ConversationId: rejects bad input and mistyped access", "[client][convo_id]") { + constexpr auto sid = "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + constexpr auto bad_prefix = + "07fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + + CHECK_THROWS_AS(ConversationId::dm(bad_prefix), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::group(bad_prefix), std::invalid_argument); + + CHECK_THROWS_AS(ConversationId::parse(""), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::parse("nonsense"), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::parse(oxenc::to_hex(bad_prefix)), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::parse("05zz"), std::invalid_argument); + CHECK_THROWS_AS(ConversationId::parse("community:noroom"), std::invalid_argument); + + // Extracting the wrong kind is a programming error, not a parse error. + auto dm = ConversationId::dm(sid); + CHECK_THROWS_AS(dm.group_id(), std::logic_error); + CHECK_THROWS_AS(dm.community(), std::logic_error); +} + +// ── Schema ────────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: applies its migrations under the client owner", "[client][schema]") { + TempClient c; + + // Both schemas are created from their full_schema.sql, each marking its own owner. + CHECK(TestHelper::migration_applied(c->core, "client:@created")); + CHECK(TestHelper::migration_applied(c->core, "@created")); + + // Not Connection::table_exists(): its query in session-sqlite is missing a closing paren and + // throws "incomplete input" for every caller. + auto has_table = [&](std::string_view name) { + return c->core.database() + .conn() + .prepared_maybe_get( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", name) + .has_value(); + }; + CHECK(has_table("accounts")); + CHECK(has_table("conversations")); + CHECK(has_table("messages")); + CHECK(has_table("message_raw_content")); + // Client's tables live in Core's database, not a second file. + CHECK(has_table("globals")); +} diff --git a/tests/test_client/interop_and_threading.cpp b/tests/test_client/interop_and_threading.cpp new file mode 100644 index 000000000..f359f4c1c --- /dev/null +++ b/tests/test_client/interop_and_threading.cpp @@ -0,0 +1,135 @@ +#include "common.hpp" + +// ── Core interoperability ─────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: an asynchronous call reports that it succeeded", "[client][callbacks]") { + TempClient c; + SenderKeys sender; + approve(*c, sender.session_id); + deliver(*c, sender, "hello", from_epoch_ms(1000), "h1"); + sync(*c); + + // Qualified, because Client masks the asynchronous forms deliberately: choosing the easy + // class means choosing it for everything. + std::optional reported_error = "not called"; + std::vector got; + c->Client::conversations( + [&](std::optional error, std::vector cs) { + reported_error = std::move(error); + got = std::move(cs); + }); + sync(*c); + + // Called exactly once, and saying it worked rather than leaving the caller to assume so. + CHECK_FALSE(reported_error.has_value()); + REQUIRE(got.size() == 1); + CHECK(got[0].id() == ConversationId::dm(sender.session_id)); +} + +TEST_CASE("Client: handlers arrive through the dispatcher", "[client][callbacks]") { + // Stands in for an application's loop: jobs are collected rather than run, so a handler that + // ran on Core's loop instead of being handed over is visible as one that never happened. + std::vector> queued; + std::thread::id dispatched_on; + + Recorder r; + TempClient c{r.handlers()}; + c->set_dispatcher([&](std::function job) { + dispatched_on = std::this_thread::get_id(); + queued.push_back(std::move(job)); + }); + + SenderKeys sender; + deliver(*c, sender, "hello", from_epoch_ms(1000), "h1"); + sync(*c); + + // Handed over rather than called: nothing has reached the application yet. + CHECK(r.msg_added.empty()); + CHECK(r.added.empty()); + REQUIRE(!queued.empty()); + + // Handed over from Core's loop, which is the thread an application must not be touched from. + CHECK(dispatched_on != std::this_thread::get_id()); + + for (auto& job : queued) + job(); + queued.clear(); + + REQUIRE(r.msg_added.size() == 1); + CHECK(r.msg_added[0].second.body == "hello"); + CHECK(r.added.size() == 1); + + // Unsetting puts things back the way they are without one, which is what an application does + // when its loop stops accepting work. + c->set_dispatcher(nullptr); + deliver(*c, sender, "direct", from_epoch_ms(2000), "h2"); + sync(*c); + + CHECK(queued.empty()); + REQUIRE(r.msg_added.size() == 2); + CHECK(r.msg_added[1].second.body == "direct"); +} + +TEST_CASE("Client: Core is usable directly through the Client", "[client][callbacks]") { + TempClient c; + + // The account state is Core's, and reachable without Client wrapping any of it. + CHECK(c->core.globals.session_id()[0] == std::byte{0x05}); + CHECK_FALSE(c->core.devices.device_id().empty()); + + // Globals set through Core survive a Client restart, i.e. it really is one database. + c->core.globals.set("client_test_key", "value"sv); + c.reopen(); + CHECK(c->core.globals.get_text("client_test_key") == "value"); +} + +// ── Threading ─────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: reads are safe while messages arrive on another thread", "[client][threads]") { + // An application reads conversations and history from its UI thread while Core's poll thread + // writes arriving messages. Those are two connections from the shared pool against a WAL + // database, so readers should not block behind the writer nor see SQLITE_BUSY -- but that is a + // claim about configuration, and nothing exercised it until this. + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + constexpr int N = 300; + std::atomic writing{true}; + std::atomic reads{0}; + std::exception_ptr writer_err, reader_err; + + std::thread writer{[&] { + try { + for (int i = 1; i <= N; i++) + deliver(*c, sender, "msg{}"_format(i), from_epoch_ms(i * 1000), "h{}"_format(i)); + } catch (...) { + writer_err = std::current_exception(); + } + writing = false; + }}; + + std::thread reader{[&] { + try { + while (writing) { + for (const auto& convo_row : c->conversations(await)) + convo_row.messages(50, await); + reads++; + } + } catch (...) { + reader_err = std::current_exception(); + } + }}; + + writer.join(); + reader.join(); + + // Rethrown on this thread: Catch2's assertion macros are not safe to use from the others. + if (writer_err) + std::rethrow_exception(writer_err); + if (reader_err) + std::rethrow_exception(reader_err); + + CHECK(reads > 0); // the reader really did run alongside, rather than after + CHECK(c->conversation(convo, await)->messages(N + 10, await).size() == N); +} diff --git a/tests/test_client/profile_pictures.cpp b/tests/test_client/profile_pictures.cpp new file mode 100644 index 000000000..3c212df60 --- /dev/null +++ b/tests/test_client/profile_pictures.cpp @@ -0,0 +1,303 @@ +#include + +#include +#include +#include + +#include "../../src/client/download_cache.hpp" +#include "config_helpers.hpp" + +namespace cache = session::client::cache; + +namespace { + +// Encrypts as a client did before the stream scheme: AES-256-GCM, nonce prepended, tag appended. +// Written here rather than in libsession because nothing of ours produces this format any more -- +// it exists only to be read -- and a test that encrypted with our own code would be checking that +// we agree with ourselves. +std::vector gcm_encrypt( + std::span plain, std::span key) { + std::vector out(12 + plain.size() + 16); + session::random::fill(std::span{out.data(), 12}); + + struct gcm_aes256_ctx ctx; + gcm_aes256_set_key(&ctx, session::to_unsigned(key.data())); + gcm_aes256_set_iv(&ctx, 12, session::to_unsigned(out.data())); + gcm_aes256_encrypt( + &ctx, + plain.size(), + session::to_unsigned(out.data() + 12), + session::to_unsigned(plain.data())); + gcm_aes256_digest(&ctx, 16, session::to_unsigned(out.data() + 12 + plain.size())); + return out; +} + +// Gives `them` a picture at `url` with `key`, as another device would have. +void set_picture( + TempClient& c, + const std::string& them, + const std::string& url, + std::span key) { + auto pushed = contacts_from_another_device(*c.client, them, [&](auto& e) { + e.set_name("Padmé"); + e.profile_picture = config::profile_pic{url, {key.begin(), key.end()}}; + }); + merge_contacts(*c.client, pushed); +} + +} // namespace + +TEST_CASE( + "Client: a stream-encrypted picture round-trips through the cache", "[client][pictures]") { + TempCacheDir dir; + TempClient c; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + std::vector image(9000); + session::random::fill(image); + + // As a current client uploads one: the stream scheme, with the url saying so. + auto seed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; + auto [encrypted, key] = attachment::encrypt(seed, image, attachment::Domain::PROFILE_PIC); + net->served["pic1"] = encrypted; + auto url = network::file_server::generate_download_url("pic1", {}, true); + + auto them = "05" + std::string(64, 'a'); + set_picture(c, them, url, key); + auto id = dm_from_hex(them); + + std::vector> progress; + std::optional> got; + std::optional err; + c->profile_picture( + id, + [&](int64_t, int64_t, std::optional r) { progress.push_back(r); }, + [&](std::optional e, auto pic) { + err = std::move(e); + got = std::move(pic); + }); + + // The fetch is posted to the loop, so let it get as far as asking before answering it. + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + + REQUIRE_FALSE(err.has_value()); + REQUIRE(got); + CHECK(*got == image); + + // Watched from start to finish: the 0/0 that says it began, and the terminal 0 that says it + // arrived. + REQUIRE(progress.size() >= 2); + CHECK_FALSE(progress.front().has_value()); + CHECK(progress.back() == 0); + + // It landed in the cache under the url, not under the url plus its fragment. + auto file = cache::path_for(dir.path, cache::PROFILE_DIR, url); + CHECK(std::filesystem::exists(file)); + + // ...and the second ask is served from there: no download, and no progress reported, since + // there is nothing to watch. + progress.clear(); + got.reset(); + c->profile_picture( + id, + [&](int64_t, int64_t, std::optional r) { progress.push_back(r); }, + [&](std::optional e, auto pic) { + err = std::move(e); + got = std::move(pic); + }); + sync(*c); + + CHECK(net->downloads.empty()); + REQUIRE(got); + CHECK(*got == image); + CHECK(progress.empty()); +} + +TEST_CASE("Client: a picture from before the stream scheme still opens", "[client][pictures]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + std::vector image(2000); + session::random::fill(image); + + b32 key; + session::random::fill(key); + net->served["pic2"] = gcm_encrypt(image, key); + + // No `d` fragment, which is what says "not the stream scheme" -- and for a display picture that + // means GCM rather than the legacy *attachment* scheme, which is the distinction this exists to + // check. Nothing in the bytes says which. + auto url = network::file_server::generate_download_url("pic2", {}, false); + + auto them = "05" + std::string(64, 'd'); + set_picture(c, them, url, key); + + std::optional> got; + std::optional err; + c->profile_picture(dm_from_hex(them), [&](std::optional e, auto pic) { + err = std::move(e); + got = std::move(pic); + }); + + // The fetch is posted to the loop, so let it get as far as asking before answering it. + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + + REQUIRE_FALSE(err.has_value()); + REQUIRE(got); + CHECK(*got == image); +} + +TEST_CASE( + "Client: a picture that will not decrypt is an error, not an absence", + "[client][pictures]") { + TempCacheDir dir; + TempClient c; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + std::vector image(500); + session::random::fill(image); + + b32 key, wrong; + session::random::fill(key); + session::random::fill(wrong); + net->served["pic3"] = gcm_encrypt(image, key); + auto url = network::file_server::generate_download_url("pic3", {}, false); + + auto them = "05" + std::string(64, 'e'); + set_picture(c, them, url, wrong); + + std::optional> got; + std::optional err; + bool called = false; + c->profile_picture(dm_from_hex(them), [&](std::optional e, auto pic) { + err = std::move(e); + got = std::move(pic); + called = true; + }); + + // The fetch is posted to the loop, so let it get as far as asking before answering it. + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + + REQUIRE(called); + // The difference that matters to a viewer: an empty pane because there is no picture, versus a + // broken one because we could not read it. + CHECK(err.has_value()); + CHECK_FALSE(got.has_value()); + + // And nothing was cached, so asking again tries again rather than serving the failure forever. + CHECK_FALSE(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, url))); +} + +TEST_CASE("Client: a replaced profile picture stops taking up room", "[client][pictures]") { + TempCacheDir dir; + TempClient c; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + auto seed = session::random::random(32); + auto them = "05" + std::string(64, 'b'); + auto id = dm_from_hex(them); + + // Descending from our own history rather than pushed fresh each time: two rivals at the same + // seqno merge to the union, and a test about a picture being *replaced* needs the second to + // supersede the first. + auto publish = [&](std::string_view file_id, std::chrono::sys_seconds when) { + std::vector image(1000); + session::random::fill(image); + auto [ct, key] = attachment::encrypt(seed, image, attachment::Domain::PROFILE_PIC); + net->served[std::string{file_id}] = ct; + auto url = network::file_server::generate_download_url(file_id, {}, true); + + auto pushed = contacts_update_from_another_device(*c.client, [&](config::Contacts& theirs) { + auto e = theirs.get_or_construct(them); + e.set_name("Padmé"); + e.profile_updated = when; + e.profile_picture = config::profile_pic{url, {key.begin(), key.end()}}; + theirs.set(e); + }); + merge_contacts(*c.client, pushed); + return url; + }; + + auto first = publish("old_pic", std::chrono::sys_seconds{1000s}); + + // Fetched, so there is something on disk to reclaim. + c->profile_picture(id, [](auto, auto) {}); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + REQUIRE(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, first))); + + // They change it. Nothing about the old file is referenced any more, and nothing else in the + // client would ever look at it again. + auto second = publish("new_pic", std::chrono::sys_seconds{2000s}); + sync(*c); + CHECK_FALSE(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, first))); + + // ...and the new one still fetches, so what went was the stale file and not the directory. + std::optional> got; + c->profile_picture(id, [&](std::optional, auto pic) { got = std::move(pic); }); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + REQUIRE(got); + CHECK(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, second))); +} + +TEST_CASE("Client: learning a picture's url fetches it unasked", "[client][pictures]") { + TempCacheDir dir; + + std::vector>> reported; + callbacks cbs; + cbs.display_picture_progress = + [&](const ConversationId& id, int64_t, int64_t, std::optional r) { + reported.emplace_back(id, r); + }; + + TempClient c{std::move(cbs)}; + auto* net = attach_mock_network(c->core); + c->set_cache_dir(dir.path); + + std::vector image(1500); + session::random::fill(image); + auto seed = session::random::random(32); + auto [ct, key] = attachment::encrypt(seed, image, attachment::Domain::PROFILE_PIC); + net->served["auto_pic"] = ct; + auto url = network::file_server::generate_download_url("auto_pic", {}, true); + + auto them = "05" + std::string(64, 'c'); + auto id = dm_from_hex(them); + + // Nobody has asked for anything: the url arriving in a config is the whole of the trigger. + set_picture(c, them, url, key); + sync(*c); + REQUIRE(serve_downloads(*net) == 1); + sync(*c); + + CHECK(std::filesystem::exists(cache::path_for(dir.path, cache::PROFILE_DIR, url))); + + // Watched from the outside, which is what a list of conversations needs to draw a placeholder: + // the 0/0 that says it began, and a terminal result. + REQUIRE(reported.size() >= 2); + CHECK(reported.front().first == id); + CHECK_FALSE(reported.front().second.has_value()); + CHECK(reported.back().second == 0); + + // And the display, arriving afterwards, is served from the cache rather than fetching the same + // picture a second time. + std::optional> got; + c->profile_picture(id, [&](std::optional, auto pic) { got = std::move(pic); }); + sync(*c); + + CHECK(net->downloads.empty()); + REQUIRE(got); + CHECK(*got == image); +} diff --git a/tests/test_client/receiving.cpp b/tests/test_client/receiving.cpp new file mode 100644 index 000000000..e248d5357 --- /dev/null +++ b/tests/test_client/receiving.cpp @@ -0,0 +1,851 @@ +#include "common.hpp" + +// ── Receiving ─────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: a received DM creates a conversation and a message", "[client][receive]") { + TempClient c; + SenderKeys sender; + approve(*c, sender.session_id); + + deliver(*c, sender, "hello there", from_epoch_ms(5000), "hash1", "Obi-Wan"); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + CHECK(convos[0].id() == ConversationId::dm(sender.session_id)); + CHECK(convos[0].display_name() == "Obi-Wan"); + CHECK(preview_body(convos[0]) == "hello there"); + CHECK(convos[0].last_activity() == from_epoch_ms(5000)); + CHECK(convos[0].unread() == 1); + + auto msgs = c->conversation(convos[0].id(), await)->messages(await); + REQUIRE(msgs.size() == 1); + CHECK(msgs[0].body == "hello there"); + CHECK_FALSE(msgs[0].outgoing); + CHECK(msgs[0].sender == sender.session_id); + CHECK(msgs[0].timestamp == from_epoch_ms(5000)); + CHECK(msgs[0].hash == "hash1"); + CHECK_FALSE(msgs[0].send_state.has_value()); + + CHECK(c->message(msgs[0].id, await)->body == "hello there"); + CHECK_FALSE(c->message(msgs[0].id + 1000, await).has_value()); +} + +TEST_CASE( + "Client: our own sent message lands in the recipient's conversation", "[client][receive]") { + TempClient c; + SenderKeys peer; + + // A one-to-one message is stored on both swarms, so this is what our own send looks like coming + // back to us: sender is us, and the conversation it belongs to is only in syncTarget. + deliver(*c, + self_keys(*c), + "sent from my phone", + from_epoch_ms(5000), + "sync1", + "", + peer.session_id); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + CHECK(convos[0].id() == ConversationId::dm(peer.session_id)); + CHECK(convos[0].unread() == 0); + + auto msgs = c->conversation(convos[0].id(), await)->messages(await); + REQUIRE(msgs.size() == 1); + CHECK(msgs[0].body == "sent from my phone"); + CHECK(msgs[0].outgoing); + CHECK(msgs[0].sender == own_sid(*c)); + CHECK(msgs[0].send_state == SendState::sent); +} + +TEST_CASE("Client: a message to ourselves is a conversation with ourselves", "[client][receive]") { + TempClient c; + auto me = own_sid(*c); + + // Note to Self is not a distinct kind of conversation, in Session or here: it is the DM whose + // peer is our own account, which is what both spellings below resolve to. + deliver(*c, self_keys(*c), "targeted", from_epoch_ms(5000), "self1", "", me); + deliver(*c, self_keys(*c), "untargeted", from_epoch_ms(6000), "self2"); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + CHECK(convos[0].id() == ConversationId::dm(me)); + CHECK(convos[0].unread() == 0); + CHECK(convos[0].dm()->note_to_self); + CHECK(c->is_note_to_self(convos[0].id())); + + auto msgs = c->conversation(convos[0].id(), await)->messages(await); + REQUIRE(msgs.size() == 2); + CHECK(msgs[0].outgoing); + CHECK(msgs[1].outgoing); +} + +TEST_CASE("Client: reads answer emptily before an account exists", "[client][convos]") { + // An application opening the database under defer_account renders before onboarding has run, so + // every read has to survive having no identity: "no account" and "no conversations" are the + // same answer. Only writes may insist on one. + TempClient c{core::defer_account{}}; + REQUIRE_FALSE(c->core.globals.have_account()); + + constexpr auto sid = "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + auto convo = ConversationId::dm(sid); + + CHECK(c->conversations(await).empty()); + CHECK(c->message_requests(await).empty()); + CHECK_FALSE(c->conversation(convo, await).has_value()); + CHECK_FALSE(c->dm(convo, await).has_value()); + CHECK_FALSE(c->message(1, await).has_value()); + CHECK_FALSE(c->is_note_to_self(convo)); + + // Nothing here asks what a *nonexistent* conversation's messages are, or what marking one read + // does: with the operations on the conversation, there is nothing to call them on, so the + // nullopt above is the whole answer. +} + +TEST_CASE("Client: note to self is reported, not left to the caller", "[client][convos]") { + TempClient c; + SenderKeys peer; + + // open_dm hands back a DM rather than an AnyConversation, so the flag is a plain field: asking + // for the kind is what removes the narrowing. + auto self = c->open_dm(ConversationId::dm(own_sid(*c)), await); + CHECK(self.note_to_self); + CHECK(c->conversation(self.id, await)->dm()->note_to_self); + CHECK(c->is_note_to_self(self.id)); + + auto other = c->open_dm(ConversationId::dm(peer.session_id), await); + CHECK_FALSE(other.note_to_self); + CHECK_FALSE(c->conversation(other.id, await)->dm()->note_to_self); + CHECK_FALSE(c->is_note_to_self(other.id)); + + // A group or community is never note-to-self, whatever its id happens to be. + constexpr auto gid = "03fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + CHECK_FALSE(c->is_note_to_self(ConversationId::group(gid))); + CHECK_FALSE(c->is_note_to_self(ConversationId::community("http://example.com", "room"))); + + // The list form agrees with the single-conversation form. + for (const auto& convo : c->conversations(await)) + CHECK(convo.dm()->note_to_self == c->is_note_to_self(convo.id())); +} + +TEST_CASE("Client: syncTarget from another sender is ignored", "[client][receive]") { + TempClient c; + SenderKeys peer, elsewhere; + approve(*c, peer.session_id); + + deliver(*c, peer, "not yours to file", from_epoch_ms(5000), "h1", "", elsewhere.session_id); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + CHECK(convos[0].id() == ConversationId::dm(peer.session_id)); + + auto msgs = c->conversation(convos[0].id(), await)->messages(await); + REQUIRE(msgs.size() == 1); + CHECK_FALSE(msgs[0].outgoing); +} + +TEST_CASE("Client: redelivery of the same swarm hash is ignored", "[client][receive]") { + TempClient c; + SenderKeys sender; + + deliver(*c, sender, "only once", from_epoch_ms(5000), "dup"); + deliver(*c, sender, "only once", from_epoch_ms(5000), "dup"); + + auto convo = ConversationId::dm(sender.session_id); + CHECK(c->conversation(convo, await)->messages(await).size() == 1); + CHECK(c->conversation(convo, await)->unread() == 1); + + // A genuinely different message from the same sender still lands. + deliver(*c, sender, "and again", from_epoch_ms(6000), "notdup"); + CHECK(c->conversation(convo, await)->messages(await).size() == 2); +} + +TEST_CASE("Client: the same message under a different swarm hash is deduped", "[client][receive]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + // One message stored twice -- a sender who retried a store that had actually succeeded, say -- + // lands under two swarm hashes. The swarm hash cannot recognise that; the msgid can, being the + // one identifier every copy of a message carries. + deliver(*c, + sender, + "said once", + from_epoch_ms(5000), + "first_hash", + "", + std::nullopt, + nullptr, + 7); + deliver(*c, + sender, + "said once", + from_epoch_ms(5000), + "second_hash", + "", + std::nullopt, + nullptr, + 7); + + CHECK(c->conversation(convo, await)->messages(await).size() == 1); + CHECK(c->conversation(convo, await)->unread() == 1); + + // Same millisecond, different message: the case the timestamp alone cannot tell apart, and the + // whole reason the id exists. Identical body, so nothing but the id distinguishes them. + deliver(*c, + sender, + "said once", + from_epoch_ms(5000), + "third_hash", + "", + std::nullopt, + nullptr, + 8); + CHECK(c->conversation(convo, await)->messages(await).size() == 2); + + // A sender too old to set one has no identity beyond its timestamp, so two arrivals under + // different swarm hashes cannot be told from one message stored twice. Both land: a visible + // duplicate is the failure we chose over silently dropping a real message. + deliver(*c, sender, "from an old client", from_epoch_ms(6000), "old_a"); + deliver(*c, sender, "from an old client", from_epoch_ms(6000), "old_b"); + CHECK(c->conversation(convo, await)->messages(await).size() == 4); +} + +TEST_CASE("Client: display name is unset rather than empty until known", "[client][convos]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "hi", from_epoch_ms(1000), "h1"); + + // No profile has been seen, so the column holds NULL rather than an empty string. + auto stored = c->core.database().conn().prepared_get>( + "SELECT name FROM accounts WHERE session_id = ?", sender.session_id); + CHECK_FALSE(stored.has_value()); + CHECK(c->conversation(convo, await)->display_name().empty()); +} + +TEST_CASE("Client: a later profile name updates the conversation", "[client][receive]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "one", from_epoch_ms(1000), "h1"); + CHECK(c->conversation(convo, await)->display_name().empty()); + // With no name known, name_or_id() falls back to the id rather than an empty string. + CHECK(c->conversation(convo, await)->name_or_id() == convo.to_string()); + + deliver(*c, sender, "two", from_epoch_ms(2000), "h2", "Padmé"); + CHECK(c->conversation(convo, await)->display_name() == "Padmé"); + CHECK(c->conversation(convo, await)->name_or_id() == "Padmé"); + + // A message with no profile does not erase the name we already have. + deliver(*c, sender, "three", from_epoch_ms(3000), "h3"); + CHECK(c->conversation(convo, await)->display_name() == "Padmé"); +} + +TEST_CASE("Client: non-conversation content does not create a conversation", "[client][receive]") { + TempClient c; + SenderKeys sender; + + auto deliver_content = [&](const SessionProtos::Content& content, std::string hash) { + auto plaintext = content.SerializeAsString(); + auto encoded = encode_dm_v1( + std::as_bytes(std::span{plaintext}), + sender.ed_sk, + from_epoch_ms(1000), + own_sid(*c), + std::nullopt); + core::SwarmMessage sm{encoded, std::move(hash), from_epoch_ms(1000), from_epoch_ms(99999)}; + c->core.receive_messages({&sm, 1}, config::Namespace::Default, true); + }; + + // A typing indicator: valid Content, but nothing that belongs in message history. + SessionProtos::Content typing; + typing.set_sigtimestamp(1000); + typing.mutable_typingmessage()->set_timestamp(1000); + typing.mutable_typingmessage()->set_action(SessionProtos::TypingMessage::STARTED); + deliver_content(typing, "typing"); + + // A DataMessage carrying only a profile update, with no body. + SessionProtos::Content bodyless; + bodyless.set_sigtimestamp(1000); + bodyless.mutable_datamessage()->mutable_profile()->set_displayname("Ghost"); + deliver_content(bodyless, "bodyless"); + + CHECK(c->conversations(await).empty()); +} + +// ── Ordering, unread, drafts ──────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: conversations are ordered by most recent activity", "[client][convos]") { + TempClient c; + SenderKeys alice, bob; + approve(*c, alice.session_id); + approve(*c, bob.session_id); + + deliver(*c, alice, "first", from_epoch_ms(1000), "a1"); + deliver(*c, bob, "second", from_epoch_ms(2000), "b1"); + + auto convos = c->conversations(await); + REQUIRE(convos.size() == 2); + CHECK(convos[0].id() == ConversationId::dm(bob.session_id)); + CHECK(convos[1].id() == ConversationId::dm(alice.session_id)); + + // Alice speaking again moves her back to the top. + deliver(*c, alice, "third", from_epoch_ms(3000), "a2"); + convos = c->conversations(await); + CHECK(convos[0].id() == ConversationId::dm(alice.session_id)); + CHECK(preview_body(convos[0]) == "third"); +} + +TEST_CASE("Client: unread counting and the read watermark", "[client][unread]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "one", from_epoch_ms(1000), "h1"); + deliver(*c, sender, "two", from_epoch_ms(2000), "h2"); + deliver(*c, sender, "three", from_epoch_ms(3000), "h3"); + CHECK(c->conversation(convo, await)->unread() == 3); + + c->conversation(convo, await)->mark_read(from_epoch_ms(2000), await); + CHECK(c->conversation(convo, await)->unread() == 1); + + // The watermark never moves backwards. + c->conversation(convo, await)->mark_read(from_epoch_ms(1000), await); + CHECK(c->conversation(convo, await)->unread() == 1); + + c->conversation(convo, await)->mark_read(await); + CHECK(c->conversation(convo, await)->unread() == 0); + + // A new arrival after a full read is unread again: "read everything" must not mean "read + // everything that will ever arrive". + deliver(*c, sender, "four", from_epoch_ms(4000), "h4"); + CHECK(c->conversation(convo, await)->unread() == 1); + + // Even one that arrives late, bearing a timestamp older than what we already read to. + c->conversation(convo, await)->mark_read(await); + deliver(*c, sender, "late", from_epoch_ms(3500), "h5"); + CHECK(c->conversation(convo, await)->unread() == + 0); // known limitation of a timestamp watermark + + // Marking read on a conversation with nothing to read is a no-op, not an error. + auto empty = ConversationId::dm( + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b); + CHECK_NOTHROW(c->open_dm(empty, await).mark_read(await)); + CHECK(c->conversation(empty, await)->unread() == 0); +} + +TEST_CASE("Client: cached counts stay in step with the messages table", "[client][convos]") { + TempClient c; + SenderKeys alice, bob; + auto convo = ConversationId::dm(alice.session_id); + auto other = ConversationId::dm(bob.session_id); + + auto conn = c->core.database().conn(); + + // The cached counters alongside what counting the rows actually yields. Nothing in the public + // API can tell a counter from a subquery, which is exactly why drift needs asserting directly. + auto counts = [&](std::span sid) { + return conn.prepared_get( + R"( + SELECT c.count, c.unread_count, + (SELECT COUNT(*) FROM messages WHERE conversation = c.id), + (SELECT COUNT(*) FROM messages + WHERE conversation = c.id AND outgoing = 0 AND timestamp > c.last_read) + FROM conversations c + JOIN accounts a ON a.id = c.dm + WHERE a.session_id = ? + )", + sid); + }; + + deliver(*c, alice, "one", from_epoch_ms(1000), "h1"); + deliver(*c, alice, "two", from_epoch_ms(2000), "h2"); + deliver(*c, alice, "three", from_epoch_ms(3000), "h3"); + deliver(*c, bob, "elsewhere", from_epoch_ms(1500), "h4"); + + SECTION("arrivals update both") { + auto [n, unread, actual_n, actual_unread] = counts(alice.session_id); + CHECK(n == actual_n); + CHECK(unread == actual_unread); + CHECK(n == 3); + CHECK(unread == 3); + CHECK(c->conversation(convo, await)->unread() == 3); + } + + SECTION("marking read moves unread without touching the total") { + c->conversation(convo, await)->mark_read(from_epoch_ms(2000), await); + auto [n, unread, actual_n, actual_unread] = counts(alice.session_id); + CHECK(n == actual_n); + CHECK(unread == actual_unread); + CHECK(n == 3); + CHECK(unread == 1); + } + + SECTION("count tracks deletes made behind the application's back") { + auto id = c->conversation(convo, await)->messages(await).front().id; + conn.prepared_exec("DELETE FROM messages WHERE id = ?", id); + + auto [n, unread, actual_n, actual_unread] = counts(alice.session_id); + CHECK(n == actual_n); + CHECK(n == 2); + + // unread_count is the application's to maintain, so a raw delete leaves it behind -- that + // is the deliberate split, not a bug. Whatever next recomputes it puts it right. + CHECK(unread == 3); + CHECK(actual_unread == 2); + + c->conversation(convo, await)->mark_read(from_epoch_ms(1000), await); + auto [n2, unread2, actual_n2, actual_unread2] = counts(alice.session_id); + CHECK(unread2 == actual_unread2); + } + + SECTION("count follows a message moved between conversations") { + auto convo_row = conn.prepared_get( + "SELECT c.id FROM conversations c JOIN accounts a ON a.id = c.dm" + " WHERE a.session_id = ?", + bob.session_id); + auto id = c->conversation(convo, await)->messages(await).front().id; + conn.prepared_exec("UPDATE messages SET conversation = ? WHERE id = ?", convo_row, id); + + auto [n, unread, actual_n, actual_unread] = counts(alice.session_id); + CHECK(n == actual_n); + CHECK(n == 2); + + auto [n2, unread2, actual_n2, actual_unread2] = counts(bob.session_id); + CHECK(n2 == actual_n2); + CHECK(n2 == 2); + } +} + +TEST_CASE("Client: explicit conversation creation", "[client][convos]") { + TempClient c; + constexpr auto sid = "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + auto convo = ConversationId::dm(sid); + + CHECK_FALSE(c->conversation(convo, await).has_value()); + + auto created = c->open_dm(convo, await); + CHECK(created.id == convo); + CHECK(created.unread == 0); + // Nothing to preview at all, which is a different answer from an empty body. + CHECK(!created.last_preview); + CHECK(c->conversations(await).size() == 1); + + // Opening one that already exists is not an error and does not duplicate it -- which is why + // this is `open` and not `create`. + c->open_dm(convo, await); + CHECK(c->conversations(await).size() == 1); +} + +// ── Paging ────────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: message history pages backwards by cursor", "[client][messages]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + for (int i = 1; i <= 10; i++) + deliver(*c, sender, "msg{}"_format(i), from_epoch_ms(i * 1000), "h{}"_format(i)); + + auto page1 = c->conversation(convo, await)->messages(4, await); + REQUIRE(page1.size() == 4); + CHECK(page1[0].body == "msg10"); + CHECK(page1[3].body == "msg7"); + + auto page2 = c->conversation(convo, await)->messages(4, page1.back().cursor(), await); + REQUIRE(page2.size() == 4); + CHECK(page2[0].body == "msg6"); + CHECK(page2[3].body == "msg3"); + + auto page3 = c->conversation(convo, await)->messages(4, page2.back().cursor(), await); + REQUIRE(page3.size() == 2); + CHECK(page3[0].body == "msg2"); + CHECK(page3[1].body == "msg1"); + + CHECK(c->conversation(convo, await)->messages(4, page3.back().cursor(), await).empty()); +} + +TEST_CASE("Client: paging is stable across equal timestamps", "[client][messages]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + // Three messages sharing one timestamp: only the id tiebreak keeps paging from repeating or + // skipping rows. + for (int i = 1; i <= 3; i++) + deliver(*c, sender, "same{}"_format(i), from_epoch_ms(1000), "same_h{}"_format(i)); + + std::vector seen; + std::optional cursor; + while (true) { + auto page = c->conversation(convo, await)->messages(1, cursor, await); + if (page.empty()) + break; + seen.push_back(page[0].body); + cursor = page[0].cursor(); + } + + CHECK(seen == std::vector{"same3", "same2", "same1"}); +} + +TEST_CASE("Client: a message can be shown as it was on the wire", "[client][messages][debug]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver( + *c, + sender, + "hello there", + from_epoch_ms(5000), + "h1", + "Obi-Wan", + std::nullopt, + [](SessionProtos::DataMessage& d) { + auto* a = d.add_attachments(); + a->set_id(12345); + a->set_contenttype("image/png"); + a->set_key("\x01\x02\x03\x04"); + }, + 77); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 1); + + auto dump = c->message_debug(msgs[0].id, await); + REQUIRE(dump); + INFO("dump:\n" << *dump); + + // Named *and* numbered, and nested beneath the field that holds them. The number is what the + // wire carries, so it is what a dump is compared against. + // + // A message field opens a brace and closes it at its own indent, rather than being followed by + // an empty-looking `{}` with its contents underneath. + CHECK(dump->find("dataMessage [1] {\n") != std::string::npos); + CHECK(dump->find("\n}\n") != std::string::npos); + CHECK(dump->find(" body [1]: \"hello there\"\n") != std::string::npos); + CHECK(dump->find(" displayName [1]: \"Obi-Wan\"\n") != std::string::npos); + CHECK(dump->find(" contentType [2]: \"image/png\"\n") != std::string::npos); + // Bytes summarised as length plus hex rather than dumped raw. + CHECK(dump->find(" key [3]: (4 bytes) 01020304\n") != std::string::npos); + // Scalars at the top level of Content. + CHECK(dump->find("sigTimestamp [15]: 5000\n") != std::string::npos); + CHECK(dump->find("msgId [18]: 77\n") != std::string::npos); + + // The name as declared in the .proto, not the lowercased spelling protoc gives the accessor. + CHECK(dump->find("sourcedevice") == std::string::npos); + + // A message with no stored wire form, and one that does not exist, are both "nothing to show" + // rather than errors. + CHECK_FALSE(c->message_debug(msgs[0].id + 1000, await).has_value()); +} + +TEST_CASE( + "Client: deleting a message empties it but keeps its place", "[client][messages][delete]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "one", from_epoch_ms(1000), "h1"); + deliver(*c, sender, "two", from_epoch_ms(2000), "h2"); + deliver(*c, sender, "three", from_epoch_ms(3000), "h3"); + REQUIRE(c->conversation(convo, await)->unread() == 3); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 3); + auto middle = msgs[1]; // "two" + REQUIRE(middle.body == "two"); + REQUIRE(middle.hash == "h2"); + + CHECK(c->delete_message(middle.id, await)); + + // Hidden by default... + auto visible = c->conversation(convo, await)->messages(await); + REQUIRE(visible.size() == 2); + CHECK(visible[0].body == "three"); + CHECK(visible[1].body == "one"); + + // ...and in its original place when asked for, saying who and when but nothing else. + auto all = c->conversation(convo, await)->messages(50, std::nullopt, true, await); + REQUIRE(all.size() == 3); + CHECK(all[1].id == middle.id); + CHECK(all[1].deleted == Deletion::here); + CHECK(all[1].body.empty()); + CHECK(all[1].timestamp == from_epoch_ms(2000)); + CHECK(all[1].sender == sender.session_id); + // The hash stays: it is what stops a redelivery bringing the message back. + CHECK(all[1].hash == "h2"); + + // Nothing left to read, so it stops being unread. + CHECK(c->conversation(convo, await)->unread() == 2); + + // The stored wire form goes with it, which is where the body actually survived. + CHECK_FALSE(c->message_debug(middle.id, await).has_value()); + + // A redelivery under the same hash is still recognised as one we have seen. + deliver(*c, sender, "two", from_epoch_ms(2000), "h2"); + CHECK(c->conversation(convo, await)->messages(50, std::nullopt, true, await).size() == 3); + + // Deleting a message that does not exist is false, not an error. + CHECK_FALSE(c->delete_message(middle.id + 10000, await)); +} + +TEST_CASE("Client: the list preview skips a deleted last message", "[client][convos][delete]") { + TempClient c; + SenderKeys sender; + approve(*c, sender.session_id); + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "older", from_epoch_ms(1000), "h1"); + deliver(*c, sender, "newest", from_epoch_ms(2000), "h2"); + + auto newest = c->conversation(convo, await)->messages(await).front(); + REQUIRE(newest.body == "newest"); + REQUIRE(preview_body(*c->conversation(convo, await)) == "newest"); + + c->delete_message(newest.id, await); + + // The list says what was last actually said, rather than going blank. + CHECK(preview_body(*c->conversation(convo, await)) == "older"); +} + +TEST_CASE("Client: purging removes what a deletion left", "[client][messages][delete]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "one", from_epoch_ms(1000), "h1"); + deliver(*c, sender, "two", from_epoch_ms(2000), "h2"); + deliver(*c, sender, "three", from_epoch_ms(3000), "h3"); + + auto msgs = c->conversation(convo, await)->messages(await); + auto live = msgs[0].id; // "three" + c->delete_message(msgs[1].id, await); + c->delete_message(msgs[2].id, await); + + SECTION("one at a time, and only what was deleted") { + // A live message is refused: this is the one call that destroys history outright. + CHECK_FALSE(c->purge_deleted_message(live, await)); + CHECK(c->conversation(convo, await)->messages(await).size() == 1); + + CHECK(c->purge_deleted_message(msgs[1].id, await)); + CHECK(c->conversation(convo, await)->messages(50, std::nullopt, true, await).size() == 2); + + // Purging the same row twice is false the second time rather than an error. + CHECK_FALSE(c->purge_deleted_message(msgs[1].id, await)); + } + + SECTION("all of them at once") { + CHECK(c->conversation(convo, await)->purge_deleted(await) == 2); + + auto left = c->conversation(convo, await)->messages(50, std::nullopt, true, await); + REQUIRE(left.size() == 1); + CHECK(left[0].body == "three"); + + // Nothing left to purge. + CHECK(c->conversation(convo, await)->purge_deleted(await) == 0); + + // And having forgotten it, a redelivery is a new message again -- the cost the header warns + // about, asserted here so it is a decision rather than a surprise. + deliver(*c, sender, "two", from_epoch_ms(2000), "h2"); + CHECK(c->conversation(convo, await)->messages(await).size() == 2); + } +} + +TEST_CASE("Client: deleting for everyone is only for what we sent", "[client][messages][delete]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "theirs", from_epoch_ms(1000), "h1"); + auto theirs = c->conversation(convo, await)->messages(await).front(); + + // Their message, in their swarm as well as ours: an unsend request from us would be ignored at + // the other end, so we do not pretend to offer it. + CHECK_FALSE(c->delete_message_everywhere(theirs.id, await)); + CHECK_FALSE(c->message(theirs.id, await)->deleted.has_value()); + CHECK(c->message(theirs.id, await)->body == "theirs"); + + // Deleting it for ourselves is what that caller wanted, and still works. + CHECK(c->delete_message(theirs.id, await)); + CHECK(c->message(theirs.id, await)->deleted == Deletion::here); + + // A message that does not exist is false either way, not an error. + CHECK_FALSE(c->delete_message_everywhere(theirs.id + 10000, await)); +} + +TEST_CASE("Client: an unsend request from the author deletes the message", "[client][delete]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + auto unsend = [&](const SenderKeys& from, + const b33& author, + int64_t ts, + std::optional msgid, + std::string hash) { + SessionProtos::Content content; + content.set_sigtimestamp(9000); + auto* req = content.mutable_unsendrequest(); + req->set_msgtimestamp(static_cast(ts)); + req->set_author(oxenc::to_hex(author)); + if (msgid) + req->set_msgid(*msgid); + + auto plaintext = content.SerializeAsString(); + auto encoded = encode_dm_v1( + std::as_bytes(std::span{plaintext}), + from.ed_sk, + from_epoch_ms(9000), + own_sid(*c), + std::nullopt); + core::SwarmMessage sm{encoded, std::move(hash), from_epoch_ms(9000), from_epoch_ms(1e12)}; + c->core.loop().call_get([&] { + c->core.receive_messages({&sm, 1}, config::Namespace::Default, true); + return 0; + }); + }; + + deliver(*c, sender, "regrettable", from_epoch_ms(1000), "h1", "", std::nullopt, nullptr, 42); + auto msg = c->conversation(convo, await)->messages(await).front(); + REQUIRE(msg.body == "regrettable"); + + SECTION("naming someone else's message, ignored") { + // The one that actually exercises the authorisation check: a stranger who correctly names + // somebody else's message, so the lookup *would* find it. Naming a message they did write + // is a different test -- it would be honoured, and rightly. + SenderKeys stranger; + unsend(stranger, sender.session_id, 1000, 42, "u2"); + CHECK(c->message(msg.id, await)->body == "regrettable"); + CHECK_FALSE(c->message(msg.id, await)->deleted.has_value()); + } + + SECTION("from the author, honoured") { + unsend(sender, sender.session_id, 1000, 42, "u3"); + auto after = c->message(msg.id, await); + REQUIRE(after); + CHECK(after->deleted == Deletion::everywhere); + CHECK(after->body.empty()); + // The row stays, so a redelivery cannot resurrect it. + CHECK(after->hash == "h1"); + CHECK(c->conversation(convo, await)->unread() == 0); + } + + SECTION("matching nothing, ignored") { + unsend(sender, sender.session_id, 5555, 42, "u4"); + CHECK(c->message(msg.id, await)->body == "regrettable"); + } + + SECTION("ambiguous, refused rather than guessed") { + // Two messages in the same millisecond from the same sender, neither carrying a msgid: the + // case the id exists for, and the case a sender most often unsends. + deliver(*c, sender, "one", from_epoch_ms(7000), "amb1"); + deliver(*c, sender, "two", from_epoch_ms(7000), "amb2"); + REQUIRE(c->conversation(convo, await)->messages(await).size() == 3); + + unsend(sender, sender.session_id, 7000, std::nullopt, "u5"); + + auto left = c->conversation(convo, await)->messages(await); + CHECK(left.size() == 3); + for (const auto& m : left) + CHECK_FALSE(m.deleted.has_value()); + } +} + +TEST_CASE("Client: a sender's picture arrives with their message", "[client][receive][pictures]") { + TempClient c; + SenderKeys sender; + auto convo = ConversationId::dm(sender.session_id); + + std::vector key(32, std::byte{0x5}); + auto with_profile = [&](std::string_view display, + std::optional url, + std::optional> k, + std::optional stamp) { + return [=](SessionProtos::DataMessage& d) { + auto* p = d.mutable_profile(); + if (!display.empty()) + p->set_displayname(std::string{display}); + if (url) + p->set_profilepicture(*url); + if (stamp) + p->set_lastupdateseconds(*stamp); + if (k) + d.set_profilekey(std::string{reinterpret_cast(k->data()), k->size()}); + }; + }; + + deliver(*c, + sender, + "hi", + from_epoch_ms(1000), + "h1", + "", + std::nullopt, + with_profile("Padmé", "http://fs.example/file/7#pubkey=aa", key, 500)); + + auto pic = c->conversation(convo, await)->picture(); + CHECK(pic.url == "http://fs.example/file/7#pubkey=aa"); + CHECK(pic.key == key); + CHECK(c->conversation(convo, await)->display_name() == "Padmé"); + + SECTION("a newer profile replaces it") { + std::vector key2(32, std::byte{0x9}); + deliver(*c, + sender, + "again", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + with_profile("Padme", "http://fs.example/file/8", key2, 900)); + + CHECK(c->conversation(convo, await)->picture().url == "http://fs.example/file/8"); + CHECK(c->conversation(convo, await)->picture().key == key2); + } + + SECTION("a message that took a week to arrive does not undo a newer change") { + // Same sender, older profile stamp: the message is new to us but the profile in it is not. + std::vector stale(32, std::byte{0x1}); + deliver(*c, + sender, + "delayed", + from_epoch_ms(3000), + "h3", + "", + std::nullopt, + with_profile("Old Name", "http://fs.example/file/old", stale, 100)); + + CHECK(c->conversation(convo, await)->picture().url == "http://fs.example/file/7#pubkey=aa"); + CHECK(c->conversation(convo, await)->display_name() == "Padmé"); + } + + SECTION("a url with no key is not stored, since it could only fail to open") { + deliver(*c, + sender, + "keyless", + from_epoch_ms(4000), + "h4", + "", + std::nullopt, + with_profile("", "http://fs.example/file/nokey", std::nullopt, 900)); + + CHECK(c->conversation(convo, await)->picture().url == "http://fs.example/file/7#pubkey=aa"); + } + + SECTION("a message carrying no profile leaves it alone") { + deliver(*c, sender, "plain", from_epoch_ms(5000), "h5"); + CHECK(c->conversation(convo, await)->picture().url == "http://fs.example/file/7#pubkey=aa"); + CHECK(c->conversation(convo, await)->display_name() == "Padmé"); + } + + SECTION("and it reaches the config, so our other devices learn it too") { + auto entry = c->core.configs.contacts().get(oxenc::to_hex(sender.session_id)); + REQUIRE(entry); + CHECK(entry->profile_picture.url == "http://fs.example/file/7#pubkey=aa"); + CHECK(entry->profile_picture.key == key); + } +} diff --git a/tests/test_client/replies.cpp b/tests/test_client/replies.cpp new file mode 100644 index 000000000..45fae8f2d --- /dev/null +++ b/tests/test_client/replies.cpp @@ -0,0 +1,555 @@ +#include "common.hpp" + +namespace { + +/// Adds a quote naming `author` at `ts`, optionally with the target's msgid. +auto quoting(const b33& author, sys_ms ts, std::optional msgid = std::nullopt) { + return [author, ts, msgid](SessionProtos::DataMessage& d) { + auto* q = d.mutable_quote(); + q->set_msgtimestamp(static_cast(ts.time_since_epoch().count())); + q->set_author(oxenc::to_hex(author.begin(), author.end())); + if (msgid) + q->set_msgid(*msgid); + }; +} + +} // namespace + +TEST_CASE("Client: a reply names what it answers", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + auto first = from_epoch_ms(1000); + deliver(*c, peer, "the original", first, "h1", "", std::nullopt, nullptr, 7777); + deliver(*c, + peer, + "the answer", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2); + // Newest first. + const auto& answer = msgs[0]; + const auto& original = msgs[1]; + + REQUIRE(answer.reply); + CHECK(answer.reply->author == peer.session_id); + CHECK(answer.reply->timestamp == first); + REQUIRE(answer.reply->message_id.has_value()); + CHECK(*answer.reply->message_id == original.id); + + // The original is not itself a reply -- unset means "not a reply", which is a different + // statement from an unresolved reference. + CHECK_FALSE(original.reply.has_value()); +} + +TEST_CASE( + "Client: a reply to a message we do not have still names its author", "[client][replies]") { + TempClient c; + SenderKeys peer; + + auto missing = from_epoch_ms(500); + deliver(*c, + peer, + "answering something we never got", + from_epoch_ms(2000), + "h1", + "", + std::nullopt, + quoting(peer.session_id, missing, 4242)); + sync(*c); + + auto msgs = c->conversation(ConversationId::dm(peer.session_id), await)->messages(await); + REQUIRE(msgs.size() == 1); + REQUIRE(msgs[0].reply); + // Unresolved, but the author and timestamp came off the wire, so a client can still say who + // was replied to over an "original message not found" line. + CHECK_FALSE(msgs[0].reply->message_id.has_value()); + CHECK(msgs[0].reply->author == peer.session_id); + CHECK(msgs[0].reply->timestamp == missing); +} + +TEST_CASE("Client: a reply resolves once its target arrives", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // Out of order: the answer first. This is the case that makes resolving-on-read necessary + // rather than merely tidy -- resolved once at receipt, this reference would stay empty forever. + auto first = from_epoch_ms(1000); + deliver(*c, + peer, + "the answer", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 1); + REQUIRE(msgs[0].reply); + REQUIRE_FALSE(msgs[0].reply->message_id.has_value()); + + deliver(*c, peer, "the original", first, "h1", "", std::nullopt, nullptr, 7777); + sync(*c); + + msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2); + REQUIRE(msgs[0].reply); + REQUIRE(msgs[0].reply->message_id.has_value()); + CHECK(*msgs[0].reply->message_id == msgs[1].id); +} + +TEST_CASE("Client: msgid disambiguates a same-millisecond target", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // Two messages from one sender stamped identically: exactly what msgid exists for. + auto same = from_epoch_ms(1000); + deliver(*c, peer, "first", same, "h1", "", std::nullopt, nullptr, 111); + deliver(*c, peer, "second", same, "h2", "", std::nullopt, nullptr, 222); + deliver(*c, + peer, + "answering the second", + from_epoch_ms(2000), + "h3", + "", + std::nullopt, + quoting(peer.session_id, same, 222)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 3); + + int64_t second_id = 0; + for (const auto& m : msgs) + if (m.body == "second") + second_id = m.id; + REQUIRE(second_id != 0); + + REQUIRE(msgs[0].reply); + REQUIRE(msgs[0].reply->message_id.has_value()); + CHECK(*msgs[0].reply->message_id == second_id); +} + +TEST_CASE("Client: an ambiguous target resolves stably", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // No msgid on the quote: the wire cannot say which of the two was meant, so we pick the lowest + // id. Arbitrary, but the same answer on every read, which is what matters to a display. + auto same = from_epoch_ms(1000); + deliver(*c, peer, "first", same, "h1", "", std::nullopt, nullptr, 111); + deliver(*c, peer, "second", same, "h2", "", std::nullopt, nullptr, 222); + deliver(*c, + peer, + "answering one of them", + from_epoch_ms(2000), + "h3", + "", + std::nullopt, + quoting(peer.session_id, same)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 3); + REQUIRE(msgs[0].reply); + REQUIRE(msgs[0].reply->message_id.has_value()); + auto picked = *msgs[0].reply->message_id; + + auto again = c->conversation(convo, await)->messages(await); + REQUIRE(again[0].reply->message_id == picked); + + // And it is one of the two candidates, not something else. + CHECK((again[1].id == picked || again[2].id == picked)); +} + +TEST_CASE("Client: a quote with an unusable author is dropped", "[client][replies]") { + TempClient c; + SenderKeys peer; + + auto target = from_epoch_ms(1000); + deliver(*c, peer, "the original", target, "h0", "", std::nullopt, nullptr, 999); + deliver(*c, + peer, + "body survives", + from_epoch_ms(2000), + "h1", + "", + std::nullopt, + [](SessionProtos::DataMessage& d) { + auto* q = d.mutable_quote(); + q->set_msgtimestamp(1000); + q->set_author("not a session id"); + }); + // A well-formed one alongside it, so that "no reply" here means the bad reference was rejected + // rather than that nothing writes references at all. + deliver(*c, + peer, + "a good reply", + from_epoch_ms(3000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, target, 999)); + sync(*c); + + auto msgs = c->conversation(ConversationId::dm(peer.session_id), await)->messages(await); + REQUIRE(msgs.size() == 3); + + auto by_body = [&](std::string_view b) -> const Message& { + auto it = std::ranges::find_if(msgs, [&](const Message& m) { return m.body == b; }); + REQUIRE(it != msgs.end()); + return *it; + }; + + // The message carrying the bad quote is kept; only the unusable reference is discarded. + CHECK_FALSE(by_body("body survives").reply.has_value()); + CHECK(by_body("a good reply").reply.has_value()); +} + +TEST_CASE("Client: a reply is re-reported when its target arrives", "[client][replies]") { + Recorder rec; + TempClient c{rec.handlers()}; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + auto target_ts = from_epoch_ms(1000); + deliver(*c, + peer, + "the answer", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, target_ts, 7777)); + sync(*c); + + REQUIRE(rec.msg_added.size() == 1); + auto reply_id = rec.msg_added[0].second.id; + REQUIRE_FALSE(rec.msg_added[0].second.reply->message_id.has_value()); + rec.msg_updated.clear(); + + // The target lands afterwards. Nothing about the reply's own row changes, but what it resolves + // to does -- and a display holding it has no way to know that without being told. + deliver(*c, peer, "the original", target_ts, "h1", "", std::nullopt, nullptr, 7777); + sync(*c); + + auto reported = std::ranges::find_if( + rec.msg_updated, [&](const auto& p) { return p.second.id == reply_id; }); + REQUIRE(reported != rec.msg_updated.end()); + REQUIRE(reported->second.reply); + CHECK(reported->second.reply->message_id.has_value()); +} + +TEST_CASE("Client: deleting a target re-reports the replies to it", "[client][replies]") { + Recorder rec; + TempClient c{rec.handlers()}; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + auto target_ts = from_epoch_ms(1000); + deliver(*c, peer, "the original", target_ts, "h1", "", std::nullopt, nullptr, 7777); + deliver(*c, + peer, + "the answer", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, target_ts, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2); + auto reply_id = msgs[0].id; + auto target_id = msgs[1].id; + rec.msg_updated.clear(); + + c->delete_message(target_id, await); + sync(*c); + + // The target itself is reported, and so is the reply that points at it: what it should draw + // has changed even though its own row has not. + CHECK(std::ranges::any_of( + rec.msg_updated, [&](const auto& p) { return p.second.id == target_id; })); + CHECK(std::ranges::any_of( + rec.msg_updated, [&](const auto& p) { return p.second.id == reply_id; })); +} + +TEST_CASE("Client: a reply carries the message it answers", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + auto first = from_epoch_ms(1000); + deliver(*c, peer, "what was said", first, "h1", "", std::nullopt, nullptr, 7777); + deliver(*c, + peer, + "the answer", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2); + REQUIRE(msgs[0].reply); + REQUIRE(msgs[0].reply->message); + + // The whole message, not a summary of it: a caller drawing a reply line needs whatever it + // needs, and that is not knowable from here. + const auto& target = *msgs[0].reply->message; + CHECK(target.id == msgs[1].id); + CHECK(target.body == "what was said"); + CHECK(target.sender == peer.session_id); + CHECK(target.timestamp == first); +} + +TEST_CASE("Client: a quoted attachment-only message arrives whole", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // No body at all: exactly the case a body-only summary could not draw. + auto first = from_epoch_ms(1000); + deliver( + *c, + peer, + "", + first, + "h1", + "", + std::nullopt, + [](SessionProtos::DataMessage& d) { + auto* a = d.add_attachments(); + a->set_id(1); + a->set_url("http://fs.example/file/1#d"); + a->set_key(std::string(32, 'k')); + a->set_contenttype("image/png"); + a->set_filename("photo.png"); + }, + 7777); + deliver(*c, + peer, + "nice one", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2); + REQUIRE(msgs[0].reply); + REQUIRE(msgs[0].reply->message); + + const auto& target = *msgs[0].reply->message; + CHECK(target.body.empty()); + REQUIRE(target.attachments.size() == 1); + CHECK(target.attachments[0].filename == "photo.png"); + // Derived on the nested message too, not left at its default. + CHECK(target.gallery_viewable); +} + +TEST_CASE("Client: reply loading stops one level down", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // A chain: A <- B <- C. Reading C must give B, and B must name A without carrying it. + auto a_ts = from_epoch_ms(1000); + auto b_ts = from_epoch_ms(2000); + deliver(*c, peer, "A", a_ts, "h1", "", std::nullopt, nullptr, 111); + deliver(*c, peer, "B", b_ts, "h2", "", std::nullopt, quoting(peer.session_id, a_ts, 111), 222); + deliver(*c, + peer, + "C", + from_epoch_ms(3000), + "h3", + "", + std::nullopt, + quoting(peer.session_id, b_ts, 222)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 3); + const auto& cmsg = msgs[0]; + + REQUIRE(cmsg.reply); + REQUIRE(cmsg.reply->message); + CHECK(cmsg.reply->message->body == "B"); + + // B is itself a reply, and says so -- but the payload stops here, which is what keeps a read + // from walking a chain of unbounded length. The id is still there to ask with. + const auto& nested = *cmsg.reply->message; + REQUIRE(nested.reply); + CHECK(nested.reply->author == peer.session_id); + CHECK(nested.reply->timestamp == a_ts); + CHECK(nested.reply->message_id.has_value()); + CHECK(nested.reply->message == nullptr); +} + +TEST_CASE("Client: several replies to one message share one copy", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + auto first = from_epoch_ms(1000); + deliver(*c, peer, "the original", first, "h1", "", std::nullopt, nullptr, 7777); + deliver(*c, + peer, + "answer one", + from_epoch_ms(2000), + "h2", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + deliver(*c, + peer, + "answer two", + from_epoch_ms(3000), + "h3", + "", + std::nullopt, + quoting(peer.session_id, first, 7777)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 3); + REQUIRE(msgs[0].reply); + REQUIRE(msgs[1].reply); + REQUIRE(msgs[0].reply->message); + + // One read, one payload: the pointers are the same object, not two copies of it. + CHECK(msgs[0].reply->message == msgs[1].reply->message); +} + +TEST_CASE("Client: many distinct reply targets load correctly", "[client][replies]") { + TempClient c; + SenderKeys peer; + auto convo = ConversationId::dm(peer.session_id); + + // More distinct targets than the lookup caches a statement for, so this exercises the one-off + // statement branch as well as the binding of a longer placeholder list. + constexpr int n = 6; + std::vector stamps; + for (int i = 0; i < n; i++) { + auto ts = from_epoch_ms(1000 + i); + stamps.push_back(ts); + deliver(*c, + peer, + "original {}"_format(i), + ts, + "o{}"_format(i), + "", + std::nullopt, + nullptr, + 100 + i); + } + for (int i = 0; i < n; i++) + deliver(*c, + peer, + "answer {}"_format(i), + from_epoch_ms(5000 + i), + "a{}"_format(i), + "", + std::nullopt, + quoting(peer.session_id, stamps[i], 100 + i)); + sync(*c); + + auto msgs = c->conversation(convo, await)->messages(await); + REQUIRE(msgs.size() == 2 * n); + + // Every answer resolved, and each to its *own* target rather than all to one of them -- which + // is what a mis-bound placeholder list would produce. + int checked = 0; + for (const auto& m : msgs) { + if (!m.body.starts_with("answer ")) + continue; + auto which = m.body.substr(7); + REQUIRE(m.reply); + REQUIRE(m.reply->message); + CHECK(m.reply->message->body == "original {}"_format(which)); + checked++; + } + CHECK(checked == n); +} + +TEST_CASE("Client: a sent reply names what it answers", "[client][replies][send]") { + TempClient c; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + auto convo = ConversationId::dm(me); + + auto first = c->send_message(convo, {.body = "the original"}, await); + auto second = c->send_message(convo, {.body = "answering", .reply_to = first}, await); + sync(*c); + + auto msg = c->message(second, await); + REQUIRE(msg); + REQUIRE(msg->reply); + CHECK(msg->reply->author == me); + REQUIRE(msg->reply->message_id.has_value()); + CHECK(*msg->reply->message_id == first); + + // Our own reply resolves through the same rule as an incoming one, so the message comes with + // it rather than only the reference. + REQUIRE(msg->reply->message); + CHECK(msg->reply->message->body == "the original"); +} + +TEST_CASE("Client: a sent reply puts a quote on the wire", "[client][replies][send]") { + TempClient c; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + auto convo = ConversationId::dm(me); + + auto first = c->send_message(convo, {.body = "the original"}, await); + auto second = c->send_message(convo, {.body = "answering", .reply_to = first}, await); + sync(*c); + + // What was stored is what goes out, so the wire form is checkable from the raw content. + auto dump = c->message_debug(second, await); + REQUIRE(dump); + CHECK(dump->find("quote") != std::string::npos); + CHECK(dump->find("msgTimestamp") != std::string::npos); + // The snippet is deliberately absent: current clients do not send one, and one that arrives + // would not be trusted. + CHECK(dump->find("text") == std::string::npos); +} + +TEST_CASE("Client: reply_to must name a message in this conversation", "[client][replies][send]") { + TempClient c; + SenderKeys peer; + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + auto mine = c->send_message(ConversationId::dm(me), {.body = "a note"}, await); + + // Nonexistent, and in another conversation: both are caller error, and both are refused on the + // calling thread rather than accepted and then silently dropped. + CHECK_THROWS_AS( + c->send_message(ConversationId::dm(me), {.body = "x", .reply_to = 999999}, await), + std::invalid_argument); + + c->open_dm(ConversationId::dm(peer.session_id), await); + TestHelper::seed_pfs_nak(c->core, peer.session_id); + CHECK_THROWS_AS( + c->send_message( + ConversationId::dm(peer.session_id), {.body = "x", .reply_to = mine}, await), + std::invalid_argument); +} diff --git a/tests/test_client/requests.cpp b/tests/test_client/requests.cpp new file mode 100644 index 000000000..076d743e8 --- /dev/null +++ b/tests/test_client/requests.cpp @@ -0,0 +1,142 @@ +#include "config_helpers.hpp" + +TEST_CASE("Client: a stranger's message is a request, not a conversation", "[client][requests]") { + Recorder r; + TempClient c{r.handlers()}; + SenderKeys sender; + auto id = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "hi, remember me?", from_epoch_ms(5000), "h1", "Jar Jar"); + sync(*c); + + CHECK(c->conversations(await).empty()); + auto requests = c->message_requests(await); + REQUIRE(requests.size() == 1); + CHECK(requests[0].id() == id); + CHECK(requests[0].dm()->request); + CHECK(requests[0].display_name() == "Jar Jar"); + CHECK(requests[0].unread() == 1); + + // A conversation in every other respect, including being announced as one -- what differs is + // which list it belongs to, and `request` is what says so. + REQUIRE(r.added.size() == 1); + CHECK(r.added[0].dm()->request); + REQUIRE(c->conversation(id, await)); + CHECK(c->conversation(id, await)->dm()->request); + CHECK(c->conversation(id, await)->messages(await).size() == 1); + + // And it is synced, so a request answered on one device is not still waiting on another. Their + // writing to us is what says they approved us; nothing yet says we approved them. + auto entry = c->core.configs.contacts().get(oxenc::to_hex(sender.session_id)); + REQUIRE(entry); + CHECK(entry->approved_me); + CHECK_FALSE(entry->approved); +} + +TEST_CASE("Client: answering a request accepts it", "[client][requests]") { + Recorder r; + TempClient c{r.handlers()}; + SenderKeys sender; + auto id = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "hello?", from_epoch_ms(5000), "h1"); + sync(*c); + REQUIRE(c->message_requests(await).size() == 1); + r.order.clear(); + + // There is no separate accept: writing to someone is what approving them is. + c->send_message(id, {.body = "hello yourself"}, await); + sync(*c); + + CHECK(c->message_requests(await).empty()); + REQUIRE(c->conversations(await).size() == 1); + CHECK_FALSE(c->conversations(await)[0].dm()->request); + CHECK(c->core.configs.contacts().get(oxenc::to_hex(sender.session_id))->approved); + + // It left one list and joined the other, which is neither an addition nor a removal to either, + // so both are replaced. + CHECK(std::ranges::count(r.order, "replaced") == 1); + CHECK(std::ranges::count(r.order, "requests") == 1); +} + +TEST_CASE("Client: a linked device's answer accepts the request", "[client][requests]") { + TempClient c; + SenderKeys sender; + auto id = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "hello?", from_epoch_ms(5000), "h1"); + REQUIRE(c->message_requests(await).size() == 1); + + // Our own message coming back off our own swarm because another device sent it. syncTarget + // says who it was addressed to, and sending to them is what approved them. + deliver(*c, + self_keys(*c), + "answered elsewhere", + from_epoch_ms(6000), + "h2", + "", + sender.session_id); + + CHECK(c->message_requests(await).empty()); + REQUIRE(c->conversations(await).size() == 1); + CHECK(c->conversations(await)[0].id() == id); +} + +TEST_CASE("Client: writing first leaves us awaiting their approval", "[client][requests]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + + c->send_message(id, {.body = "are you there?"}, await); + sync(*c); + + // The mirror of a request: we are in *their* requests list, and nothing they could be sent + // says so -- only a message back from them clears it. Meanwhile it is an ordinary conversation + // of ours, since we chose to start it. + REQUIRE(c->conversations(await).size() == 1); + CHECK(c->conversations(await)[0].dm()->awaiting_approval); + CHECK_FALSE(c->conversations(await)[0].dm()->request); + CHECK(c->message_requests(await).empty()); + + deliver(*c, them, "here", from_epoch_ms(9000), "h1"); + + REQUIRE(c->conversation(id, await)); + CHECK_FALSE(c->conversation(id, await)->dm()->awaiting_approval); + CHECK_FALSE(c->conversation(id, await)->dm()->request); +} + +TEST_CASE("Client: note to self is never a message request", "[client][requests]") { + TempClient c; + auto me = self_convo(*c.client); + + c->send_message(me, {.body = "a note"}, await); + sync(*c); + + CHECK(c->message_requests(await).empty()); + REQUIRE(c->conversation(me, await)); + CHECK_FALSE(c->conversation(me, await)->dm()->request); + + // Nor awaiting anything: there is nobody at the other end to accept. + CHECK_FALSE(c->conversation(me, await)->dm()->awaiting_approval); +} + +TEST_CASE("Client: a blocked account's messages are refused", "[client][requests]") { + TempClient c; + SenderKeys sender; + auto id = ConversationId::dm(sender.session_id); + + deliver(*c, sender, "first", from_epoch_ms(5000), "h1"); + REQUIRE(c->conversation(id, await)->messages(await).size() == 1); + + c->dm(id, await)->set_blocked(true, await); + deliver(*c, sender, "and again", from_epoch_ms(6000), "h2"); + + // Refused on arrival rather than hidden when drawing, so nothing they send becomes history or + // an unread count. + CHECK(c->conversation(id, await)->messages(await).size() == 1); + CHECK(c->conversation(id, await)->unread() == 1); + + c->dm(id, await)->set_blocked(false, await); + deliver(*c, sender, "still there?", from_epoch_ms(7000), "h3"); + CHECK(c->conversation(id, await)->messages(await).size() == 2); +} diff --git a/tests/test_client/sending.cpp b/tests/test_client/sending.cpp new file mode 100644 index 000000000..ef5f03f92 --- /dev/null +++ b/tests/test_client/sending.cpp @@ -0,0 +1,414 @@ +#include "common.hpp" + +// ── Sending ───────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: send_message stores, dispatches and reaches sent", "[client][send]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + constexpr auto peer = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + auto convo = ConversationId::dm(peer); + + // No PFS keys published for either of us, so these fall back to v1 sends: the recipient's copy + // and the copy for our own swarm, which needs our own keys answered too. + TestHelper::seed_pfs_nak(c->core, peer); + TestHelper::seed_pfs_nak(c->core, own_sid(*c)); + + auto id = c->send_message(convo, {.body = "general kenobi"}, await); + + // A store to the recipient's swarm and one to our own, both into the default namespace, with + // something in them. + auto sent = stores(*net); + REQUIRE(sent.size() == 2); + for (const auto* r : sent) { + auto body = store_body(*r); + CHECK(body["namespace"] == static_cast(config::Namespace::Default)); + CHECK_FALSE(body["data"].get().empty()); + } + CHECK(accept_stores(*net) == 2); + + auto msg = c->message(id, await); + REQUIRE(msg.has_value()); + CHECK(msg->body == "general kenobi"); + CHECK(msg->outgoing); + CHECK(msg->sender == own_sid(*c)); + CHECK(msg->send_state == SendState::sent); + + // Of the two hashes the two stores were assigned, the one kept is our own swarm's: it is the + // copy we can still act on, and the one a redelivery would arrive under. + CHECK(msg->hash == store_hash_for(oxenc::to_hex(own_sid(*c)))); + + // The conversation was created by the send and shows the outgoing message as its preview. + auto convos = c->conversations(await); + REQUIRE(convos.size() == 1); + CHECK(preview_body(convos[0]) == "general kenobi"); + // Ours, which is what lets a row prefix "You: ". + REQUIRE(convos[0].last_preview()); + CHECK(convos[0].last_preview()->outgoing); + // Our own message is never unread. + CHECK(convos[0].unread() == 0); +} + +TEST_CASE("Client: a failed send is recorded as failed", "[client][send]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + constexpr auto peer = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + TestHelper::seed_pfs_nak(c->core, peer); + TestHelper::seed_pfs_nak(c->core, own_sid(*c)); + + auto id = c->send_message(ConversationId::dm(peer), {.body = "into the void"}, await); + + // The swarm refusing the store is what a failure is, rather than us declining to attempt one. + auto sent = stores(*net); + REQUIRE(sent.size() == 2); + for (auto* r : sent) + r->callback(false, false, 500, {}, "nope"); + + CHECK(c->message(id, await)->send_state == SendState::failed); +} + +TEST_CASE("Client: sending to a non-DM conversation is rejected", "[client][send]") { + TempClient c; + constexpr auto gid = "03fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + CHECK_THROWS_AS( + c->send_message(ConversationId::group(gid), {.body = "hi"}, await), + std::invalid_argument); +} + +TEST_CASE("Client: an in-flight send becomes interrupted after a restart", "[client][send]") { + // The store is captured and never answered, which leaves the message mid-flight -- exactly the + // state a crash would leave behind. + TempClient c; + auto* net = attach_mock_network(c->core); + + constexpr auto peer = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + TestHelper::seed_pfs_nak(c->core, peer); + TestHelper::seed_pfs_nak(c->core, own_sid(*c)); + + auto id = c->send_message(ConversationId::dm(peer), {.body = "did this land?"}, await); + CHECK(c->message(id, await)->send_state == SendState::sending); + + c.reopen(); + + // Not "failed": we genuinely do not know whether the swarm stored it. + CHECK(c->message(id, await)->send_state == SendState::interrupted); + CHECK(c->message(id, await)->body == "did this land?"); +} + +// ── Signals ───────────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("Client: the application is told what changed", "[client][signals]") { + SenderKeys sender; + Recorder r; + TempClient c{r.handlers()}; + + deliver(*c, sender, "ping", from_epoch_ms(1000), "h1"); + sync(*c); + + auto convo = ConversationId::dm(sender.session_id); + CHECK(r.order == std::vector{"added", "message", "updated"}); + + // Every handler is given the state itself, not something to go and look up. + REQUIRE(r.added.size() == 1); + CHECK(r.added[0].id() == convo); + REQUIRE(r.msg_added.size() == 1); + CHECK(r.msg_added[0].first == convo); + CHECK(r.msg_added[0].second.body == "ping"); + REQUIRE(r.updated.size() == 1); + CHECK(preview_body(r.updated[0]) == "ping"); + CHECK(r.updated[0].unread() == 1); + + // A second message on an existing conversation does not re-announce the conversation. + r.order.clear(); + deliver(*c, sender, "pong", from_epoch_ms(2000), "h2"); + sync(*c); + CHECK(r.order == std::vector{"message", "updated"}); +} + +TEST_CASE( + "Client: a batch reports each message but settles the conversation once", + "[client][signals]") { + SenderKeys sender; + Recorder r; + TempClient c{r.handlers()}; + + // One delivery carrying several messages, as a swarm poll produces. + std::vector encoded; + for (int i = 0; i < 5; i++) { + SessionProtos::Content content; + auto ts = from_epoch_ms(1000 + i); + content.set_sigtimestamp(static_cast(epoch_ms(ts))); + content.mutable_datamessage()->set_body("m{}"_format(i)); + encoded.push_back(content.SerializeAsString()); + } + std::vector> wire; + for (int i = 0; i < 5; i++) + wire.push_back(encode_dm_v1( + std::as_bytes(std::span{encoded[i]}), + sender.ed_sk, + from_epoch_ms(1000 + i), + own_sid(*c), + std::nullopt)); + std::vector batch; + for (int i = 0; i < 5; i++) + batch.push_back(core::SwarmMessage{ + wire[i], + "b{}"_format(i), + from_epoch_ms(1000 + i), + from_epoch_ms(1'000'000'000'000)}); + + c->core.loop().call_get([&] { + c->core.receive_messages(batch, config::Namespace::Default, true); + return 0; + }); + sync(*c); + + // Five messages, but the conversation settles once rather than being rebuilt five times. + CHECK(std::ranges::count(r.order, "message") == 5); + CHECK(std::ranges::count(r.order, "updated") == 1); + REQUIRE(r.updated.size() == 1); + CHECK(r.updated[0].unread() == 5); + CHECK(preview_body(r.updated[0]) == "m4"); +} + +TEST_CASE("Client: state is committed before the handler fires", "[client][signals]") { + SenderKeys sender; + std::optional body_seen_from_handler; + Client* self = nullptr; + + TempClient c{callbacks{.message_added = [&](const ConversationId&, const Message& m) { + // Waiting from inside a handler: the loop runs it inline, since it is already this thread. + body_seen_from_handler = self->message(m.id, await)->body; + }}}; + self = &*c; + + deliver(*c, sender, "readable already", from_epoch_ms(1000), "h1"); + CHECK(body_seen_from_handler == "readable already"); +} + +TEST_CASE("Client: a throwing handler is contained", "[client][signals]") { + SenderKeys sender; + TempClient c{callbacks{.message_added = [](const ConversationId&, const Message&) { + throw std::runtime_error{"deliberate"}; + }}}; + + // The exception is caught and logged rather than escaping into Core's event loop, and the + // message is stored regardless: a broken listener must not cost us data. + CHECK_NOTHROW(deliver(*c, sender, "still fine", from_epoch_ms(1000), "h1")); + CHECK(c->conversation(ConversationId::dm(sender.session_id), await)->messages(await).size() == + 1); +} + +TEST_CASE("Client: send status changes are reported as message_updated", "[client][signals]") { + Recorder r; + TempClient c{r.handlers()}; + auto* net = attach_mock_network(c->core); + + constexpr auto peer = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + + // Only the peer's keys are answered, so only the recipient's copy is dispatched: the copy for + // our own swarm stays waiting on our keys. That is what makes the single update below the one + // for `send_state` rather than for the sync copy's. + TestHelper::seed_pfs_nak(c->core, peer); + + auto id = c->send_message(ConversationId::dm(peer), {.body = "hello"}, await); + sync(*c); + r.order.clear(); + r.msg_updated.clear(); + + REQUIRE(accept_stores(*net) == 1); + + CHECK(r.order == std::vector{"message_updated"}); + REQUIRE(r.msg_updated.size() == 1); + CHECK(r.msg_updated[0].second.id == id); + CHECK(r.msg_updated[0].second.send_state == SendState::sent); + + // The only store that landed was the recipient's, and that hash belongs to their swarm: it is + // not something we could ever look up, so it is not recorded as ours. + CHECK_FALSE(r.msg_updated[0].second.hash.has_value()); +} + +TEST_CASE("Client: priority orders the list and hides", "[client][convos]") { + TempClient c; + SenderKeys a, b, d; + for (const auto& k : {a, b, d}) + approve(*c, k.session_id); + + // Three conversations, most recent first: d, b, a. + deliver(*c, a, "first", from_epoch_ms(1000), "h1"); + deliver(*c, b, "second", from_epoch_ms(2000), "h2"); + deliver(*c, d, "third", from_epoch_ms(3000), "h3"); + sync(*c); + + auto ida = ConversationId::dm(a.session_id); + auto idb = ConversationId::dm(b.session_id); + auto idd = ConversationId::dm(d.session_id); + + auto ids = [&] { + std::vector out; + for (const auto& convo : c->conversations(await)) + out.push_back(convo.id()); + return out; + }; + CHECK(ids() == std::vector{idd, idb, ida}); + + // Higher priority sorts first, regardless of recency. + c->conversation(ida, await)->set_priority(1, await); + CHECK(ids() == std::vector{ida, idd, idb}); + CHECK(c->conversation(ida, await)->priority() == 1); + + // A bigger number outranks a smaller one. + c->conversation(idb, await)->set_priority(5, await); + CHECK(ids() == std::vector{idb, ida, idd}); + + // Equal priorities form a block that sorts among itself by recency: b is pinned alongside a but + // is the more recently active of the two, so it leads. d stays below both, unpinned. + c->conversation(ida, await)->set_priority(5, await); + CHECK(ids() == std::vector{idb, ida, idd}); + + // Negative is hidden: gone from the list entirely rather than sorted last. + c->conversation(idb, await)->set_priority(-1, await); + CHECK(ids() == std::vector{ida, idd}); + + // Still reachable by name, though: hidden is a statement about the list, and this is the only + // way back to one. + REQUIRE(c->conversation(idb, await).has_value()); + CHECK(c->conversation(idb, await)->priority() == -1); + + // ...and unhiding brings it back where its priority says. + c->conversation(idb, await)->set_priority(0, await); + CHECK(ids() == std::vector{ida, idd, idb}); +} + +TEST_CASE("Client: a priority change replaces the whole list", "[client][signals]") { + SenderKeys a, b; + Recorder r; + TempClient c{r.handlers()}; + approve(*c, a.session_id); + approve(*c, b.session_id); + + deliver(*c, a, "first", from_epoch_ms(1000), "h1"); + deliver(*c, b, "second", from_epoch_ms(2000), "h2"); + sync(*c); + r.order.clear(); + + c->conversation(ConversationId::dm(a.session_id), await)->set_priority(3, await); + + // Reported as a replacement, not as an update to the one conversation whose priority changed: + // what moved is the list. Both lists are replaced together, because hiding takes a + // conversation out of whichever one it was in and the caller does not have to work out which. + CHECK(r.order == std::vector{"replaced", "requests"}); + REQUIRE(r.replaced.size() == 1); + REQUIRE(r.replaced[0].size() == 2); + CHECK(r.replaced[0][0].id() == ConversationId::dm(a.session_id)); + CHECK(r.replaced[0][0].priority() == 3); + + // Hiding removes it from the replacement list, which is how a subscriber learns it is gone. + r.order.clear(); + r.replaced.clear(); + c->conversation(ConversationId::dm(a.session_id), await)->set_priority(-1, await); + CHECK(r.order == std::vector{"replaced", "requests"}); + REQUIRE(r.replaced.size() == 1); + REQUIRE(r.replaced[0].size() == 1); + CHECK(r.replaced[0][0].id() == ConversationId::dm(b.session_id)); + + // Setting the same value again changes nothing, so it says nothing. + r.order.clear(); + c->conversation(ConversationId::dm(a.session_id), await)->set_priority(-1, await); + CHECK(r.order.empty()); +} + +TEST_CASE("Client: the two copies of a send report separately", "[client][send]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + constexpr auto peer = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + TestHelper::seed_pfs_nak(c->core, peer); + TestHelper::seed_pfs_nak(c->core, own_sid(*c)); + + auto id = c->send_message(ConversationId::dm(peer), {.body = "two ways"}, await); + + // Which swarm a store is bound for is the pubkey it names, so the two copies can be answered + // independently and in either order. + auto sent = stores(*net); + REQUIRE(sent.size() == 2); + + auto my_hex = oxenc::to_hex(own_sid(*c)); + auto is_self = [&](const MockNetwork::SentRequest* r) { + return store_body(*r)["pubkey"].get() == my_hex; + }; + auto to_peer = std::ranges::find_if_not(sent, is_self); + auto to_self = std::ranges::find_if(sent, is_self); + REQUIRE(to_peer != sent.end()); + REQUIRE(to_self != sent.end()); + + // The recipient's copy lands; our own swarm has not answered yet. + (*to_peer)->callback(true, false, 200, {}, "{}"); + CHECK(c->message(id, await)->send_state == SendState::sent); + CHECK(c->message(id, await)->sync_send_state == SendState::sending); + + // The sync copy fails, which says nothing about whether the message arrived. + (*to_self)->callback(false, false, 500, {}, "nope"); + CHECK(c->message(id, await)->send_state == SendState::sent); + CHECK(c->message(id, await)->sync_send_state == SendState::failed); +} + +TEST_CASE("Client: sending to ourselves stores once", "[client][send]") { + TempClient c; + auto* net = attach_mock_network(c->core); + + auto me = own_sid(*c); + TestHelper::seed_pfs_nak(c->core, me); + + // Mirrors opening the conversation first, as a UI does, before sending into it. + auto convo = c->open_dm(ConversationId::dm(me), await); + CHECK(convo.id == ConversationId::dm(me)); + + auto id = c->send_message(ConversationId::dm(me), {.body = "note to self"}, await); + CHECK(c->message(id, await)->body == "note to self"); + CHECK(preview_body(*c->conversation(ConversationId::dm(me), await)) == "note to self"); + + // One store reaching the swarm, not two: our own swarm is the recipient's, so the sync copy + // would be the same store twice. + CHECK(accept_stores(*net) == 1); + + // One swarm, so one send: there is no separate sync copy to have a state for. + CHECK(c->message(id, await)->send_state.has_value()); + CHECK_FALSE(c->message(id, await)->sync_send_state.has_value()); + + // That single store went to our own swarm, so its hash is one worth keeping. + CHECK(c->message(id, await)->hash == store_hash_for(oxenc::to_hex(me))); + CHECK(c->conversation(ConversationId::dm(me), await)->messages(await).size() == 1); + + // ...and when our own swarm hands it straight back on the next poll, which is what note to self + // does, it must recognise its own message rather than storing a second copy. What makes that + // work is the msgid: the copy coming back carries the one we generated when sending, which is + // exactly what a hash of the two copies could not do. + auto ts = c->message(id, await)->timestamp; + auto msgid = c->core.loop().call_get([&] { + return c->core.database().conn().prepared_get( + "SELECT msgid FROM messages WHERE id = ?", id); + }); + SessionProtos::Content sent; + sent.set_sigtimestamp(static_cast(epoch_ms(ts))); + sent.set_msgid(msgid); + sent.mutable_datamessage()->set_body("note to self"); + sent.mutable_datamessage()->set_timestamp(static_cast(epoch_ms(ts))); + sent.mutable_datamessage()->set_synctarget(oxenc::to_hex(me)); + auto plaintext = sent.SerializeAsString(); + auto encoded = encode_dm_v1( + std::as_bytes(std::span{plaintext}), self_keys(*c).ed_sk, ts, me, std::nullopt); + core::SwarmMessage sm{encoded, "swarmhash", ts, from_epoch_ms(1'000'000'000'000)}; + c->core.loop().call_get([&] { + c->core.receive_messages({&sm, 1}, config::Namespace::Default, true); + return 0; + }); + + CHECK(c->conversation(ConversationId::dm(me), await)->messages(await).size() == 1); +} diff --git a/tests/test_client/volatile.cpp b/tests/test_client/volatile.cpp new file mode 100644 index 000000000..97d1723ff --- /dev/null +++ b/tests/test_client/volatile.cpp @@ -0,0 +1,125 @@ +#include "config_helpers.hpp" + +// Recent, because ConvoInfoVolatile refuses to store a last-read older than PRUNE_LOW (30 days) +// and would silently keep nothing at all from the 1970 timestamps the other tests here use. +sys_ms recently(std::chrono::milliseconds ago) { + return clock_now_ms() - ago; +} + +TEST_CASE("Client: reading a conversation publishes the watermark", "[client][volatile]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + auto hex = oxenc::to_hex(them.session_id); + approve(*c, them.session_id); + + auto newest = recently(2s); + deliver(*c, them, "one", recently(3s), "h1"); + deliver(*c, them, "two", newest, "h2"); + REQUIRE(c->conversation(id, await)->unread() == 2); + + c->conversation(id, await)->mark_read(await); + + CHECK(c->conversation(id, await)->unread() == 0); + auto entry = c->core.configs.convo_info_volatile().get_1to1(hex); + REQUIRE(entry); + CHECK(entry->last_read == newest.time_since_epoch().count()); +} + +TEST_CASE("Client: a watermark from another device applies", "[client][volatile]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + auto hex = oxenc::to_hex(them.session_id); + approve(*c, them.session_id); + + auto older = recently(30s); + deliver(*c, them, "one", older, "h1"); + deliver(*c, them, "two", recently(10s), "h2"); + REQUIRE(c->conversation(id, await)->unread() == 2); + + // Read up to the first message on another device. + auto read = volatile_from_another_device(*c.client, [&](config::ConvoInfoVolatile& theirs) { + auto e = theirs.get_or_construct_1to1(hex); + e.last_read = older.time_since_epoch().count(); + theirs.set(e); + }); + merge_volatile(*c.client, read); + + CHECK(c->conversation(id, await)->unread() == 1); +} + +TEST_CASE("Client: a stale watermark cannot unread what we have read", "[client][volatile]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + auto hex = oxenc::to_hex(them.session_id); + approve(*c, them.session_id); + + auto older = recently(30s); + auto newest = recently(10s); + deliver(*c, them, "one", older, "h1"); + deliver(*c, them, "two", newest, "h2"); + c->conversation(id, await)->mark_read(await); + REQUIRE(c->conversation(id, await)->unread() == 0); + + // The config permits a value to be written backwards on purpose, and a same-seqno conflict + // resolves by a tie-break that knows nothing about which value is newer -- so an older one + // really can arrive, and applying it would make read messages unread again. + auto stale = volatile_from_another_device(*c.client, [&](config::ConvoInfoVolatile& theirs) { + auto e = theirs.get_or_construct_1to1(hex); + e.last_read = older.time_since_epoch().count(); + theirs.set(e); + }); + merge_volatile(*c.client, stale); + + CHECK(c->conversation(id, await)->unread() == 0); + + // ...and we do not publish the stale value back out, either. + TestHelper::sync_convo_volatile(*c.client, id); + auto entry = c->core.configs.convo_info_volatile().get_1to1(hex); + REQUIRE(entry); + CHECK(entry->last_read == newest.time_since_epoch().count()); +} + +TEST_CASE("Client: marking unread syncs, and reading clears it", "[client][volatile]") { + TempClient c; + SenderKeys them; + auto id = ConversationId::dm(them.session_id); + auto hex = oxenc::to_hex(them.session_id); + approve(*c, them.session_id); + + deliver(*c, them, "one", recently(5s), "h1"); + c->conversation(id, await)->mark_read(await); + REQUIRE(c->conversation(id, await)->unread() == 0); + + c->conversation(id, await)->set_marked_unread(true, await); + + // Survives having read everything, which is the whole point of it. + CHECK(c->conversation(id, await)->marked_unread()); + CHECK(c->conversation(id, await)->unread() == 0); + CHECK(c->core.configs.convo_info_volatile().get_1to1(hex)->unread); + + c->conversation(id, await)->mark_read(await); + CHECK_FALSE(c->conversation(id, await)->marked_unread()); + CHECK_FALSE(c->core.configs.convo_info_volatile().get_1to1(hex)->unread); +} + +TEST_CASE("Client: read state for a conversation we do not have is ignored", "[client][volatile]") { + TempClient c; + auto them = "05" + std::string(64, '8'); + auto id = dm_from_hex(them); + + // An entry outlives the conversation it describes: this config is pruned by age, not by + // anything noticing a deletion. Creating a conversation from one would resurrect what another + // device deleted. + auto orphan = volatile_from_another_device(*c.client, [&](config::ConvoInfoVolatile& theirs) { + auto e = theirs.get_or_construct_1to1(them); + e.last_read = clock_now_ms().time_since_epoch().count(); + theirs.set(e); + }); + merge_volatile(*c.client, orphan); + + CHECK_FALSE(c->conversation(id, await)); + CHECK(c->conversations(await).empty()); +} diff --git a/tests/test_compression.cpp b/tests/test_compression.cpp index 2667d3423..c0eea2db6 100644 --- a/tests/test_compression.cpp +++ b/tests/test_compression.cpp @@ -5,32 +5,29 @@ #include #include +#include #include #include "utils.hpp" -namespace session::config { -void compress_message(std::vector& msg, int level); -} - TEST_CASE("compression", "[config][compression]") { - auto data = + auto data = session::to_vector( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hexbytes; + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hex_b); CHECK(data.size() == 81); auto d = data; session::config::compress_message(d, 1); - CHECK(d[0] == 'z'); + CHECK(d[0] == std::byte{'z'}); CHECK(d.size() == 18); CHECK(to_hex(d) == "7a28b52ffd205145000010aaaa01008c022c"); d = data; session::config::compress_message(d, 5); - CHECK(d[0] == 'z'); + CHECK(d[0] == std::byte{'z'}); CHECK(d.size() == 17); CHECK(to_hex(d) == "7a28b52ffd20513d000008aa01000dea84"); @@ -49,7 +46,7 @@ TEST_CASE("compression", "[config][compression]") { "l" "i0e" "32:" + - session::to_string("ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hexbytes) + + session::to_string("ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hex_b) + "de" "e" "e" @@ -73,7 +70,7 @@ TEST_CASE("compression", "[config][compression]") { "l" "i0e" "32:" + - session::to_string("ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hexbytes) + + session::to_string("ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hex_b) + "de" "e" "e" @@ -91,7 +88,7 @@ TEST_CASE("compression", "[config][compression]") { // Doesn't compress, so shouldn't change: CHECK(d.size() == 142); session::config::compress_message(d, 1); - CHECK(d[0] == 'd'); + CHECK(d[0] == std::byte{'d'}); CHECK(d.size() == 142); CHECK(reinterpret_cast(d.data()) == dptr); @@ -100,7 +97,7 @@ TEST_CASE("compression", "[config][compression]") { // version of zstd). d = data2; session::config::compress_message(d, 1); - CHECK(d[0] == 'z'); + CHECK(d[0] == std::byte{'z'}); CHECK(d.size() == 161); CHECK(d.size() < data2.size()); CHECK(to_hex(d) == @@ -111,7 +108,7 @@ TEST_CASE("compression", "[config][compression]") { d = data2; session::config::compress_message(d, 5); - CHECK(d[0] == 'z'); + CHECK(d[0] == std::byte{'z'}); CHECK(d.size() == 156); CHECK(d.size() < data2.size()); CHECK(to_hex(d) == @@ -122,12 +119,12 @@ TEST_CASE("compression", "[config][compression]") { d = data2; session::config::compress_message(d, 19); - CHECK(d[0] == 'z'); - CHECK(d.size() == 157); // Yeah, it actually gets *bigger* with supposedly "higher" compression + CHECK(d[0] == std::byte{'z'}); + CHECK(d.size() == 156); CHECK(d.size() < data2.size()); CHECK(to_hex(d) == - "7a28b52ffd20aa9d0400e40764313a23693165313a2664313a6e31323a4b616c6c6965313a7032393a68" - "7474703a2f2f6b2e6578616d706c652e6f72672f4b626d70313a71323473656372657465313a3c6c6c69" - "306533323aea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c96564653d6431" - "3a6e303a313a7071303a6565070028812c55282f03fceac460149b57cd509a"); + "7a28b52ffd20aa95040022881f1f907d9c93291a7627219a79d06bb82c3c69341b104115dbf3c0860176" + "f63013ff7ba4247de211d1275be493fffff6eb7892db81b9dc9da26f40955e5d868586cd577bb69e00f7" + "caf2110f04219f7cf49bda3f19a5f4091966d5c199a3f14132c4d26f7cc7e14914edbca3903ef91e0862" + "955712d1275be1939f78844fb606008c12e0cb50be3a1c18c5e655339426"); } diff --git a/tests/test_config_contacts.cpp b/tests/test_config_contacts.cpp index e7bb0ae5a..cb123b35c 100644 --- a/tests/test_config_contacts.cpp +++ b/tests/test_config_contacts.cpp @@ -2,12 +2,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include @@ -16,15 +16,13 @@ static constexpr int64_t created_ts = 1680064059; +using namespace session; + TEST_CASE("Contacts", "[config][contacts]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -33,7 +31,7 @@ TEST_CASE("Contacts", "[config][contacts]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Contacts contacts{std::span{seed}, std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; constexpr auto definitely_real_id = "050000000000000000000000000000000000000000000000000000000000000000"sv; @@ -58,6 +56,8 @@ TEST_CASE("Contacts", "[config][contacts]") { CHECK_FALSE(c.profile_picture); CHECK(c.created == 0); CHECK(c.notifications == session::config::notify_mode::defaulted); + CHECK(c.delete_before == std::chrono::sys_seconds{}); + CHECK(c.delete_attach_before == std::chrono::sys_seconds{}); CHECK(c.mute_until == 0); CHECK_FALSE(contacts.needs_push()); @@ -72,6 +72,8 @@ TEST_CASE("Contacts", "[config][contacts]") { c.created = created_ts * 1'000; c.notifications = session::config::notify_mode::all; c.mute_until = (now + 1800) * 1'000'000; + c.delete_before = std::chrono::sys_seconds{std::chrono::seconds{now - 100}}; + c.delete_attach_before = std::chrono::sys_seconds{std::chrono::seconds{now - 50}}; contacts.set(c); @@ -84,6 +86,10 @@ TEST_CASE("Contacts", "[config][contacts]") { CHECK(contacts.get(definitely_real_id)->approved_me); CHECK_FALSE(contacts.get(definitely_real_id)->profile_picture); CHECK_FALSE(contacts.get(definitely_real_id)->blocked); + CHECK(contacts.get(definitely_real_id)->delete_before.time_since_epoch() == + std::chrono::seconds{now - 100}); + CHECK(contacts.get(definitely_real_id)->delete_attach_before.time_since_epoch() == + std::chrono::seconds{now - 50}); CHECK(contacts.get(definitely_real_id)->session_id == definitely_real_id); CHECK(contacts.needs_push()); @@ -133,7 +139,7 @@ TEST_CASE("Contacts", "[config][contacts]") { CHECK(seqno == 2); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash2", to_push[0]); contacts.merge(merge_configs); contacts2.confirm_pushed(seqno, {"fakehash2"}); @@ -177,7 +183,8 @@ TEST_CASE("Contacts", "[config][contacts]") { session::config::profile_pic p; { // These don't stay alive, so we use set_key/set_url to make a local copy: - std::vector key = "qwerty78901234567890123456789012"_bytes; + constexpr auto k = "qwerty78901234567890123456789012"_bytes; + std::vector key(k.begin(), k.end()); std::string url = "http://example.com/huge.bmp"; p.set_key(std::move(key)); p.url = std::move(url); @@ -266,13 +273,9 @@ TEST_CASE("Contacts", "[config][contacts]") { } TEST_CASE("Contacts (C API)", "[config][contacts][c]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -282,7 +285,7 @@ TEST_CASE("Contacts (C API)", "[config][contacts][c]") { oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); config_object* conf; - REQUIRE(0 == contacts_init(&conf, ed_sk.data(), NULL, 0, NULL)); + REQUIRE(0 == contacts_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, NULL)); const char* const definitely_real_id = "050000000000000000000000000000000000000000000000000000000000000000"; @@ -329,7 +332,7 @@ TEST_CASE("Contacts (C API)", "[config][contacts][c]") { CHECK(to_push->seqno == 1); config_object* conf2; - REQUIRE(contacts_init(&conf2, ed_sk.data(), NULL, 0, NULL) == 0); + REQUIRE(contacts_init(&conf2, to_unsigned(ed_sk.data()), NULL, 0, NULL) == 0); const char* merge_hash[1]; const unsigned char* merge_data[1]; @@ -442,20 +445,16 @@ TEST_CASE("huge contacts compression", "[config][compression][contacts]") { // Test that we can produce a config message whose *uncompressed* length exceeds the maximum // message length as long as its *compressed* length does not. - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - session::config::Contacts contacts{std::span{seed}, std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; for (uint16_t i = 0; i < 12000; i++) { char buf[2]; @@ -492,20 +491,16 @@ TEST_CASE("huger contacts with multipart messages", "[config][multipart][contact // Test that we can produce a config message whose *uncompressed* length exceeds the maximum // message length as long as its *compressed* length does not. - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - session::config::Contacts contacts{session::to_span(seed), std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; std::string friend42; @@ -514,8 +509,8 @@ TEST_CASE("huger contacts with multipart messages", "[config][multipart][contact // are randomly generated and thus not usefully compressible, which results in a much larger // (compressed) config. std::mt19937_64 rng{i}; - std::array random_sessionid; - random_sessionid[0] = 0x05; + b33 random_sessionid; + random_sessionid[0] = std::byte{0x05}; for (int i = 1; i < 33; i += 8) oxenc::write_host_as_little(rng(), random_sessionid.data() + i); @@ -575,9 +570,9 @@ TEST_CASE("huger contacts with multipart messages", "[config][multipart][contact dump = contacts.dump(); CHECK(dump.size() == base_dump_size + 12 * 13); // 12 x "10:fakehashNN" - auto c2 = std::make_unique(session::to_span(seed), std::nullopt); + auto c2 = std::make_unique(seed, std::nullopt); - std::vector>> merge_configs, merge_more; + std::vector>> merge_configs, merge_more; bool dump_load_in_between = false; std::mt19937_64 rng{12345}; @@ -656,8 +651,7 @@ TEST_CASE("huger contacts with multipart messages", "[config][multipart][contact CHECK(dump.size() < total_dumps + 500 /* ~ various other dump overhead */); if (dump_load_in_between) { - auto c2b = - std::make_unique(session::to_span(seed), c2->dump()); + auto c2b = std::make_unique(seed, c2->dump()); CHECK_FALSE(c2b->needs_dump()); c2 = std::move(c2b); CHECK_FALSE(c2->needs_dump()); @@ -687,37 +681,28 @@ TEST_CASE("huger contacts with multipart messages", "[config][multipart][contact TEST_CASE("multipart message expiry", "[config][multipart][contacts][expiry]") { // Tests that stored multipart message expires as expected. - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - session::config::Contacts contacts{session::to_span(seed), std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; std::string friend42; - std::array seedi = {0}; + b32 seedi = {}; for (uint16_t i = 0; i < 2000; i++) { // Unlike the above case where we have nearly identical Session IDs, here our session IDs // are randomly generated from fixed seeds and thus not usefully compressible, which results // in a much larger (compressed) config. - seedi[0] = i % 256; - seedi[1] = i >> 8; - std::array i_ed_pk, i_curve_pk; - std::array i_ed_sk; - crypto_sign_ed25519_seed_keypair( - i_ed_pk.data(), - i_ed_sk.data(), - reinterpret_cast(seedi.data())); - rc = crypto_sign_ed25519_pk_to_curve25519(i_curve_pk.data(), i_ed_pk.data()); + seedi[0] = static_cast(i % 256); + seedi[1] = static_cast(i >> 8); + auto [i_ed_pk, i_ed_sk] = ed25519::keypair(seedi); + auto i_curve_pk = ed25519::pk_to_x25519(i_ed_pk); std::string session_id = "05" + oxenc::to_hex(i_curve_pk.begin(), i_curve_pk.end()); auto c = contacts.get_or_construct(session_id); @@ -742,7 +727,7 @@ TEST_CASE("multipart message expiry", "[config][multipart][contacts][expiry]") { contacts.confirm_pushed(seqno, {"fakehash0", "fakehash1"}); - auto c2 = std::make_unique(session::to_span(seed), std::nullopt); + auto c2 = std::make_unique(seed, std::nullopt); c2->MULTIPART_MAX_WAIT = 200ms; c2->MULTIPART_MAX_REMEMBER = 600ms; @@ -750,7 +735,7 @@ TEST_CASE("multipart message expiry", "[config][multipart][contacts][expiry]") { auto old_seqno = std::get(c2->push()); REQUIRE(old_seqno == 0); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash0", to_push[0]); std::unordered_set accepted; @@ -856,9 +841,9 @@ TEST_CASE("multipart message expiry", "[config][multipart][contacts][expiry]") { TEST_CASE("needs_dump bug", "[config][needs_dump]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; - session::config::Contacts contacts{std::span{seed}, std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; CHECK_FALSE(contacts.needs_dump()); @@ -890,13 +875,9 @@ TEST_CASE("needs_dump bug", "[config][needs_dump]") { TEST_CASE("Contacts", "[config][blinded_contacts]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -905,7 +886,7 @@ TEST_CASE("Contacts", "[config][blinded_contacts]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Contacts contacts{std::span{seed}, std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; constexpr auto definitely_real_id = "150000000000000000000000000000000000000000000000000000000000000000"sv; @@ -993,7 +974,7 @@ TEST_CASE("Contacts", "[config][blinded_contacts]") { CHECK(seqno == 2); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash2", to_push[0]); contacts.merge(merge_configs); contacts2.confirm_pushed(seqno, {"fakehash2"}); @@ -1039,7 +1020,7 @@ TEST_CASE("Contacts", "[config][blinded_contacts]") { session::config::profile_pic p; { // These don't stay alive, so we use set_key/set_url to make a local copy: - std::vector key = "qwerty78901234567890123456789012"_bytes; + auto key = to_vector("qwerty78901234567890123456789012"_bytes); std::string url = "http://example.com/huge.bmp"; p.set_key(std::move(key)); p.url = std::move(url); @@ -1120,9 +1101,9 @@ TEST_CASE("Contacts", "[config][blinded_contacts]") { TEST_CASE("Contacts Pro Storage", "[config][contacts][pro]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; - session::config::Contacts contacts{std::span{seed}, std::nullopt}; + session::config::Contacts contacts{seed, std::nullopt}; REQUIRE(contacts.is_clean()); @@ -1130,50 +1111,50 @@ TEST_CASE("Contacts Pro Storage", "[config][contacts][pro]") { { auto c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000000"sv); - CHECK(c.profile_bitset.data == 0); + CHECK(c.profile_flags == ProProfileFlags::None); - c.profile_bitset.set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE); + c.profile_flags |= ProProfileFlags::ProBadge; contacts.set(c); CHECK(contacts.is_dirty()); c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000000"sv); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); + CHECK(contains(c.profile_flags, ProProfileFlags::ProBadge)); contacts.set(c); c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000000"sv); - CHECK_FALSE(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK_FALSE(contains(c.profile_flags, ProProfileFlags::AnimatedAvatar)); - c.profile_bitset.set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR); + c.profile_flags |= ProProfileFlags::AnimatedAvatar; contacts.set(c); c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000000"sv); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK(contains(c.profile_flags, ProProfileFlags::ProBadge)); + CHECK(contains(c.profile_flags, ProProfileFlags::AnimatedAvatar)); } // Set and unset the bitset from the profile on a new contact { auto c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000001"sv); - CHECK(c.profile_bitset.data == 0); + CHECK(c.profile_flags == ProProfileFlags::None); - c.profile_bitset.set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE); + c.profile_flags |= ProProfileFlags::ProBadge; contacts.set(c); c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000001"sv); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); + CHECK(contains(c.profile_flags, ProProfileFlags::ProBadge)); - c.profile_bitset.unset(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE); + c.profile_flags &= ~ProProfileFlags::ProBadge; contacts.set(c); c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000001"sv); - CHECK(!c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); + CHECK(!contains(c.profile_flags, ProProfileFlags::ProBadge)); } CHECK(contacts.needs_push()); @@ -1191,8 +1172,8 @@ TEST_CASE("Contacts Pro Storage", "[config][contacts][pro]") { { auto c = contacts.get_or_construct( "050000000000000000000000000000000000000000000000000000000000000000"sv); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - CHECK(c.profile_bitset.is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK(contains(c.profile_flags, ProProfileFlags::ProBadge)); + CHECK(contains(c.profile_flags, ProProfileFlags::AnimatedAvatar)); // This previously exposed a bug in set_erase_impl where the contact was being dirtied even // when nothing was actually being removed. contacts.set(c); diff --git a/tests/test_config_convo_info_volatile.cpp b/tests/test_config_convo_info_volatile.cpp index 207ea395f..3307359dd 100644 --- a/tests/test_config_convo_info_volatile.cpp +++ b/tests/test_config_convo_info_volatile.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -13,13 +12,9 @@ TEST_CASE("Conversations", "[config][conversations]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -28,7 +23,7 @@ TEST_CASE("Conversations", "[config][conversations]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::ConvoInfoVolatile convos{std::span{seed}, std::nullopt}; + session::config::ConvoInfoVolatile convos{seed, std::nullopt}; constexpr auto definitely_real_id = "055000000000000000000000000000000000000000000000000000000000000000"sv; @@ -72,7 +67,7 @@ TEST_CASE("Conversations", "[config][conversations]") { CHECK(convos.needs_dump()); const auto community_pubkey = - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hexbytes; + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; auto og = convos.get_or_construct_community( "http://Example.ORG:5678", "SudokuRoom", community_pubkey); @@ -190,7 +185,7 @@ TEST_CASE("Conversations", "[config][conversations]") { CHECK(seqno == 2); REQUIRE(to_push.size() == 1); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("hash2", to_push[0]); convos.merge(merge_configs); convos2.confirm_pushed(seqno, {"hash2"}); @@ -296,13 +291,9 @@ TEST_CASE("Conversations", "[config][conversations]") { } TEST_CASE("Conversations (C API)", "[config][conversations][c]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -312,7 +303,7 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); config_object* conf; - REQUIRE(0 == convo_info_volatile_init(&conf, ed_sk.data(), NULL, 0, NULL)); + REQUIRE(0 == convo_info_volatile_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, NULL)); const char* const definitely_real_id = "055000000000000000000000000000000000000000000000000000000000000000"; @@ -354,7 +345,7 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { CHECK(config_needs_dump(conf)); const auto community_pubkey = - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hexbytes; + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; convo_info_volatile_community og; @@ -363,18 +354,22 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { &og, "bad-url", "room", - "0000000000000000000000000000000000000000000000000000000000000000"_hexbytes.data())); + "0000000000000000000000000000000000000000000000000000000000000000"_hex_u.data())); CHECK(conf->last_error == "Invalid URL: invalid/missing protocol://"sv); CHECK_FALSE(convo_info_volatile_get_or_construct_community( conf, &og, "https://example.com", "bad room name", - "0000000000000000000000000000000000000000000000000000000000000000"_hexbytes.data())); + "0000000000000000000000000000000000000000000000000000000000000000"_hex_u.data())); CHECK(conf->last_error == "Invalid community URL: room token contains invalid characters"sv); CHECK(convo_info_volatile_get_or_construct_community( - conf, &og, "http://Example.ORG:5678", "SudokuRoom", community_pubkey.data())); + conf, + &og, + "http://Example.ORG:5678", + "SudokuRoom", + to_unsigned(community_pubkey.data()))); CHECK(conf->last_error == nullptr); CHECK(og.base_url == "http://example.org:5678"sv); // Note: lower-case CHECK(og.room == "sudokuroom"sv); // Note: lower-case @@ -414,7 +409,7 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { config_dump(conf, &dump, &dumplen); config_object* conf2; - REQUIRE(convo_info_volatile_init(&conf2, ed_sk.data(), dump, dumplen, NULL) == 0); + REQUIRE(convo_info_volatile_init(&conf2, to_unsigned(ed_sk.data()), dump, dumplen, NULL) == 0); free(dump); CHECK_FALSE(config_needs_push(conf2)); @@ -496,19 +491,14 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { } convo_info_volatile_iterator_free(it); - CHECK(seen == std::vector{ - "1-to-1: " - "051111111111111111111111111111111111111111111111111111111111111111", - "1-to-1: " - "055000000000000000000000000000000000000000000000000000000000000000", - "comm: http://example.org:5678/r/sudokuroom", - "lgr: " - "05cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "b: " - "150000000000000000000000000000000000101010111010000110100001210000", - "b: " - "2512345cccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - "c"}); + CHECK(seen == + std::vector{ + "1-to-1: 051111111111111111111111111111111111111111111111111111111111111111", + "1-to-1: 055000000000000000000000000000000000000000000000000000000000000000", + "comm: http://example.org:5678/r/sudokuroom", + "lgr: 05cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "b: 150000000000000000000000000000000000101010111010000110100001210000", + "b: 2512345ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}); } CHECK_FALSE(config_needs_push(conf)); @@ -581,13 +571,9 @@ TEST_CASE("Conversations (C API)", "[config][conversations][c]") { TEST_CASE("Conversation pruning", "[config][conversations][pruning]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -596,12 +582,12 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::ConvoInfoVolatile convos{std::span{seed}, std::nullopt}; + session::config::ConvoInfoVolatile convos{seed, std::nullopt}; - auto some_pubkey = [](unsigned char x) -> std::vector { - std::vector s = - "0000000000000000000000000000000000000000000000000000000000000000"_hexbytes; - s[31] = x; + auto some_pubkey = [](unsigned char x) -> std::vector { + std::vector s; + s.resize(32); + s[31] = static_cast(x); return s; }; auto some_session_id = [&](unsigned char x) -> std::string { @@ -627,9 +613,8 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") { std::chrono::sys_seconds{std::chrono::duration_cast( std::chrono::milliseconds{unix_timestamp(i)})}; - session::array_uc32 hash{}; - std::fill(hash.begin(), hash.end(), static_cast(i % 256)); - c.pro_revocation_tag = hash; + auto& hash = c.pro_revocation_tag.emplace(); + std::fill(hash.begin(), hash.end(), static_cast(i % 256)); } convos.set(c); @@ -700,13 +685,9 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") { TEST_CASE("Conversation dump/load state bug", "[config][conversations][dump-load]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -716,7 +697,7 @@ TEST_CASE("Conversation dump/load state bug", "[config][conversations][dump-load oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); config_object* conf; - REQUIRE(0 == convo_info_volatile_init(&conf, ed_sk.data(), NULL, 0, NULL)); + REQUIRE(0 == convo_info_volatile_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, NULL)); convo_info_volatile_1to1 c; CHECK(convo_info_volatile_get_or_construct_1to1( @@ -745,7 +726,7 @@ TEST_CASE("Conversation dump/load state bug", "[config][conversations][dump-load // Load the dump: config_object* conf2; - REQUIRE(0 == convo_info_volatile_init(&conf2, ed_sk.data(), dump, dumplen, NULL)); + REQUIRE(0 == convo_info_volatile_init(&conf2, to_unsigned(ed_sk.data()), dump, dumplen, NULL)); free(dump); @@ -811,13 +792,9 @@ TEST_CASE("Conversation dump/load state bug", "[config][conversations][dump-load TEST_CASE("Conversation pro data", "[config][conversations][pro]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -827,7 +804,7 @@ TEST_CASE("Conversation pro data", "[config][conversations][pro]") { oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); config_object* conf; - REQUIRE(0 == convo_info_volatile_init(&conf, ed_sk.data(), NULL, 0, NULL)); + REQUIRE(0 == convo_info_volatile_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, NULL)); convo_info_volatile_1to1 c; CHECK(convo_info_volatile_get_or_construct_1to1( @@ -837,9 +814,10 @@ TEST_CASE("Conversation pro data", "[config][conversations][pro]") { .count(); c.pro_expiry_ts = 10000; - session::array_uc32 hash{}; - std::fill(hash.begin(), hash.end(), static_cast(3)); - std::memcpy(c.pro_revocation_tag.data, hash.data(), hash.size()); + std::fill( + c.pro_revocation_tag.data, + c.pro_revocation_tag.data + 32, + static_cast(3)); c.has_pro_revocation_tag = true; convo_info_volatile_set_1to1(conf, &c); @@ -862,7 +840,7 @@ TEST_CASE("Conversation pro data", "[config][conversations][pro]") { // Load the dump: config_object* conf2; - REQUIRE(0 == convo_info_volatile_init(&conf2, ed_sk.data(), dump, dumplen, NULL)); + REQUIRE(0 == convo_info_volatile_init(&conf2, to_unsigned(ed_sk.data()), dump, dumplen, NULL)); free(dump); @@ -874,4 +852,4 @@ TEST_CASE("Conversation pro data", "[config][conversations][pro]") { CHECK(c.has_pro_revocation_tag); CHECK(c2.has_pro_revocation_tag); CHECK(oxenc::to_hex(c2.pro_revocation_tag.data) == oxenc::to_hex(c.pro_revocation_tag.data)); -} \ No newline at end of file +} diff --git a/tests/test_config_local.cpp b/tests/test_config_local.cpp index fc3e29392..86cde79b9 100644 --- a/tests/test_config_local.cpp +++ b/tests/test_config_local.cpp @@ -1,29 +1,26 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include "utils.hpp" +using namespace session; using namespace std::literals; TEST_CASE("Local", "[config][local]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -32,7 +29,7 @@ TEST_CASE("Local", "[config][local]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::Local local{std::span{seed}, std::nullopt}; + session::config::Local local{seed, std::nullopt}; CHECK(local.get_notification_content() == session::config::notify_content::defaulted); CHECK(local.get_ios_notification_sound() == 0); @@ -60,7 +57,7 @@ TEST_CASE("Local", "[config][local]") { CHECK(local.size_settings() == 1); // Ensure all of these settings were stored in the dump and loaded correctly - session::config::Local local2{std::span{seed}, local.dump()}; + session::config::Local local2{seed, local.dump()}; CHECK_FALSE(local.needs_dump()); CHECK(local2.get_notification_content() == session::config::notify_content::name_no_preview); diff --git a/tests/test_config_pro.cpp b/tests/test_config_pro.cpp index 04074d7cc..d07d236dd 100644 --- a/tests/test_config_pro.cpp +++ b/tests/test_config_pro.cpp @@ -6,16 +6,15 @@ #include #include #include + +#include "session/crypto/ed25519.hpp" +#include "utils.hpp" using namespace oxenc::literals; TEST_CASE("Pro", "[config][pro]") { // Setup keys - std::array rotating_pk, signing_pk; - session::cleared_uc64 rotating_sk, signing_sk; - { - crypto_sign_ed25519_keypair(rotating_pk.data(), rotating_sk.data()); - crypto_sign_ed25519_keypair(signing_pk.data(), signing_sk.data()); - } + auto [rotating_pk, rotating_sk] = ed25519::keypair(); + auto [signing_pk, signing_sk] = ed25519::keypair(); // Setup the Pro data structure session::config::ProConfig pro_cpp = {}; @@ -38,29 +37,15 @@ TEST_CASE("Pro", "[config][pro]") { std::memcpy(pro.proof.revocation_tag.data, revocation_tag.data(), revocation_tag.size()); } - // Sign the proof with the faux pro backend key (Ed25519 over the message directly). The C and - // C++ proof representations above mirror each other, so a single message signs both. + // Sign the proof with the faux pro backend key { + // Sign the proof with the faux pro backend key (Ed25519 over the message directly). The C + // and C++ proof representations mirror each other, so a single message signs both. static_assert(crypto_sign_ed25519_BYTES == pro_cpp.proof.sig.max_size()); auto msg_to_sign = pro_cpp.proof.signed_message(); - // Write the signature into the C++ proof - int sig_result = crypto_sign_ed25519_detached( - pro_cpp.proof.sig.data(), - nullptr, - msg_to_sign.data(), - msg_to_sign.size(), - signing_sk.data()); - CHECK(sig_result == 0); - - // ... and into the C proof - sig_result = crypto_sign_ed25519_detached( - pro.proof.sig.data, - nullptr, - msg_to_sign.data(), - msg_to_sign.size(), - signing_sk.data()); - CHECK(sig_result == 0); + ed25519::sign(pro_cpp.proof.sig, signing_sk, msg_to_sign); + ed25519::sign(to_byte_span(pro.proof.sig.data), signing_sk, msg_to_sign); } // Verify expiry @@ -75,20 +60,13 @@ TEST_CASE("Pro", "[config][pro]") { // Verify it can verify messages signed with the rotating public key { std::string_view body = "hello world"; - std::array sig = {}; - int sign_result = crypto_sign_ed25519_detached( - sig.data(), - nullptr, - reinterpret_cast(body.data()), - body.size(), - rotating_sk.data()); - CHECK(sign_result == 0); - CHECK(pro_cpp.proof.verify_message(sig, session::to_span(body))); + auto sig = ed25519::sign(rotating_sk, to_span(body)); + CHECK(pro_cpp.proof.verify_message(sig, to_span(body))); CHECK(session_protocol_pro_proof_verify_message( &pro.proof, - sig.data(), + to_unsigned(sig.data()), sig.size(), - reinterpret_cast(body.data()), + reinterpret_cast(body.data()), body.size())); } diff --git a/tests/test_config_user_groups.cpp b/tests/test_config_user_groups.cpp index dfb5ce339..3f2c3c278 100644 --- a/tests/test_config_user_groups.cpp +++ b/tests/test_config_user_groups.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -79,13 +78,9 @@ TEST_CASE("Open Group URLs", "[config][community_urls]") { TEST_CASE("User Groups", "[config][groups]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -94,7 +89,7 @@ TEST_CASE("User Groups", "[config][groups]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::UserGroups groups{std::span{seed}, std::nullopt}; + session::config::UserGroups groups{seed, std::nullopt}; constexpr auto definitely_real_id = "055000000000000000000000000000000000000000000000000000000000000000"sv; @@ -159,11 +154,8 @@ TEST_CASE("User Groups", "[config][groups]") { CHECK(c.members() == expected_members); const auto lgroup_seed = - "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hexbytes; - std::array lg_pk; - std::array lg_sk; - crypto_sign_ed25519_seed_keypair( - lg_pk.data(), lg_sk.data(), reinterpret_cast(lgroup_seed.data())); + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hex_b; + auto [lg_pk, lg_sk] = ed25519::keypair(lgroup_seed); // Note: this isn't exactly what Session actually does here for legacy groups (rather it // uses X25519 keys) but for this test the distinction doesn't matter. c.enc_pubkey.assign(lg_pk.data(), lg_pk.data() + lg_pk.size()); @@ -182,7 +174,7 @@ TEST_CASE("User Groups", "[config][groups]") { CHECK(groups.needs_dump()); const auto community_pubkey = - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hexbytes; + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; auto og = groups.get_or_construct_community( "http://Example.ORG:5678", "SudokuRoom", community_pubkey); @@ -310,7 +302,7 @@ TEST_CASE("User Groups", "[config][groups]") { CHECK_FALSE(g2.needs_dump()); REQUIRE(to_push.size() == 1); - std::vector>> to_merge; + std::vector>> to_merge; to_merge.emplace_back("fakehash2", to_push[0]); groups.merge(to_merge); auto x3 = groups.get_community("http://example.org:5678", "SudokuRoom"); @@ -427,13 +419,9 @@ TEST_CASE("User Groups", "[config][groups]") { TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -442,7 +430,7 @@ TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::UserGroups groups{std::span{seed}, std::nullopt}; + session::config::UserGroups groups{seed, std::nullopt}; constexpr auto definitely_real_id = "035000000000000000000000000000000000000000000000000000000000000000"sv; @@ -463,10 +451,10 @@ TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { c.secretkey = session::to_vector(ed_sk); // This *isn't* the right secret key for the group, so // won't propagate, and so auth data will: - c.auth_data = + c.auth_data = to_vector( "01020304050000000000000000000000000000000000000000000000000000000000000000000000000000" "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - "0000000000000000000000000000"_hexbytes; + "0000000000000000000000000000"_hex_b); groups.set(c); @@ -478,7 +466,7 @@ TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { auto d1 = groups.dump(); - session::config::UserGroups g2{std::span{seed}, d1}; + session::config::UserGroups g2{seed, d1}; auto c2 = g2.get_group(definitely_real_id); REQUIRE(c2.has_value()); @@ -504,16 +492,18 @@ TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { c2b.secretkey = session::to_vector(ed_sk); // This one does match the group ID, so should propagate c2b.auth_data = // should get ignored, since we have a valid secret key set: - "01020304050000000000000000000000000000000000000000000000000000000000000000000000000000" - "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - "0000000000000000000000000000"_hexbytes; + to_vector( + "0102030405000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000" + "00000000"_hex_b); g2.set(c2b); std::tie(seqno, to_push, obs) = g2.push(); g2.confirm_pushed(seqno, {"fakehash2"}); REQUIRE(to_push.size() == 1); - std::vector>> to_merge; + std::vector>> to_merge; to_merge.emplace_back("fakehash2", to_push[0]); groups.merge(to_merge); @@ -582,13 +572,9 @@ TEST_CASE("User Groups -- (non-legacy) groups", "[config][groups][new]") { TEST_CASE("User Groups members C API", "[config][groups][c]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -599,7 +585,7 @@ TEST_CASE("User Groups members C API", "[config][groups][c]") { char err[256]; config_object* conf; - rc = user_groups_init(&conf, ed_sk.data(), NULL, 0, err); + auto rc = user_groups_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, err); REQUIRE(rc == 0); constexpr auto definitely_real_id = @@ -719,13 +705,11 @@ TEST_CASE("User Groups members C API", "[config][groups][c]") { REQUIRE(keys); REQUIRE(key_len == 1); - session::config::UserGroups c2{std::span{seed}, std::nullopt}; + session::config::UserGroups c2{seed, std::nullopt}; REQUIRE(to_push->n_configs == 1); - std::vector>> to_merge; - to_merge.emplace_back( - "fakehash1", - std::span{to_push->config[0], to_push->config_lens[0]}); + std::vector>> to_merge; + to_merge.emplace_back("fakehash1", to_byte_span(to_push->config[0], to_push->config_lens[0])); CHECK(c2.merge(to_merge) == std::unordered_set{{"fakehash1"}}); auto grp = c2.get_legacy_group(definitely_real_id); @@ -739,18 +723,14 @@ TEST_CASE("User groups empty member bug", "[config][groups][bug]") { // the config, even when the current members (or admin) list is empty. (This isn't strictly // specific to user groups, but that's where the bug is easily encountered). - const auto seed = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::UserGroups c{std::span{seed}, std::nullopt}; + session::config::UserGroups c{seed, std::nullopt}; CHECK_FALSE(c.needs_push()); @@ -825,18 +805,14 @@ TEST_CASE("User groups mute_until & joined_at are always seconds", "[config][gro // the config, even when the current members (or admin) list is empty. (This isn't strictly // specific to user groups, but that's where the bug is easily encountered). - const auto seed = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::UserGroups c{std::span{seed}, std::nullopt}; + session::config::UserGroups c{seed, std::nullopt}; CHECK_FALSE(c.needs_push()); @@ -872,7 +848,7 @@ TEST_CASE("User groups mute_until & joined_at are always seconds", "[config][gro { const auto community_pubkey = - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hexbytes; + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; const auto url = "http://example.org:5678"; const auto room = "sudoku_room"; auto comm = c.get_or_construct_community(url, room, community_pubkey); @@ -901,8 +877,8 @@ TEST_CASE("User groups mute_until & joined_at are always seconds", "[config][gro "3a03" "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef64313a21303a313a4b" "303a" - "313a6a303a65656565313a28303a313a296c6565"_hexbytes; - session::config::UserGroups c2{std::span{seed}, dump_with_not_seconds}; + "313a6a303a65656565313a28303a313a296c6565"_hex_b; + session::config::UserGroups c2{seed, dump_with_not_seconds}; auto gr = c2.get_or_construct_group( "031234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"); diff --git a/tests/test_config_userprofile.cpp b/tests/test_config_userprofile.cpp index 9133e38a8..6f13049e1 100644 --- a/tests/test_config_userprofile.cpp +++ b/tests/test_config_userprofile.cpp @@ -1,17 +1,19 @@ #include #include #include -#include #include #include #include #include +#include #include #include +#include "../src/config/internal.hpp" #include "utils.hpp" +using namespace session; using namespace std::literals; namespace { @@ -51,13 +53,9 @@ struct UserProfileTester { TEST_CASE("UserProfile", "[config][user_profile]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -66,7 +64,7 @@ TEST_CASE("UserProfile", "[config][user_profile]") { CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - session::config::UserProfile profile{std::span{seed}, std::nullopt}; + session::config::UserProfile profile{seed, std::nullopt}; CHECK_THROWS( profile.set_name("123456789012345678901234567890123456789012345678901234567890123456789" @@ -93,24 +91,19 @@ TEST_CASE("UserProfile", "[config][user_profile]") { TEST_CASE("user profile C API", "[config][user_profile][c]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); - int rc = crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data()); - REQUIRE(rc == 0); + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - CHECK(oxenc::to_hex(seed) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); // Initialize a brand new, empty config because we have no dump data to deal with. char err[256]; config_object* conf; - rc = user_profile_init(&conf, ed_sk.data(), NULL, 0, err); + int rc = user_profile_init(&conf, to_unsigned(ed_sk.data()), NULL, 0, err); REQUIRE(rc == 0); // We don't need to push anything, since this is an empty config @@ -171,10 +164,8 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { pic = user_profile_get_pic(conf); REQUIRE(pic.url != ""s); - REQUIRE(pic.key != session::to_vector("").data()); CHECK(pic.url == "http://example.org/omg-pic-123.bmp"sv); - CHECK(session::to_vector(std::span{pic.key, 32}) == - "secret78901234567890123456789012"_bytes); + CHECK(std::ranges::equal(to_byte_span(pic.key), "secret78901234567890123456789012"_bytes)); CHECK(user_profile_get_nts_priority(conf) == 9); @@ -188,7 +179,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { // between dumps; even though we changed two fields here). // The hash of a completely empty, initial seqno=0 message: - auto exp_hash0 = "ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hexbytes; + auto exp_hash0 = "ea173b57beca8af18c3519a7bbf69c3e7a05d1c049fa9558341d8ebb48b0c965"_hex_b; // The data to be actually pushed, expanded like this to make it somewhat human-readable: // clang-format off @@ -221,7 +212,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { "2d146da44915063a07a78556ab5eff4f67f6aa26211e8d330b53d28567a931028c393709a325425d" "e7486ccde24416a7fd4a8ba5fa73899c65f4276dfaddd5b2100adcf0f793104fb235b31ce32ec656" "056009a9ebf58d45d7d696b74e0c7ff0499c4d23204976f19561dc0dba6dc53a2497d28ce03498ea" - "49bf122762d7bc1d6d9c02f6d54f8384"_hexbytes; + "49bf122762d7bc1d6d9c02f6d54f8384"_hex_b; // Copy this out; we need to hold onto it to do the confirmation later on seqno_t seqno = to_push->seqno; @@ -285,7 +276,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { // Start with an empty config, as above: config_object* conf2; - REQUIRE(user_profile_init(&conf2, ed_sk.data(), NULL, 0, err) == 0); + REQUIRE(user_profile_init(&conf2, to_unsigned(ed_sk.data()), NULL, 0, err) == 0); CHECK_FALSE(config_needs_dump(conf2)); // Now imagine we just pulled down the encrypted string from the swarm; we merge it into conf2: @@ -293,7 +284,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { const char* merge_hash[1]; size_t merge_size[1]; merge_hash[0] = "fakehash1"; - merge_data[0] = exp_push1_encrypted.data(); + merge_data[0] = to_unsigned(exp_push1_encrypted.data()); merge_size[0] = exp_push1_encrypted.size(); config_string_list* accepted = config_merge(conf2, merge_hash, merge_data, merge_size, 1); REQUIRE(accepted->len == 1); @@ -329,6 +320,33 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { user_profile_set_nts_expiry(conf2, 86400); CHECK(user_profile_get_nts_expiry(conf2) == 86400); + // Note-to-self's own delete-before pair: it needs them here because it has no contacts entry to + // carry them, unlike every other conversation. + CHECK(user_profile_get_nts_delete_before(conf2) == 0); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 0); + user_profile_set_nts_delete_before(conf2, 1700000000); + user_profile_set_nts_delete_attach_before(conf2, 1700000500); + CHECK(user_profile_get_nts_delete_before(conf2) == 1700000000); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 1700000500); + // Zero clears, rather than meaning "the epoch". + user_profile_set_nts_delete_before(conf2, 0); + CHECK(user_profile_get_nts_delete_before(conf2) == 0); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 1700000500); + + // Deleting the messages takes their attachments too, so an attachment instruction the message + // one already covers is dropped rather than kept saying nothing. + user_profile_set_nts_delete_before(conf2, 1700000500); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 0); + + // And from the other side: one that arrives already covered is not recorded at all. + user_profile_set_nts_delete_attach_before(conf2, 1700000400); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 0); + + // A later one still means something, and is kept. + user_profile_set_nts_delete_attach_before(conf2, 1700000900); + CHECK(user_profile_get_nts_delete_attach_before(conf2) == 1700000900); + CHECK(user_profile_get_nts_delete_before(conf2) == 1700000500); + CHECK(user_profile_get_blinded_msgreqs(conf2) == -1); user_profile_set_blinded_msgreqs(conf2, 0); CHECK(user_profile_get_blinded_msgreqs(conf2) == 0); @@ -433,7 +451,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { #else REQUIRE(pic.key != nullptr); #endif - CHECK(oxenc::to_hex(std::span{pic.key, 32}) == + CHECK(oxenc::to_hex(to_byte_span(pic.key)) == "7177657274007975696f31323334353637383930313233343536373839303132"); pic = user_profile_get_pic(conf2); #if defined(__APPLE__) || defined(__clang__) || defined(__llvm__) @@ -447,7 +465,7 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { #else REQUIRE(pic.key != nullptr); #endif - CHECK(oxenc::to_hex(std::span{pic.key, 32}) == + CHECK(oxenc::to_hex(to_byte_span(pic.key)) == "7177657274007975696f31323334353637383930313233343536373839303132"); CHECK(user_profile_get_nts_priority(conf) == 9); @@ -474,10 +492,9 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { // Check the current pic pic = user_profile_get_pic(conf); REQUIRE(pic.url != ""s); - REQUIRE(pic.key != session::to_vector("").data()); + CHECK(pic.url == "http://new.example.com/pic"sv); - CHECK(session::to_vector(std::span{pic.key, 32}) == - "qwert\0yuio1234567890123456789012"_bytes); + CHECK(std::ranges::equal(to_byte_span(pic.key), "qwert\0yuio1234567890123456789012"_bytes)); // Reupload the "current" pic and confirm it gets returned strcpy(p.url, "testUrl"); @@ -486,10 +503,9 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { pic = user_profile_get_pic(conf); REQUIRE(pic.url != ""s); - REQUIRE(pic.key != session::to_vector("").data()); + CHECK(pic.url == "testUrl"sv); - CHECK(session::to_vector(std::span{pic.key, 32}) == - "secret78901234567890123456789000"_bytes); + CHECK(std::ranges::equal(to_byte_span(pic.key), "secret78901234567890123456789000"_bytes)); // Upload a "new" pic and it now gets returned strcpy(p.url, "testNewUrl"); @@ -497,10 +513,9 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { CHECK(0 == user_profile_set_pic(conf, p)); pic = user_profile_get_pic(conf); REQUIRE(pic.url != ""s); - REQUIRE(pic.key != session::to_vector("").data()); + CHECK(pic.url == "testNewUrl"sv); - CHECK(session::to_vector(std::span{pic.key, 32}) == - "secret78901234567890123456789111"_bytes); + CHECK(std::ranges::equal(to_byte_span(pic.key), "secret78901234567890123456789111"_bytes)); // Ensure the timestamp for the last modified pic gets updated correctly when the name gets set UserProfileTester::set_profile_updated(conf, std::chrono::sys_seconds{0s}); @@ -554,15 +569,57 @@ TEST_CASE("user profile C API", "[config][user_profile][c]") { CHECK((raw_value >= before_seconds && raw_value <= after_seconds)); } +TEST_CASE("UserProfile media-saved notifications", "[config][user_profile]") { + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + session::config::UserProfile profile{seed, std::nullopt}; + + // The default is to tell them, and it costs nothing to carry: an account that has never touched + // this has no key for it, which is the whole reason the key is stored the other way up. + CHECK(profile.get_notify_media_saved()); + CHECK_FALSE(profile.needs_push()); + + profile.set_notify_media_saved(false); + CHECK_FALSE(profile.get_notify_media_saved()); + CHECK(profile.needs_push()); + + // Setting it back removes the key rather than storing a 0. + profile.set_notify_media_saved(true); + CHECK(profile.get_notify_media_saved()); + + // Whether the key is really gone is not something the accessor can tell us, since absent and + // false answer the same; a config built from the dump carries only what was stored. + auto [seqno, push, obs] = profile.push(); + profile.confirm_pushed(seqno, {"fakehash"}); + session::config::UserProfile reloaded{seed, profile.make_dump()}; + CHECK(reloaded.get_notify_media_saved()); + reloaded.set_notify_media_saved(true); + CHECK_FALSE(reloaded.needs_push()); +} + +TEST_CASE("UserProfile media-saved does not age the profile", "[config][user_profile]") { + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + session::config::UserProfile profile{seed, std::nullopt}; + + profile.set_name("Leela"); + auto stamped = profile.get_profile_updated(); + REQUIRE(stamped > std::chrono::sys_seconds{}); + + // `t`/`T` say when the *profile* last changed, and a client uses that to decide whose name and + // picture win. This is not the profile, so advancing it would claim ours is fresher than it is + // and could make a stale name beat a newer one from another device. + profile.set_notify_media_saved(false); + CHECK(profile.get_profile_updated() == stamped); +} + TEST_CASE("user profile timestamp update bug", "[config][user_profile]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; - session::config::UserProfile profile{std::span{seed}, std::nullopt}; + session::config::UserProfile profile{seed, std::nullopt}; // Initially the code would update `profile_updated` even if the data hadn't changed, this test // verifies that no longer happens - std::vector key = "qwerty78901234567890123456789012"_bytes; + auto key = to_vector("qwerty78901234567890123456789012"_bytes); std::string url = "http://example.com/huge.bmp"; profile.set_name("Nibbler"); profile.set_blinded_msgreqs(true); @@ -582,52 +639,43 @@ TEST_CASE("user profile timestamp update bug", "[config][user_profile]") { TEST_CASE("UserProfile Pro Storage", "[config][user_profile][pro]") { - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; - session::config::UserProfile profile{std::span{seed}, std::nullopt}; + session::config::UserProfile profile{seed, std::nullopt}; // Ensure the bitset is being updated correctly - CHECK(profile.get_profile_bitset().data == 0); + CHECK(profile.get_profile_flags() == ProProfileFlags::None); profile.set_pro_badge(true); - CHECK(profile.get_profile_bitset().is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); + CHECK(contains(profile.get_profile_flags(), ProProfileFlags::ProBadge)); profile.set_pro_badge(false); - CHECK(profile.get_profile_bitset().data == 0); + CHECK(profile.get_profile_flags() == ProProfileFlags::None); profile.set_animated_avatar(true); - CHECK(profile.get_profile_bitset().is_set( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK(contains(profile.get_profile_flags(), ProProfileFlags::AnimatedAvatar)); profile.set_animated_avatar(false); - CHECK(profile.get_profile_bitset().data == 0); + CHECK(profile.get_profile_flags() == ProProfileFlags::None); profile.set_pro_badge(true); profile.set_animated_avatar(true); - CHECK(profile.get_profile_bitset().is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - CHECK(profile.get_profile_bitset().is_set( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK(contains(profile.get_profile_flags(), ProProfileFlags::ProBadge)); + CHECK(contains(profile.get_profile_flags(), ProProfileFlags::AnimatedAvatar)); profile.set_animated_avatar(false); - CHECK(profile.get_profile_bitset().is_set(SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - CHECK_FALSE(profile.get_profile_bitset().is_set( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + CHECK(contains(profile.get_profile_flags(), ProProfileFlags::ProBadge)); + CHECK_FALSE(contains(profile.get_profile_flags(), ProProfileFlags::AnimatedAvatar)); { - session::config::UserProfile profile2{std::span{seed}, profile.dump()}; - CHECK(profile2.get_profile_bitset().is_set( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - CHECK_FALSE(profile2.get_profile_bitset().is_set( - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_ANIMATED_AVATAR)); + session::config::UserProfile profile2{seed, profile.dump()}; + CHECK(contains(profile2.get_profile_flags(), ProProfileFlags::ProBadge)); + CHECK_FALSE(contains(profile2.get_profile_flags(), ProProfileFlags::AnimatedAvatar)); } // Ensure the pro config is being stored correctly - std::array rotating_pk, signing_pk; - session::cleared_uc64 rotating_sk, signing_sk; - { - crypto_sign_ed25519_keypair(rotating_pk.data(), rotating_sk.data()); - crypto_sign_ed25519_keypair(signing_pk.data(), signing_sk.data()); - } + auto [rotating_pk, rotating_sk] = ed25519::keypair(); + auto [signing_pk, signing_sk] = ed25519::keypair(); session::config::ProConfig pro_cpp = {}; pro_pro_config pro = {}; @@ -637,7 +685,7 @@ TEST_CASE("UserProfile Pro Storage", "[config][user_profile][pro]") { pro_cpp.proof.rotating_pubkey = rotating_pk; pro_cpp.proof.expiry_at = std::chrono::sys_seconds(1s); constexpr auto revocation_tag = - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_u; + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; static_assert(pro_cpp.proof.revocation_tag.max_size() == revocation_tag.size()); std::memcpy( pro_cpp.proof.revocation_tag.data(), revocation_tag.data(), revocation_tag.size()); @@ -657,7 +705,7 @@ TEST_CASE("UserProfile Pro Storage", "[config][user_profile][pro]") { CHECK(profile.get_profile_updated().time_since_epoch().count() != 123); { - session::config::UserProfile profile2{std::span{seed}, profile.dump()}; + session::config::UserProfile profile2{seed, profile.dump()}; CHECK(profile.get_pro_config() == pro_cpp); } @@ -739,7 +787,7 @@ TEST_CASE("UserProfile Pro Storage", "[config][user_profile][pro]") { { // Round-trips through a dump/reload. - session::config::UserProfile profile2{std::span{seed}, profile.dump()}; + session::config::UserProfile profile2{seed, profile.dump()}; CHECK(profile2.get_refund_requested() == refund_at); } @@ -791,7 +839,7 @@ TEST_CASE("UserProfile Pro Storage", "[config][user_profile][pro]") { // pro_renewal_target: centralised "when to renew" decision. { - session::config::UserProfile pr{std::span{seed}, std::nullopt}; + session::config::UserProfile pr{seed, std::nullopt}; // No proof and no purchase in flight -> not Pro, nothing to fetch. CHECK_FALSE(pr.pro_renewal_target(now).has_value()); diff --git a/tests/test_configdata.cpp b/tests/test_configdata.cpp index 24948943d..e0f602f5d 100644 --- a/tests/test_configdata.cpp +++ b/tests/test_configdata.cpp @@ -1,11 +1,11 @@ #include #include -#include -#include #include #include #include +#include +#include #include #include "session/bt_merge.hpp" @@ -98,6 +98,8 @@ TEST_CASE("config pruning", "[config][prune]") { }); } +namespace { + // shortcut to access a nested dict auto& d(config::dict_value& v) { return std::get(v); @@ -107,12 +109,7 @@ auto& s(config::dict_value& v) { return std::get(v); } -std::vector blake2b(std::span data) { - std::vector result; - result.resize(32); - crypto_generichash_blake2b(result.data(), 32, data.data(), data.size(), nullptr, 0); - return result; -} +} // namespace TEST_CASE("config diff", "[config][diff]") { MutableConfigMessage m; @@ -330,20 +327,15 @@ TEST_CASE("config message signature", "[config][signing]") { constexpr auto skey_hex = "79f530dbf3d81aecc04072933c1b3e3edc0b7d91f2dcc2f7756f2611886cca5f" "4384261cdd338f5820ca9cbbe3fc72ac8944ee60d3b795b797fbbf5597b09f17"sv; - std::array secretkey; + b64 secretkey; oxenc::from_hex(skey_hex.begin(), skey_hex.end(), secretkey.begin()); - auto signer = [&secretkey](std::span data) { - std::vector result; - result.resize(64); - crypto_sign_ed25519_detached( - result.data(), nullptr, data.data(), data.size(), secretkey.data()); - return result; + ed25519::PrivKeySpan sk{secretkey}; + auto signer = [&sk](std::span data) { + auto sig = ed25519::sign(sk, data); + return std::vector{sig.begin(), sig.end()}; }; - auto verifier = [&secretkey]( - std::span data, - std::span signature) { - return 0 == crypto_sign_verify_detached( - signature.data(), data.data(), data.size(), secretkey.data() + 32); + auto verifier = [&sk](std::span data, std::span signature) { + return signature.size() == 64 && ed25519::verify(signature.first<64>(), sk.pubkey(), data); }; m.signer = signer; @@ -377,15 +369,13 @@ TEST_CASE("config message signature", "[config][signing]") { auto expected_sig = "77267f4de7701ae348eba0ef73175281512ba3f1051cfed22dc3e31b9c699330" - "2938863e09bc8b33638161071bd8dc397d5c1d3f674120d08fbb9c64dde2e907"_hexbytes; - std::vector sig(64, '\0'); + "2938863e09bc8b33638161071bd8dc397d5c1d3f674120d08fbb9c64dde2e907"_hex_b; // Sign it ourselves, and check what we get: - crypto_sign_ed25519_detached( - sig.data(), nullptr, m_signing_value.data(), m_signing_value.size(), secretkey.data()); + auto sig = ed25519::sign(sk, m_signing_value); CHECK(to_hex(sig) == to_hex(expected_sig)); auto key_bytes = "1:~64:"_bytes; auto end_bytes = "e"_bytes; - auto m_expected = m_signing_value; + auto m_expected = to_vector(m_signing_value); m_expected.insert(m_expected.end(), key_bytes.begin(), key_bytes.end()); m_expected.insert(m_expected.end(), expected_sig.begin(), expected_sig.end()); m_expected.insert(m_expected.end(), end_bytes.begin(), end_bytes.end()); @@ -398,8 +388,8 @@ TEST_CASE("config message signature", "[config][signing]") { // Deliberately modify the signature to break it: auto m_broken = m_expected; - REQUIRE(m_broken[m_broken.size() - 2] == 0x07); - m_broken[m_broken.size() - 2] = 0x17; + REQUIRE(m_broken[m_broken.size() - 2] == std::byte{0x07}); + m_broken[m_broken.size() - 2] = std::byte{0x17}; using Catch::Matchers::Message; CHECK_THROWS_AS(ConfigMessage(m_broken, verifier), config::signature_error); @@ -426,7 +416,7 @@ TEST_CASE("config message signature", "[config][signing]") { config::config_error, Message("Config signature failed verification")); - auto m_unsigned = m_signing_value; + auto m_unsigned = to_vector(m_signing_value); m_unsigned.insert(m_unsigned.end(), end_bytes.begin(), end_bytes.end()); CHECK_THROWS_MATCHES( ConfigMessage(m_unsigned, verifier), @@ -445,10 +435,10 @@ const config::dict data118{ {"string2", "goodbye"}, }; -const auto h119 = "43094f68c1faa37eff79e1c2f3973ffd5f9d6423b00ccda306fc6e7dac5f0c44"_hexbytes; -const auto h120 = "e3a237f91014d31e4d30569c4a8bfcd72157804f99b8732c611c48bf126432b5"_hexbytes; -const auto h121 = "1a7f602055124deaf21175ef3f32983dee7c9de570e5d9c9a0bbc2db71dcb97f"_hexbytes; -const auto h122 = "46560604fe352101bb869435260d7100ccfe007be5f741c7e96303f02f394e8a"_hexbytes; +const auto h119 = "43094f68c1faa37eff79e1c2f3973ffd5f9d6423b00ccda306fc6e7dac5f0c44"_hex_b; +const auto h120 = "e3a237f91014d31e4d30569c4a8bfcd72157804f99b8732c611c48bf126432b5"_hex_b; +const auto h121 = "1a7f602055124deaf21175ef3f32983dee7c9de570e5d9c9a0bbc2db71dcb97f"_hex_b; +const auto h122 = "46560604fe352101bb869435260d7100ccfe007be5f741c7e96303f02f394e8a"_hex_b; const auto m123_expected = to_vector( // clang-format off "d" @@ -489,7 +479,7 @@ const auto m123_expected = to_vector( "e" "e"); // clang-format on -const auto h123 = "d9398c597b058ac7e28e3febb76ed68eb8c5b6c369610562ab5f2b596775d73c"_hexbytes; +const auto h123 = "d9398c597b058ac7e28e3febb76ed68eb8c5b6c369610562ab5f2b596775d73c"_hex_b; TEST_CASE("config message example 1", "[config][example]") { /// This is the "Ordinary update" example described in docs/api/docs/config-merge-logic.md @@ -556,7 +546,7 @@ TEST_CASE("config message example 1", "[config][example]") { CHECK(printable(m118.serialize()) == printable(m118_expected)); - CHECK(to_hex(m118.hash()) == to_hex(blake2b(m118_expected))); + CHECK(to_hex(m118.hash()) == to_hex(hash::blake2b<32>(m118_expected))); // Increment 5 times so that our diffs will be empty. auto m123 = m118.increment(); @@ -697,7 +687,7 @@ TEST_CASE("config message empty set/list deserialization", "[config][deserializa Message("Failed to parse config file: Data contains an unpruned, empty dict")); } -void updates_124(MutableConfigMessage& m) { +static void updates_124(MutableConfigMessage& m) { m.data()["dictA"] = config::dict{ {"hello", 123}, {"goodbye", config::set{{123, 456}}}, @@ -719,7 +709,7 @@ void updates_124(MutableConfigMessage& m) { m.data().erase("great"); } -const auto h124 = "8b73f316178765b9b3b37168e865c84bb5a78610cbb59b84d0fa4d3b4b3c102b"_hexbytes; +const auto h124 = "8b73f316178765b9b3b37168e865c84bb5a78610cbb59b84d0fa4d3b4b3c102b"_hex_b; TEST_CASE("config message example 2", "[config][example]") { /// This is the "Large, but still ordinary, update" example described in @@ -760,7 +750,7 @@ TEST_CASE("config message example 2", "[config][example]") { "l" "i122e" "32:"+to_string(h122)+ "de" "e" "l" "i123e" - "32:"+to_string(blake2b(m123_expected))+ + "32:"+to_string(hash::blake2b<32>(m123_expected))+ "d" "4:int0" "1:-" "4:int1" "0:" @@ -806,8 +796,8 @@ TEST_CASE("config message example 2", "[config][example]") { CHECK(to_hex(m.hash()) == to_hex(h124)); } -const auto h125a = "80f229c3667de6d0fa6f96b53118e097fbda82db3ca1aea221a3db91ea9c45fb"_hexbytes; -const auto h125b = "ab12f0efe9a9ed00db6b17b44ae0ff36b9f49094077fb114f415522f2a0e98de"_hexbytes; +const auto h125a = "80f229c3667de6d0fa6f96b53118e097fbda82db3ca1aea221a3db91ea9c45fb"_hex_b; +const auto h125b = "ab12f0efe9a9ed00db6b17b44ae0ff36b9f49094077fb114f415522f2a0e98de"_hex_b; // clang-format off const auto m126_expected = to_vector( @@ -995,7 +985,7 @@ TEST_CASE("config message example 4 - complex conflict resolution", "[config][ex m120b.serialize()}}; REQUIRE(m124a.hash() < m124b.hash()); - REQUIRE(h125a < h125b); + REQUIRE(to_hex(h125a) < to_hex(h125b)); REQUIRE(m126a.hash() < m126b.hash()); // Now we merge m126a and m126b together and should end up with the final merged result. diff --git a/tests/test_core_configs.cpp b/tests/test_core_configs.cpp new file mode 100644 index 000000000..acf35d954 --- /dev/null +++ b/tests/test_core_configs.cpp @@ -0,0 +1,642 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_helper.hpp" + +using namespace session; +using namespace session::core; + +namespace { + +/// Reopens the same database file, which is how "does this survive a restart" is asked: the configs +/// are rebuilt from their dumps rather than from anything still in memory. +void reopen(TempCore& c) { + c.core.reset(); + c.core = std::make_unique(c.path); +} + +/// Publishes the defaults a new account is created with, so a test can start from "nothing owed" +/// rather than from "this account has just come into being and has not said so yet". +void settle_new_account(TempCore& c) { + auto& profile = c->configs.user_profile(); + auto [seqno, messages, obsolete] = profile.push(); + profile.confirm_pushed(seqno, {"seededprofile"}); + c->configs.store_dumps(); +} + +/// What another device pushed to its Contacts config, having added one contact. Unlike the profile +/// helper this starts from nothing, because a new account seeds no contacts, so there is no shared +/// history for it to descend from and nothing for it to collide with. +std::vector> contacts_from_another_device( + TempCore& c, std::string_view session_id) { + auto seed = c->globals.account_seed(); + config::Contacts theirs{seed.ed25519_secret(), std::nullopt}; + theirs.set(theirs.get_or_construct(std::string{session_id})); + auto [seqno, messages, obsolete] = theirs.push(); + return messages; +} + +int64_t stored_dumps(TempCore& c) { + return c->database().conn().prepared_get("SELECT count(*) FROM config_dumps"); +} + +/// A config message as it would arrive from the swarm: what another device on this account pushed. +/// Built from a second config object holding the same account key, since that is exactly what +/// another device is. +std::vector> push_from_another_device(TempCore& c, std::string_view name) { + // Descends from what we published rather than being invented beside it: a device built from + // nothing would land on the same seqno as our own account defaults and have to be merged with + // them, which is a different scenario (and one this file covers separately). + settle_new_account(c); + + auto seed = c->globals.account_seed(); + config::UserProfile theirs{seed.ed25519_secret(), c->configs.user_profile().make_dump()}; + theirs.set_name(name); + auto [seqno, messages, obsolete] = theirs.push(); + return messages; +} + +/// Note the spans: a SwarmMessage points at its data rather than owning it, exactly as one decoded +/// from a poll response does, so whatever is passed in here has to outlive the result. +std::vector as_swarm_messages( + const std::vector>& messages, std::string_view tag = "fakehash") { + std::vector out; + for (size_t i = 0; i < messages.size(); i++) { + SwarmMessage m; + m.hash = fmt::format("{}{}", tag, i); + m.data = messages[i]; + out.push_back(std::move(m)); + } + return out; +} + +/// A Core with a mock network attached, ready to have a push observed. +struct PushableCore { + TempCore core; + MockNetwork* net = nullptr; + + PushableCore() { + net = attach_mock_network(*core); + net->current_node.remote_pubkey[0] = std::byte{0x01}; + net->sent_requests.clear(); + } + + core::Core* operator->() { return &*core; } + + const network::Request& only_request() { + REQUIRE(net->sent_requests.size() == 1); + return net->sent_requests[0].request; + } + + /// The subrequests of the one request sent, bound to a local: ranging over + /// `parse_json(body)["requests"]` directly would iterate a reference into a dead temporary. + nlohmann::json subrequests() { return parse_json(*only_request().body)["requests"]; } + + /// Answers the outstanding request as the storage server would, one result per subrequest. + /// `hashes` gives the hash each store returns; a nullopt is a store that failed. + void answer(std::vector> hashes, int delete_code = 200) { + auto results = nlohmann::json::array(); + for (auto& h : hashes) { + if (h) + results.push_back({{"code", 200}, {"body", {{"hash", *h}}}}); + else + results.push_back({{"code", 503}, {"body", {{"reason", "nope"}}}}); + } + auto subs = subrequests(); + while (results.size() < subs.size()) + results.push_back({{"code", delete_code}, {"body", nlohmann::json::object()}}); + + auto callback = net->sent_requests[0].callback; + net->sent_requests.clear(); + callback(true, false, 200, {}, nlohmann::json{{"results", results}}.dump()); + } +}; + +} // namespace + +TEST_CASE("Configs: a fresh account starts with its defaults", "[core][configs]") { + TempCore c{}; + + // Not blank: creating an account writes the defaults it should start life with, and note to + // self starting hidden is one of them. It is owed to the swarm precisely because it is shared + // -- the account's other devices have to be told, or they would each invent their own answer. + CHECK(c->configs.user_profile().get_nts_priority() == -1); + CHECK(c->configs.needs_push()); + CHECK(stored_dumps(c) == 1); + + // Everything not defaulted is still empty. + CHECK_FALSE(c->configs.user_profile().get_name()); + CHECK(c->configs.contacts().size() == 0); + + settle_new_account(c); + CHECK_FALSE(c->configs.needs_push()); +} + +TEST_CASE("Configs: a namespace names exactly one config", "[core][configs]") { + TempCore c{}; + + auto base = [](auto& conf) { return static_cast(&conf); }; + + CHECK(c->configs.for_namespace(config::Namespace::UserProfile) == + base(c->configs.user_profile())); + CHECK(c->configs.for_namespace(config::Namespace::Contacts) == base(c->configs.contacts())); + CHECK(c->configs.for_namespace(config::Namespace::UserGroups) == + base(c->configs.user_groups())); + CHECK(c->configs.for_namespace(config::Namespace::ConvoInfoVolatile) == + base(c->configs.convo_info_volatile())); + + // Local reports UserProfile's namespace, having none of its own, so the lookup must not be + // answering from storage_namespace() -- if it were, one of these two would win arbitrarily. + CHECK(c->configs.for_namespace(config::Namespace::UserProfile) != base(c->configs.local())); + + // A namespace that holds no config at all. + CHECK(c->configs.for_namespace(config::Namespace::Default) == nullptr); +} + +TEST_CASE("Configs: a dumped config survives a restart", "[core][configs]") { + TempCore c{}; + + c->configs.user_profile().set_name("Leia"); + c->configs.local().set_setting("some_toggle", true); + c->configs.store_dumps(); + + reopen(c); + + CHECK(c->configs.user_profile().get_name() == "Leia"); + CHECK(c->configs.local().get_setting("some_toggle") == true); + + // Reloading is not a change, so it owes no new dump. + CHECK_FALSE(c->configs.user_profile().needs_dump()); +} + +TEST_CASE("Configs: merging what another device pushed", "[core][configs]") { + TempCore c{}; + + auto pushed = push_from_another_device(c, "Padmé"); + auto incoming = as_swarm_messages(pushed); + c->receive_messages(incoming, config::Namespace::UserProfile, true); + + CHECK(c->configs.user_profile().get_name() == "Padmé"); + + // Adopting someone else's config outright is not a change of ours, so there is nothing to push + // back -- but it is a change to what we hold, so it is written out. + CHECK_FALSE(c->configs.needs_push()); + CHECK(stored_dumps(c) == 1); + + reopen(c); + CHECK(c->configs.user_profile().get_name() == "Padmé"); +} + +TEST_CASE("Configs: a local change survives merging a config that predates it", "[core][configs]") { + TempCore c{}; + + // Our own unpushed change participates in the merge as though it had been pushed, so a config + // from another device that has never heard of it does not erase it. This is what makes "in our + // database but not in the config" mean deleted elsewhere rather than not yet synced. + c->configs.contacts().set(c->configs.contacts().get_or_construct("05" + std::string(64, 'a'))); + REQUIRE(c->configs.contacts().size() == 1); + + auto seed = c->globals.account_seed(); + config::Contacts theirs{seed.ed25519_secret(), std::nullopt}; + theirs.set(theirs.get_or_construct("05" + std::string(64, 'b'))); + auto [seqno, messages, obsolete] = theirs.push(); + + auto incoming = as_swarm_messages(messages); + c->receive_messages(incoming, config::Namespace::Contacts, true); + + // Both contacts are present, and the merged result is ours to push since only we hold it. + CHECK(c->configs.contacts().size() == 2); + CHECK(c->configs.needs_push()); +} + +TEST_CASE("Configs: Local is never owed to a swarm", "[core][configs]") { + TempCore c{}; + settle_new_account(c); + + c->configs.local().set_setting("a_toggle", true); + + // The change is real -- it is held, and dumped like any other config... + CHECK(c->configs.local().get_setting("a_toggle") == true); + CHECK(c->configs.local().needs_dump()); + + // ...but Local has no swarm, so it can never make the account owe a push. It declines on its + // own account (needs_push() is overridden to false) and is also left out of the pushable set, + // so neither alone is load-bearing. + CHECK_FALSE(c->configs.local().needs_push()); + CHECK_FALSE(c->configs.needs_push()); +} + +TEST_CASE("Configs: a batch holds back the dump", "[core][configs]") { + TempCore c{}; + settle_new_account(c); + + // Contacts rather than UserProfile, because a new account has already dumped the latter to + // record its defaults: counting rows only shows the deferral for a config that has none yet. + auto pushed = contacts_from_another_device(c, "05" + std::string(64, 'a')); + auto profile = as_swarm_messages(pushed); + REQUIRE(stored_dumps(c) == 1); + + { + auto held = c->configs.batch(); + c->receive_messages(profile, config::Namespace::Contacts, true); + + // The merge landed, but writing it out is deferred: nothing reads a half-processed batch. + CHECK(c->configs.contacts().size() == 1); + CHECK(stored_dumps(c) == 1); + } + + CHECK(stored_dumps(c) == 2); +} + +namespace { + +/// Records what configs_changed reported, one entry per firing. +struct ChangeWatcher { + std::vector> reported; + + core::callbacks callbacks() { + core::callbacks cbs; + cbs.configs_changed = [this](std::span changed) { + reported.emplace_back(changed.begin(), changed.end()); + }; + return cbs; + } +}; + +} // namespace + +TEST_CASE("Configs: a merge that changed something is reported", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + auto pushed = push_from_another_device(c, "Padmé"); + auto incoming = as_swarm_messages(pushed); + c->receive_messages(incoming, config::Namespace::UserProfile, true); + + REQUIRE(w.reported.size() == 1); + CHECK(w.reported[0] == std::vector{config::Namespace::UserProfile}); + + // The same message again changes nothing, and nothing is what gets reported -- otherwise every + // poll that re-fetched the same config would send the application round the houses again. + c->receive_messages(incoming, config::Namespace::UserProfile, true); + CHECK(w.reported.size() == 1); +} + +TEST_CASE("Configs: one batch reports everything it changed, once", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + auto profile_pushed = push_from_another_device(c, "Leia"); + auto profile = as_swarm_messages(profile_pushed); + auto contacts_pushed = contacts_from_another_device(c, "05" + std::string(64, 'a')); + auto contacts = as_swarm_messages(contacts_pushed); + + { + auto held = c->configs.batch(); + c->receive_messages(profile, config::Namespace::UserProfile, true); + c->receive_messages(contacts, config::Namespace::Contacts, true); + } + + // One notification carrying both, not one per config: a poll can deliver all four, and telling + // the application about each in turn shows it a half-applied state. + REQUIRE(w.reported.size() == 1); + auto changed = w.reported[0]; + std::ranges::sort(changed); + CHECK(changed == std::vector{config::Namespace::UserProfile, config::Namespace::Contacts}); +} + +TEST_CASE("Configs: a conflicting merge at our own seqno is reported", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + // A local change of our own, at some seqno. + c->configs.contacts().set(c->configs.contacts().get_or_construct("05" + std::string(64, 'a'))); + auto our_seqno = c->configs.contacts().seqno(); + + // Our config can be in either of two states here, and _merge takes a different path for each: + // Dirty means the change has not been serialised into a message at all, so nothing else can + // have built on it; Waiting means a message exists and has gone to the swarm, so another device + // may well have seen it. A generator rather than sections, so that this composes with the + // sections below into all four combinations rather than replacing them. + const bool already_a_message = GENERATE(false, true); + CAPTURE(already_a_message); + if (already_a_message) + c->configs.contacts().push(); + REQUIRE(c->configs.contacts().is_dirty() == !already_a_message); + REQUIRE(c->configs.contacts().seqno() == our_seqno); + + // Another device changed things from the same starting point, so its push carries the *same* + // seqno as ours with different contents. Both shapes of disagreement are worth covering: one + // where neither side's data contains the other, and one where theirs contains ours outright -- + // the second being the case where "adopt the superset" would leave the seqno alone if the + // superset were chosen on data rather than on the diff chain. + std::vector theirs_has; + size_t expected_contacts = 0; + + SECTION("neither side's changes contain the other's") { + theirs_has = {"05" + std::string(64, 'b')}; + expected_contacts = 2; + } + SECTION("theirs contains ours and more") { + theirs_has = {"05" + std::string(64, 'a'), "05" + std::string(64, 'b')}; + expected_contacts = 2; + } + + std::vector> pushed; + { + auto seed = c->globals.account_seed(); + config::Contacts theirs{seed.ed25519_secret(), std::nullopt}; + for (const auto& id : theirs_has) + theirs.set(theirs.get_or_construct(id)); + REQUIRE(theirs.seqno() == our_seqno); + auto [seqno, messages, obsolete] = theirs.push(); + pushed = std::move(messages); + } + auto incoming = as_swarm_messages(pushed); + + c->receive_messages(incoming, config::Namespace::Contacts, true); + + // The data changed, so the application has to be told. The risk being checked is that + // resolving two same-numbered configs might leave the seqno where it was, which a seqno + // comparison would then miss. It does not: two distinct messages at one seqno are a conflict + // whatever their contents, and a conflict resolves to one past the highest. + CHECK(c->configs.contacts().size() == expected_contacts); + CHECK(c->configs.contacts().seqno() > our_seqno); + REQUIRE(w.reported.size() == 1); + CHECK(w.reported[0] == std::vector{config::Namespace::Contacts}); +} + +TEST_CASE("Configs: merging a change identical to our own", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + auto contact = "05" + std::string(64, 'a'); + c->configs.contacts().set(c->configs.contacts().get_or_construct(contact)); + auto our_seqno = c->configs.contacts().seqno(); + + // Another device made the very same change from the same starting point. + auto pushed = contacts_from_another_device(c, contact); + auto incoming = as_swarm_messages(pushed); + c->receive_messages(incoming, config::Namespace::Contacts, true); + + // Nothing is lost: we already held exactly what arrived. + CHECK(c->configs.contacts().size() == 1); + CHECK(c->configs.contacts().get(contact).has_value()); + + // And nothing is owed: agreeing with another device settles clean against that device's + // message rather than leaving us dirty, so it costs no push carrying no changes. + CHECK_FALSE(c->configs.contacts().needs_push()); + + // The seqno is deliberately not asserted. It currently advances even though the data did not, + // because merging while dirty builds a MutableConfigMessage and that constructor increments + // unconditionally (see its comment in config.hpp) -- and the unwind for that spurious increment + // in _merge only applies when the winning config is our own, which here it is not. Pinning + // that down would be encoding an accident. + + // Deliberately not asserted: whether this reported to the application. It currently does, + // because the seqno moved even though the data did not, so the reconciling layer does a walk + // that finds nothing to do. That is the safe direction to be wrong in and is idempotent by + // design -- and if the merge ever learns to recognise an identical config and leave the seqno + // alone, this would stop reporting with no change needed here. What must never happen is the + // reverse, and the case above covers that. +} + +TEST_CASE("Configs: an adopted duplicate leaves no gap behind", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + auto seed = c->globals.account_seed(); + config::Contacts them{seed.ed25519_secret(), std::nullopt}; + + auto a = "05" + std::string(64, 'a'); + auto b = "05" + std::string(64, 'b'); + + // Both devices make the same change from the same starting point. + c->configs.contacts().set(c->configs.contacts().get_or_construct(a)); + them.set(them.get_or_construct(a)); + REQUIRE(c->configs.contacts().seqno() == them.seqno()); + auto agreed_seqno = them.seqno(); + + { + auto [seqno, messages, obsolete] = them.push(); + them.confirm_pushed(seqno, {"theirs1"}); + auto incoming = as_swarm_messages(messages, "theirs"); + c->receive_messages(incoming, config::Namespace::Contacts, true); + } + + // Adopting their identical config leaves us on the seqno the swarm actually holds, rather than + // one past it: no number is consumed that no stored message occupies. + CHECK(c->configs.contacts().seqno() == agreed_seqno); + CHECK_FALSE(c->configs.contacts().needs_push()); + + // So the other device's further change of its own lands on the next number up, with nothing of + // ours already sitting on it. + them.set(them.get_or_construct(b)); + REQUIRE(them.seqno() == agreed_seqno + 1); + + { + auto [seqno, messages, obsolete] = them.push(); + auto incoming = as_swarm_messages(messages, "theirs2-"); + c->receive_messages(incoming, config::Namespace::Contacts, true); + } + + // Their change arrives intact and ours is still there. + CHECK(c->configs.contacts().size() == 2); + CHECK(c->configs.contacts().get(a).has_value()); + CHECK(c->configs.contacts().get(b).has_value()); + + // ...and it is adopted at its own seqno rather than resolving as a conflict one past both, so + // we are left neither dirty nor owing a push for a change that was never ours. One real + // change, one seqno -- which is also what keeps the "within N" conflict window from spending + // two of its five carrying a single change. + CHECK(c->configs.contacts().seqno() == agreed_seqno + 1); + CHECK_FALSE(c->configs.contacts().needs_push()); +} + +TEST_CASE("Configs: a local change is not reported back", "[core][configs][notify]") { + ChangeWatcher w; + TempCore c{w.callbacks()}; + + { + auto held = c->configs.batch(); + c->configs.user_profile().set_name("Leia"); + } + + // The application made this change; being told about it would be news to nobody, and would + // invite it to reconcile its own write back over itself. + CHECK(w.reported.empty()); +} + +TEST_CASE("Configs: a change goes out as one signed sequence", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + // Local changes too: it must not appear in what goes out. + c->configs.local().set_setting("a_toggle", true); + c->configs.push_now(); + + CHECK(c.only_request().endpoint == "sequence"); + + auto subs = c.subrequests(); + REQUIRE(subs.size() == 1); + CHECK(subs[0]["method"] == "store"); + CHECK(subs[0]["params"]["namespace"] == 2); + CHECK(subs[0]["params"]["ttl"] == std::chrono::milliseconds{30 * 24h}.count()); + // Config namespaces are owner-write, so the store carries a signature and the key to check it. + CHECK(subs[0]["params"].contains("signature")); + CHECK(subs[0]["params"].contains("pubkey_ed25519")); + + // Nothing is obsolete on a first push, so there is nothing to delete. + CHECK_FALSE(subs[0].contains("delete")); +} + +TEST_CASE("Configs: two dirty configs share one request", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + c->configs.contacts().set(c->configs.contacts().get_or_construct("05" + std::string(64, 'a'))); + c->configs.push_now(); + + auto subs = c.subrequests(); + REQUIRE(subs.size() == 2); + std::vector namespaces{subs[0]["params"]["namespace"], subs[1]["params"]["namespace"]}; + std::ranges::sort(namespaces); + CHECK(namespaces == std::vector{2, 3}); +} + +TEST_CASE("Configs: a stored push stops being owed", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + c->configs.push_now(); + + // Handing it to the swarm is not the same as it having arrived, so it is still owed until the + // store is confirmed -- otherwise a failed push would be forgotten. + CHECK(c->configs.needs_push()); + + c.answer({"hash1"}); + CHECK_FALSE(c->configs.needs_push()); +} + +TEST_CASE("Configs: a rejected store leaves the config dirty", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + c->configs.push_now(); + c.answer({std::nullopt}); + + // The change is still ours to deliver, and pushing again offers it again. + CHECK(c->configs.needs_push()); + c->configs.push_now(); + CHECK(c.subrequests().size() == 1); +} + +TEST_CASE("Configs: the next push deletes what it replaces", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + c->configs.push_now(); + c.answer({"hash1"}); + + c->configs.user_profile().set_name("Padmé"); + c->configs.push_now(); + + auto subs = c.subrequests(); + REQUIRE(subs.size() == 2); + CHECK(subs[0]["method"] == "store"); + + // The delete goes last: a sequence stops at its first failure, so nothing is removed before + // what replaces it has been stored. + CHECK(subs[1]["method"] == "delete"); + CHECK(subs[1]["params"]["messages"] == nlohmann::json::array({"hash1"})); + CHECK(subs[1]["params"].contains("signature")); +} + +TEST_CASE("Configs: a change schedules a push rather than sending one", "[core][configs][push]") { + PushableCore c; + + { + auto held = c->configs.batch(); + c->configs.user_profile().set_name("Leia"); + } + + // Releasing the batch is what notices the change; it schedules rather than sending, so a run of + // changes coalesces into one request. + CHECK(TestHelper::push_scheduled(c->configs)); + CHECK(c.net->sent_requests.empty()); +} + +TEST_CASE("Configs: the debounce waits for quiet, up to a limit", "[core][configs][push]") { + PushableCore c; + c->configs.push_debounce = 2s; + c->configs.push_max_delay = 10s; + + { + auto held = c->configs.batch(); + c->configs.user_profile().set_name("Leia"); + } + REQUIRE(TestHelper::push_scheduled(c->configs)); + + SECTION("changes still arriving hold it back") { + TestHelper::backdate_push_state(c->configs, 500ms, 1s); + TestHelper::push_if_due(c->configs); + CHECK(c.net->sent_requests.empty()); + CHECK(TestHelper::push_scheduled(c->configs)); + } + + SECTION("quiet for long enough sends it") { + TestHelper::backdate_push_state(c->configs, 3s, 4s); + TestHelper::push_if_due(c->configs); + CHECK(c.net->sent_requests.size() == 1); + CHECK_FALSE(TestHelper::push_scheduled(c->configs)); + } + + SECTION("a steady trickle cannot defer it past the cap") { + // Never quiet -- the last change was a moment ago -- but the burst began long enough ago + // that waiting for quiet would mean waiting indefinitely. + TestHelper::backdate_push_state(c->configs, 100ms, 11s); + TestHelper::push_if_due(c->configs); + CHECK(c.net->sent_requests.size() == 1); + } +} + +TEST_CASE("Configs: pushing can be switched off entirely", "[core][configs][push]") { + PushableCore c; + c->configs.push_enabled = false; + + c->configs.user_profile().set_name("Leia"); + c->configs.push_now(); + + // Nothing goes out... + CHECK(c.net->sent_requests.empty()); + + // ...and nothing pretends it did: the change is still held and still owed, so the state reads + // as unpublished rather than as settled. + CHECK(c->configs.user_profile().get_name() == "Leia"); + CHECK(c->configs.needs_push()); + + // Switching it back on lets everything accumulated since go out together. + c->configs.push_enabled = true; + c->configs.push_now(); + CHECK(c.net->sent_requests.size() == 1); +} + +TEST_CASE("Configs: a push already in flight is not duplicated", "[core][configs][push]") { + PushableCore c; + + c->configs.user_profile().set_name("Leia"); + c->configs.push_now(); + REQUIRE(c.net->sent_requests.size() == 1); + + // A second change while the first is out must not race it onto the wire; the completion picks + // it up instead. + c->configs.contacts().set(c->configs.contacts().get_or_construct("05" + std::string(64, 'a'))); + c->configs.push_now(); + CHECK(c.net->sent_requests.size() == 1); +} diff --git a/tests/test_core_devices.cpp b/tests/test_core_devices.cpp new file mode 100644 index 000000000..4642b9249 --- /dev/null +++ b/tests/test_core_devices.cpp @@ -0,0 +1,702 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "test_helper.hpp" +#include "utils.hpp" + +using namespace session; +using namespace session::core; +using namespace std::literals; + +namespace { + +/// A Core whose account was *restored* rather than generated, and which therefore owes no device +/// group: this is the state a device is in before it has joined one. +/// +/// A plain `TempCore` generates its account, which now establishes a group with itself as the only +/// member -- so anything asserting on an unregistered device has to say which of the two it means. +TempCore restored_core() { + std::array seed{}; + random::fill(seed); + return TempCore{core::predefined_seed{std::span{seed}}}; +} + +} // namespace + +TEST_CASE("Devices - identity", "[core][devices]") { + TempCore c; + + SECTION("device_id is 64-char hex") { + auto id = c->devices.device_id(); + REQUIRE(id.size() == 64); + CHECK(std::all_of(id.begin(), id.end(), [](char ch) { + return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'); + })); + } + + SECTION("device_id is stable") { + CHECK(c->devices.device_id() == c->devices.device_id()); + } + + SECTION("two independent cores have different device IDs") { + TempCore c2; + CHECK(c->devices.device_id() != c2->devices.device_id()); + } +} + +TEST_CASE("Devices - initial state", "[core][devices]") { + auto c = restored_core(); + + SECTION("device_info defaults") { + auto [info, is_registered] = c->devices.device_info(); + // seqno == 0 is the sentinel meaning no row exists yet + CHECK(info.seqno == 0); + CHECK_FALSE(is_registered); + } + + SECTION("devices() is empty") { + CHECK(c->devices.devices(true, true, true).empty()); + } + + SECTION("needs_push is false") { + auto np = c->devices.needs_push(); + CHECK_FALSE(np.device_group); + CHECK_FALSE(np.account_pubkey); + } +} + +TEST_CASE("Devices - update_info and same_user_fields", "[core][devices]") { + auto c = restored_core(); + + SECTION("update_info persists fields and sets seqno=1") { + device::Info info{}; + info.type = device::Type::Session_iOS; + info.description = "test phone"; + info.version = {1, 2, 3}; + + c->devices.update_info(info); + + auto [got, is_registered] = c->devices.device_info(); + CHECK(got.seqno == 1); + CHECK(got.type == device::Type::Session_iOS); + CHECK(got.description == "test phone"); + CHECK(got.version == std::array{1, 2, 3}); + CHECK(got.state == device::State::Unregistered); + CHECK_FALSE(is_registered); + } + + SECTION("identical update does not bump seqno") { + device::Info info{}; + info.type = device::Type::Session_Desktop; + info.description = "desktop"; + info.version = {0, 1, 0}; + + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 1); + + c->devices.update_info(info); // identical — should not bump + CHECK(c->devices.device_info().first.seqno == 1); + } + + SECTION("changed description bumps seqno") { + device::Info info{}; + info.description = "first"; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 1); + + info.description = "second"; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 2); + } + + SECTION("changed type bumps seqno") { + device::Info info{}; + info.type = device::Type::Session_Android; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 1); + + info.type = device::Type::Session_Desktop; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 2); + } + + SECTION("changed version bumps seqno") { + device::Info info{}; + info.version = {1, 0, 0}; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 1); + + info.version = {2, 0, 0}; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 2); + } + + SECTION("extra fields round-trip and participate in comparison") { + device::Info info{}; + info.extra["custom_key"] = std::string{"hello"}; + c->devices.update_info(info); + + auto [got, _] = c->devices.device_info(); + CHECK(got.seqno == 1); + REQUIRE(got.extra.count("custom_key")); + CHECK(std::get(got.extra.at("custom_key")) == "hello"); + + // Same extra — no bump + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 1); + + // Changed extra — bump + info.extra["custom_key"] = std::string{"world"}; + c->devices.update_info(info); + CHECK(c->devices.device_info().first.seqno == 2); + } + + SECTION("same_user_fields ignores state/seqno/pk_*") { + device::Info a{}, b{}; + a.type = device::Type::Session_iOS; + a.description = "foo"; + a.version = {1, 2, 3}; + b = a; + + CHECK(a.same_user_fields(b)); + + // Differ in seqno — should still be "same" user fields + b.seqno = 99; + CHECK(a.same_user_fields(b)); + + // Differ in description — not same + b.seqno = a.seqno; + b.description = "bar"; + CHECK_FALSE(a.same_user_fields(b)); + } + + SECTION("update_info device appears in devices(include_unregistered=true)") { + device::Info info{}; + info.description = "my device"; + c->devices.update_info(info); + + auto devs = c->devices.devices(false, false, true); + CHECK(devs.size() == 1); + CHECK(devs.begin()->second.description == "my device"); + } +} + +TEST_CASE("Devices - device keys", "[core][devices]") { + TempCore c; + + SECTION("active_device_keys returns at least one key with correct sizes") { + auto keys = c->devices.active_device_keys(); + REQUIRE_FALSE(keys.empty()); + CHECK(keys.front().x25519_pub.size() == 32); + CHECK(keys.front().mlkem768_pub.size() == 1184); + CHECK_FALSE(keys.front().rotated.has_value()); + } + + SECTION("rotate_device_keys produces a distinct key") { + auto before = c->devices.active_device_keys(); + REQUIRE_FALSE(before.empty()); + + c->devices.rotate_device_keys(); + auto after = c->devices.active_device_keys(); + + CHECK(after.front().x25519_pub != before.front().x25519_pub); + CHECK(after.front().mlkem768_pub != before.front().mlkem768_pub); + CHECK_FALSE(after.front().rotated.has_value()); + } + + SECTION("after one rotation active_device_keys has two entries") { + auto initial = c->devices.active_device_keys(); // ensure initial key exists + REQUIRE(initial.size() == 1); + c->devices.rotate_device_keys(); + auto keys = c->devices.active_device_keys(); + CHECK(keys.size() == 2); + CHECK_FALSE(keys.front().rotated.has_value()); + CHECK(keys.back().rotated.has_value()); + } + + SECTION("after two rotations active_device_keys has three entries") { + c->devices.active_device_keys(); // ensure initial key exists + c->devices.rotate_device_keys(); + c->devices.rotate_device_keys(); + auto keys = c->devices.active_device_keys(); + CHECK(keys.size() == 3); + CHECK_FALSE(keys[0].rotated.has_value()); + CHECK(keys[1].rotated.has_value()); + CHECK(keys[2].rotated.has_value()); + } +} + +TEST_CASE("Devices - device group payload padding", "[core][devices]") { + TempCore c; + + // Real keys, not random bytes: ML-KEM encapsulation is performed against each device's pubkey. + // Rotating produces distinct valid keypairs, and all of them stay in this device's active key + // set, so this Core can also decrypt whatever it encrypts below. + std::vector infos; + for (int i = 0; i < 5; i++) { + auto k = c->devices.rotate_device_keys(); + auto& info = infos.emplace_back(); + random::fill(info.id); + info.seqno = 1; + info.timestamp = clock_now_s(); + info.type = device::Type::Session_Desktop; + info.description = "test device"; + info.state = device::State::Registered; + info.version = {1, 0, 0}; + info.pk_x25519 = k.x25519_pub; + info.pk_mlkem768 = k.mlkem768_pub; + } + + auto encrypted_size = [&](size_t n) { + device::map m; + for (size_t i = 0; i < n; i++) + m.emplace(infos[i].id, infos[i]); + return TestHelper::encrypt_device_data(c->devices, m).size(); + }; + + SECTION("the payload is padded to 2300 + 6400N") { + device::map m; + m.emplace(infos[0].id, infos[0]); + auto enc = TestHelper::encrypt_device_data(c->devices, m); + + // The encrypted payload sits in the envelope's "d" field, and is the padded plaintext plus + // the poly1305 tag. One device is one bucket, on top of the account key allowance. + oxenc::bt_dict_consumer env{to_string_view(enc)}; + REQUIRE(env.skip_until("d")); + auto payload = env.consume_string_view(); + CHECK(payload.size() == 2300 + 6400 + 16); + } + + SECTION("groups of up to 4 devices are indistinguishable by size") { + auto one = encrypted_size(1); + CHECK(encrypted_size(2) == one); + CHECK(encrypted_size(3) == one); + CHECK(encrypted_size(4) == one); + + // The 5th device crosses into the next bucket, which is expected and unavoidable — the + // guarantee is bucketing, not constant size. + CHECK(encrypted_size(5) > one); + } + + SECTION("padding round-trips off again") { + device::map m; + for (size_t i = 0; i < 3; i++) + m.emplace(infos[i].id, infos[i]); + + auto enc = TestHelper::encrypt_device_data(c->devices, m); + auto plaintext = TestHelper::decrypt_device_data(c->devices, enc); + + // A bt-encoded dict always ends in 'e'; if any padding survived, it would not. + REQUIRE_FALSE(plaintext.empty()); + CHECK(plaintext.back() == std::byte{'e'}); + + // And the recovered payload really is the device dict, not a truncation of it. + oxenc::bt_dict_consumer btdc{to_string_view(plaintext)}; + REQUIRE(btdc.skip_until("D")); + auto devs = btdc.consume_dict_consumer(); + int count = 0; + while (!devs.is_finished()) { + devs.skip_until(devs.key()); + devs.consume_dict_consumer(); + count++; + } + CHECK(count == 3); + } + + SECTION("a kicked device is named in the payload but is not a recipient") { + device::map m; + for (size_t i = 0; i < 4; i++) + m.emplace(infos[i].id, infos[i]); + auto four_registered = TestHelper::encrypt_device_data(c->devices, m).size(); + + // A fifth entry, kicked rather than registered. + auto kicked = infos[4]; + kicked.state = device::State::Kicked; + kicked.kicked = clock_now_s(); + m.emplace(kicked.id, kicked); + + auto with_kicked = TestHelper::encrypt_device_data(c->devices, m); + + // Unchanged size: five entries, but still only four recipients, so the key and ciphertext + // lists stay in the 4 bucket. Were the kicked device handed a key they would cross into + // the 8 bucket and this would grow -- which is what makes this an assertion about the + // recipient set rather than about padding. + CHECK(with_kicked.size() == four_registered); + + // And it is still named in the payload: that is how every other device learns it is gone. + auto plaintext = TestHelper::decrypt_device_data(c->devices, with_kicked); + oxenc::bt_dict_consumer btdc{to_string_view(plaintext)}; + REQUIRE(btdc.skip_until("D")); + auto devs = btdc.consume_dict_consumer(); + std::string_view kicked_key{ + reinterpret_cast(kicked.id.data()), kicked.id.size()}; + CHECK(devs.skip_until(kicked_key)); + } +} + +TEST_CASE("Devices - account keys", "[core][devices]") { + // Restored: two sections here are about what the rotation timers say for a device that is *not* + // in a group, and a generated account is in one from the moment it exists. + auto c = restored_core(); + + SECTION("active_account_keys returns at least one key with correct sizes") { + auto keys = c->devices.active_account_keys(); + REQUIRE_FALSE(keys.empty()); + CHECK(keys.front().x25519_pub.size() == 32); + CHECK(keys.front().mlkem768_pub.size() == 1184); + CHECK_FALSE(keys.front().rotated.has_value()); + } + + SECTION("rotate_account_keys produces a distinct key: newer timestamp wins") { + auto before = c->devices.active_account_keys(); + REQUIRE(before.size() == 1); + + // Advance clock by 1s so the new key has a strictly later created timestamp and + // deterministically wins tie-breaking (created DESC, seed ASC). + ScopedClockOffset adv{1s}; + c->devices.rotate_account_keys(); + auto after = c->devices.active_account_keys(); + + REQUIRE(after.size() == 2); + CHECK_FALSE(after.front().rotated.has_value()); + CHECK(after.back().rotated.has_value()); + CHECK(after.front().x25519_pub != before.front().x25519_pub); + } + + SECTION("rotate_account_keys produces a distinct key: same timestamp, seed tiebreak") { + // Snap the adjusted clock to the start of the next second so both key-creation calls + // land in the same second with no risk of spanning a second boundary. + ScopedClockOffset pin_to_next_second{ + (clock_now_s() + 1s) - std::chrono::system_clock::now()}; + + c->devices.active_account_keys(); // ensure initial key exists at pinned second + c->devices.rotate_account_keys(); // new key created at same second + auto keys = c->devices.active_account_keys(); + REQUIRE(keys.size() == 2); + CHECK_FALSE(keys.front().rotated.has_value()); + CHECK(keys.back().rotated.has_value()); + + // Look up each key's seed via its x25519 pubkey and verify the tie-breaking rule: + // the active key must have the lexicographically smaller seed. + auto active_seed = TestHelper::account_key_seed(c->devices, keys.front().x25519_pub); + auto rotated_seed = TestHelper::account_key_seed(c->devices, keys.back().x25519_pub); + CHECK(active_seed < rotated_seed); + } + + SECTION("after one rotation active_account_keys has two entries") { + c->devices.active_account_keys(); // ensure initial key exists + c->devices.rotate_account_keys(); + auto keys = c->devices.active_account_keys(); + CHECK(keys.size() == 2); + CHECK_FALSE(keys.front().rotated.has_value()); + CHECK(keys.back().rotated.has_value()); + } + + SECTION("old key pruned after ACCOUNT_KEY_RETENTION") { + c->devices.active_account_keys(); // ensure initial key exists + c->devices.rotate_account_keys(); + { + auto keys = c->devices.active_account_keys(); + CHECK(keys.size() == 2); + } + + // Advance clock past retention window: old rotated key should be pruned + ScopedClockOffset advance_past_retention{Devices::ACCOUNT_KEY_RETENTION + 1s}; + auto keys = c->devices.active_account_keys(); + CHECK(keys.size() == 1); + CHECK_FALSE(keys.front().rotated.has_value()); + } + + SECTION("next_account_rotation returns nullopt when not in device group") { + CHECK_FALSE(c->devices.next_account_rotation().has_value()); + CHECK_FALSE(c->devices.account_rotation_due()); + } + + SECTION("next_device_rotation returns nullopt when not in device group") { + CHECK_FALSE(c->devices.next_device_rotation().has_value()); + CHECK_FALSE(c->devices.device_rotation_due()); + } +} + +TEST_CASE("Devices - build_link_request", "[core][devices]") { + // Restored, not generated: asking to join a group only makes sense for a device that adopted + // an existing account's seed. A device that generated the account *is* the group. + auto c = restored_core(); + + SECTION("returns non-empty message and 21-entry SAS") { + auto result = c->devices.build_link_request(); + CHECK_FALSE(result.message.empty()); + CHECK(result.sas.size() == 21); + for (const auto& s : result.sas) + CHECK_FALSE(s.empty()); + } + + SECTION("consecutive calls produce different messages") { + auto r1 = c->devices.build_link_request(); + auto r2 = c->devices.build_link_request(); + CHECK(r1.message != r2.message); + } +} + +TEST_CASE("Devices - build_account_pubkey_message", "[core][devices]") { + TempCore c; + + SECTION("non-empty output with correct structure") { + auto msg = c->devices.build_account_pubkey_message(); + REQUIRE_FALSE(msg.empty()); + + auto dict = oxenc::bt_dict_consumer{msg}; + + // "M" — mlkem768 pubkey (1184 bytes) + CHECK(dict.require("M").size() == 1184); + + // "X" — x25519 pubkey (32 bytes) + CHECK(dict.require("X").size() == 32); + + // "~" — XEd25519 signature (64 bytes) + CHECK(dict.require("~").size() == 64); + } + + SECTION("M and X match active account keys") { + auto keys = c->devices.active_account_keys(); + REQUIRE_FALSE(keys.empty()); + + auto msg = c->devices.build_account_pubkey_message(); + auto dict = oxenc::bt_dict_consumer{msg}; + + auto M = dict.require("M"); + auto X = dict.require("X"); + + CHECK(std::memcmp(M.data(), keys.front().mlkem768_pub.data(), 1184) == 0); + CHECK(std::memcmp(X.data(), keys.front().x25519_pub.data(), 32) == 0); + } + + SECTION("signature verifies against account x25519 pubkey") { + auto msg = c->devices.build_account_pubkey_message(); + auto dict = oxenc::bt_dict_consumer{msg}; + + dict.require("M"); + dict.require("X"); + + // Use require_signature to correctly extract the signed body (everything in the dict + // before the "~" key) and the signature value. + auto x25519_pub = c->globals.session_id().template subspan<1>(); // skip 0x05 prefix + bool sig_valid = false; + dict.require_signature( + "~", [&](std::span body, std::span sig) { + sig_valid = + sig.size() == 64 && xed25519::verify(sig.first<64>(), x25519_pub, body); + }); + CHECK(sig_valid); + } +} + +TEST_CASE("Devices - establishing the group", "[core][devices]") { + + SECTION("a generated account establishes a group with itself") { + TempCore c; + + auto [info, registered] = c->devices.device_info(); + CHECK(registered); + CHECK(info.state == device::State::Registered); + CHECK(info.id == c->devices.device_info().first.id); + + // Exactly one device, and it is us. + auto devs = c->devices.devices(true, true, true); + REQUIRE(devs.size() == 1); + CHECK(devs.begin()->first == info.id); + + // The whole point: a registered device is one that `needs_push` will speak for. Before + // this existed, nothing ever registered a device, so nothing was ever owed a push and no + // group could come into being. + CHECK(c->devices.needs_push().device_group); + + // The group payload carries the account's shared key seeds, so one is minted here. + auto keys = c->devices.active_account_keys(); + REQUIRE(keys.size() == 1); + CHECK_FALSE(keys.front().rotated.has_value()); + } + + SECTION("a restored account does not") { + auto c = restored_core(); + + auto [info, registered] = c->devices.device_info(); + CHECK_FALSE(registered); + CHECK(c->devices.devices(true, true, true).empty()); + CHECK_FALSE(c->devices.needs_push().device_group); + } + + SECTION("it survives a restart, and does not happen twice") { + std::optional> first_id; + int64_t first_seqno = 0; + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", random::unique_id("test_estab", 7)); + { + Core c{path}; + auto [info, registered] = c.devices.device_info(); + REQUIRE(registered); + first_id = info.id; + first_seqno = info.seqno; + } + { + // Reopened: the flag was cleared the first time, so this must not re-register or + // re-mint anything -- a second establish would bump the seqno and mint a second key. + Core c{path}; + auto [info, registered] = c.devices.device_info(); + CHECK(registered); + CHECK(info.id == *first_id); + CHECK(info.seqno == first_seqno); + CHECK(c.devices.active_account_keys().size() == 1); + } + std::error_code ec; + std::filesystem::remove(path, ec); + } +} + +TEST_CASE("Devices - a removal cannot be undone by a message", "[core][devices]") { + TempCore c; + + // A second device, with keys this core holds so that what we encrypt below is readable back. + auto k = c->devices.rotate_device_keys(); + device::Info other{}; + random::fill(other.id); + other.seqno = 1; + other.timestamp = clock_now_s(); + other.type = device::Type::Session_Android; + other.description = "other device"; + other.state = device::State::Registered; + other.version = {1, 0, 0}; + other.pk_x25519 = k.x25519_pub; + other.pk_mlkem768 = k.mlkem768_pub; + + auto [self, registered] = c->devices.device_info(); + REQUIRE(registered); + + auto deliver = [&](const device::map& m) { + TestHelper::receive_device_group_message( + c->devices, TestHelper::encrypt_device_data(c->devices, m)); + }; + auto state_of = [&](const std::array& id) { + auto devs = c->devices.devices(true, true, true); + auto found = devs.find(id); + REQUIRE(found != devs.end()); + return found->second; + }; + + // It joins. + deliver({{self.id, self}, {other.id, other}}); + REQUIRE(state_of(other.id).state == device::State::Registered); + + // It is removed, a while ago. + auto kicked_at = clock_now_s() - 1h; + auto gone = other; + gone.state = device::State::Kicked; + gone.kicked = kicked_at; + deliver({{self.id, self}, {gone.id, gone}}); + + auto after_kick = state_of(other.id); + REQUIRE(after_kick.state == device::State::Kicked); + REQUIRE(after_kick.kicked == kicked_at); + + // Now it pushes itself back in with a higher seqno, which it can do: it still holds the account + // seed, so it can sign and encrypt a message everyone accepts. + auto returning = other; + returning.seqno = 5; + returning.description = "back again"; + deliver({{self.id, self}, {returning.id, returning}}); + + auto after = state_of(other.id); + + // Refused: still removed, and none of its claims adopted. + CHECK(after.state == device::State::Kicked); + CHECK(after.description == "other device"); + + // And restated rather than merely ignored: the tombstone moves to the front of the removed + // list, and we owe a push so that devices which never saw the removal learn of it. + REQUIRE(after.kicked.has_value()); + CHECK(*after.kicked > kicked_at); + CHECK(c->devices.needs_push().device_group); +} + +TEST_CASE("Devices - a tombstone for an unknown device is kept", "[core][devices]") { + TempCore c; + + // Keys this core holds, so that the message we build is readable back and the returning record + // below is a usable recipient. + auto k = c->devices.rotate_device_keys(); + + auto [self, registered] = c->devices.device_info(); + REQUIRE(registered); + + auto deliver = [&](const device::map& m) { + TestHelper::receive_device_group_message( + c->devices, TestHelper::encrypt_device_data(c->devices, m)); + }; + auto state_of = [&](const std::array& id) { + auto devs = c->devices.devices(true, true, true); + auto found = devs.find(id); + REQUIRE(found != devs.end()); + return found->second; + }; + + // A removal for a device we have never held a record of -- which is what a device joining after + // the removal sees, since the group carries the tombstone but nothing else about it. + auto kicked_at = clock_now_s() - 1h; + device::Info gone{}; + random::fill(gone.id); + gone.state = device::State::Kicked; + gone.kicked = kicked_at; + deliver({{self.id, self}, {gone.id, gone}}); + + // Stored, rather than dropped for want of a row to update: the tombstone is the whole point of + // the entry, and needs no details to do its job. + auto after_kick = state_of(gone.id); + CHECK(after_kick.state == device::State::Kicked); + CHECK(after_kick.kicked == kicked_at); + + // And it does its job: the removed device cannot talk its way back in, exactly as it could not + // for a device that saw the removal first. + auto returning = gone; + returning.state = device::State::Registered; + returning.kicked.reset(); + returning.seqno = 5; + returning.timestamp = clock_now_s(); + returning.description = "back again"; + returning.type = device::Type::Session_Android; + returning.version = {1, 0, 0}; + returning.pk_x25519 = k.x25519_pub; + returning.pk_mlkem768 = k.mlkem768_pub; + deliver({{self.id, self}, {returning.id, returning}}); + + auto after = state_of(gone.id); + CHECK(after.state == device::State::Kicked); + CHECK(after.description != "back again"); +} + +TEST_CASE("Devices - a single-recipient group is readable", "[core][devices]") { + TempCore c; + + // The common case: an account with one device, which is what establishing a group produces. + // With one recipient every other slot in the message is padding, so nothing else can stand in + // for a real entry that was overwritten. + auto [self, registered] = c->devices.device_info(); + REQUIRE(registered); + + auto enc = TestHelper::encrypt_device_data(c->devices, device::map{{self.id, self}}); + auto plain = TestHelper::decrypt_device_data(c->devices, enc); + + REQUIRE_FALSE(plain.empty()); + oxenc::bt_dict_consumer btdc{to_string_view(plain)}; + REQUIRE(btdc.skip_until("D")); + auto devs = btdc.consume_dict_consumer(); + std::string_view self_key{reinterpret_cast(self.id.data()), self.id.size()}; + CHECK(devs.skip_until(self_key)); +} diff --git a/tests/test_core_globals.cpp b/tests/test_core_globals.cpp new file mode 100644 index 000000000..38f750f17 --- /dev/null +++ b/tests/test_core_globals.cpp @@ -0,0 +1,134 @@ +#include +#include + +#include "test_helper.hpp" + +using namespace session; +using namespace session::core; +using namespace oxenc::literals; + +TEST_CASE("Globals: set/get round-trip", "[core][globals]") { + TempCore c{}; + + CHECK(!c->globals.get_text("nope")); + + c->globals.set("a_string", "hello"); + c->globals.set("an_int", int64_t{42}); + c->globals.set("a_real", 1.5); + + CHECK(c->globals.get_text("a_string") == "hello"); + CHECK(c->globals.get_integer("an_int") == 42); + CHECK(c->globals.get_real("a_real") == 1.5); + + // Wrong-type reads come back empty rather than throwing. + CHECK(!c->globals.get_integer("a_string")); +} + +TEST_CASE("Globals: erase", "[core][globals]") { + TempCore c{}; + + c->globals.set("doomed", "value"); + REQUIRE(c->globals.get_text("doomed") == "value"); + + CHECK(c->globals.erase("doomed")); + CHECK(!c->globals.get_text("doomed")); + + // Erasing something that was never set is not an error, just false. + CHECK(!c->globals.erase("doomed")); + CHECK(!c->globals.erase("never_existed")); +} + +TEST_CASE("Globals: values persist across reopen", "[core][globals]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_globals", 7)); + std::filesystem::remove(path); + + { + Core core{path}; + core.globals.set("kept", "yes"); + core.globals.set("dropped", "no"); + CHECK(core.globals.erase("dropped")); + } + { + Core core{path}; + CHECK(core.globals.get_text("kept") == "yes"); + CHECK(!core.globals.get_text("dropped")); + } + + std::error_code ec; + std::filesystem::remove(path, ec); +} + +TEST_CASE("Globals: defer_account leaves the account unresolved", "[core][globals]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_defer", 7)); + std::filesystem::remove(path); + + constexpr auto seed = "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"_hex_b; + + SECTION("an unresolved account refuses everything needing an identity") { + Core core{path, defer_account{}}; + CHECK_FALSE(core.globals.have_account()); + + CHECK_THROWS_AS(core.globals.session_id(), no_account); + CHECK_THROWS_AS(core.globals.session_id_hex(), no_account); + CHECK_THROWS_AS(core.globals.pubkey_ed25519(), no_account); + CHECK_THROWS_AS(core.globals.account_seed(), no_account); + CHECK_THROWS_AS(core.globals.seed_mnemonic(), no_account); + + // Refused here rather than failing later inside a background poll. + CHECK_THROWS_AS(core.set_network(std::make_unique()), no_account); + + // Everything not needing an identity still works, which is what makes the state useful: + // the application can open the database and ask, before deciding. + core.globals.set("some_setting", "value"); + CHECK(core.globals.get_text("some_setting") == "value"); + } + + SECTION("create_account resolves it, and persists") { + std::string id; + { + Core core{path, defer_account{}}; + REQUIRE_FALSE(core.globals.have_account()); + core.globals.create_account(); + CHECK(core.globals.have_account()); + id = core.globals.session_id_hex(); + CHECK(id.starts_with("05")); + // Adopting a second identity would orphan everything stored against the first. + CHECK_THROWS_AS(core.globals.create_account(), std::logic_error); + } + // Reopening finds the stored seed, so defer_account is a no-op on an existing account. + Core core{path, defer_account{}}; + CHECK(core.globals.have_account()); + CHECK(core.globals.session_id_hex() == id); + } + + SECTION("restore_account adopts a given seed") { + std::string restored; + { + Core core{path, defer_account{}}; + core.globals.restore_account(predefined_seed{seed}); + CHECK(core.globals.have_account()); + restored = core.globals.session_id_hex(); + } + + // The same seed via the constructor must reach the same identity. + auto other = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_defer", 7)); + std::filesystem::remove(other); + { + Core core{other, predefined_seed{seed}}; + CHECK(core.globals.session_id_hex() == restored); + } + std::error_code ec2; + std::filesystem::remove(other, ec2); + } + + SECTION("without the option an account is created outright, as before") { + Core core{path}; + CHECK(core.globals.have_account()); + } + + std::error_code ec; + std::filesystem::remove(path, ec); +} diff --git a/tests/test_core_network.cpp b/tests/test_core_network.cpp new file mode 100644 index 000000000..f35bcaee3 --- /dev/null +++ b/tests/test_core_network.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + +#include "utils.hpp" + +using namespace session; + +TEST_CASE("Core can hold an optional Network interface", "[core][network]") { + core::callbacks callbacks; + auto db_path = std::filesystem::temp_directory_path() / "test_core_network.db"; + if (std::filesystem::exists(db_path)) + std::filesystem::remove(db_path); + + core::Core core{db_path, callbacks}; + + SECTION("Network is initially null") { + CHECK(core.network() == nullptr); + } + + SECTION("Network can be set and retrieved") { + auto network = std::make_unique(network::config::Config{}); + auto* attached = network.get(); + core.set_network(std::move(network)); + CHECK(core.network() == attached); + } + + if (std::filesystem::exists(db_path)) + std::filesystem::remove(db_path); +} diff --git a/tests/test_core_schema.cpp b/tests/test_core_schema.cpp new file mode 100644 index 000000000..c1949e36d --- /dev/null +++ b/tests/test_core_schema.cpp @@ -0,0 +1,163 @@ +#include +#include + +#include "schema_fingerprint.hpp" +#include "test_helper.hpp" +#include "test_schema_registry.hpp" + +using namespace session; +using namespace session::core; + +namespace { + +bool table_exists(Core& core, std::string_view name) { + return TestHelper::db_conn(core) + .prepared_maybe_get( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", name) + .has_value(); +} + +} // namespace + +TEST_CASE( + "schema_extension: migrations are applied and recorded under the owner prefix", + "[core][schema]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_schema", 7)); + std::filesystem::remove(path); + + { + Core core{path, schema_extension{"testext", session::test::schema::MIGRATIONS}}; + + CHECK(table_exists(core, "ext_thing")); + // Core's own schema must still be present despite the extension reusing a name. + CHECK(table_exists(core, "globals")); + CHECK(table_exists(core, "ext_globals")); + + CHECK(TestHelper::migration_applied(core, "testext:000_ext_thing")); + CHECK(TestHelper::migration_applied(core, "testext:000_globals")); + CHECK(TestHelper::migration_applied(core, "@created")); + CHECK(!TestHelper::migration_applied(core, "000_ext_thing")); + } + + // Reopening must not re-apply them. + { + Core core{path, schema_extension{"testext", session::test::schema::MIGRATIONS}}; + CHECK(table_exists(core, "ext_thing")); + } + + std::error_code ec; + std::filesystem::remove(path, ec); +} + +TEST_CASE("schema_extension: migrations order by name, not filename", "[core][schema]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_schema", 7)); + std::filesystem::remove(path); + + // tests/schema/ holds 001_ordering.sql plus 001_ordering+002.sql, which ALTERs the table the + // first one creates. '+' sorts below '.', so ordering by filename would run the addendum + // first and Core construction would throw here rather than reaching the checks below. + Core core{path, schema_extension{"testext", session::test::schema::MIGRATIONS}}; + + CHECK(TestHelper::migration_applied(core, "testext:001_ordering")); + CHECK(TestHelper::migration_applied(core, "testext:001_ordering+002")); + + auto cols = TestHelper::db_conn(core).get_columns("ext_ordering"); + CHECK(std::ranges::any_of(cols, [](const auto& c) { return c.name == "added_later"; })); + + std::error_code ec; + std::filesystem::remove(path, ec); +} + +TEST_CASE("schema_extension: full_schema matches replaying the migrations", "[core][schema]") { + // The whole point of full_schema.sql: it must land in exactly the same place the migration + // chain does, or fresh installs and upgraded ones diverge -- silently, since whoever makes the + // change already has an upgraded database and never sees the fresh path. + auto build = [](std::string_view full_schema) { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_drift", 7)); + std::filesystem::remove(path); + + std::string fingerprint; + { + Core core{ + path, + schema_extension{"testext", session::test::schema::MIGRATIONS, full_schema}}; + auto conn = TestHelper::db_conn(core); + fingerprint = session::test::schema_fingerprint(conn); + } + + std::error_code ec; + std::filesystem::remove(path, ec); + return fingerprint; + }; + + auto from_full = build(session::test::schema::FULL_SCHEMA); + auto from_chain = build(""); // no full schema, so every migration runs + + CHECK(from_full == from_chain); + + // Guard against the comparison passing because both sides are empty. + CHECK(from_full.find("table ext_ordering") != std::string::npos); + CHECK(from_full.find("added_later") != std::string::npos); +} + +TEST_CASE( + "schema_extension: full_schema records migrations without running them", "[core][schema]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_drift", 7)); + std::filesystem::remove(path); + + { + Core core{ + path, + schema_extension{ + "testext", + session::test::schema::MIGRATIONS, + session::test::schema::FULL_SCHEMA}}; + + // Every migration is marked applied even though none ran, so a later reopen -- and any + // migration added after this point -- behaves as if the chain had been replayed. + CHECK(TestHelper::migration_applied(core, "testext:000_ext_thing")); + CHECK(TestHelper::migration_applied(core, "testext:001_ordering")); + CHECK(TestHelper::migration_applied(core, "testext:001_ordering+002")); + CHECK(table_exists(core, "ext_ordering")); + } + + // Reopening must not now try to run any of them against the already-built schema. + { + Core core{ + path, + schema_extension{ + "testext", + session::test::schema::MIGRATIONS, + session::test::schema::FULL_SCHEMA}}; + CHECK(table_exists(core, "ext_ordering")); + } + + std::error_code ec; + std::filesystem::remove(path, ec); +} + +TEST_CASE("schema_extension: rejects unusable owners", "[core][schema]") { + auto path = std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_schema", 7)); + std::filesystem::remove(path); + + CHECK_THROWS_AS( + Core(path, schema_extension{"", session::test::schema::MIGRATIONS}), + std::invalid_argument); + CHECK_THROWS_AS( + Core(path, schema_extension{"has:colon", session::test::schema::MIGRATIONS}), + std::invalid_argument); + // Two sets under one owner would let their names collide. + CHECK_THROWS_AS( + Core(path, + schema_extension{"dup", session::test::schema::MIGRATIONS}, + schema_extension{"dup", session::test::schema::MIGRATIONS}), + std::invalid_argument); + + std::error_code ec; + std::filesystem::remove(path, ec); +} diff --git a/tests/test_curve25519.cpp b/tests/test_curve25519.cpp index 14323dd25..ad03537ba 100644 --- a/tests/test_curve25519.cpp +++ b/tests/test_curve25519.cpp @@ -1,51 +1,47 @@ #include #include +#include +#include #include #include "session/curve25519.h" -#include "session/curve25519.hpp" #include "utils.hpp" +using namespace session; +using namespace session::literals; + TEST_CASE("X25519 key pair generation", "[curve25519][keypair]") { - auto kp1 = session::curve25519::curve25519_key_pair(); - auto kp2 = session::curve25519::curve25519_key_pair(); + auto [pk1, sk1] = x25519::keypair(); + auto [pk2, sk2] = x25519::keypair(); - CHECK(kp1.first.size() == 32); - CHECK(kp1.second.size() == 32); - CHECK(kp1.first != kp2.first); - CHECK(kp1.second != kp2.second); + CHECK(pk1.size() == 32); + CHECK(sk1.size() == 32); + CHECK(pk1 != pk2); + CHECK(sk1 != sk2); } TEST_CASE("X25519 conversion", "[curve25519][to curve25519 pubkey]") { - using namespace session; - - auto ed_pk1 = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; - auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hexbytes; + auto ed_pk1 = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; + auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; - auto x_pk1 = curve25519::to_curve25519_pubkey(to_span(ed_pk1)); - auto x_pk2 = curve25519::to_curve25519_pubkey(to_span(ed_pk2)); + auto x_pk1 = ed25519::pk_to_x25519(ed_pk1); + auto x_pk2 = ed25519::pk_to_x25519(ed_pk2); - CHECK(oxenc::to_hex(x_pk1.begin(), x_pk1.end()) == - "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - CHECK(oxenc::to_hex(x_pk2.begin(), x_pk2.end()) == - "aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); + CHECK(to_hex(x_pk1) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); + CHECK(to_hex(x_pk2) == "aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); } TEST_CASE("X25519 conversion", "[curve25519][to curve25519 seckey]") { - using namespace session; - auto ed_sk1 = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab78862834829a" - "87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hexbytes; + "87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hex_b; auto ed_sk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876cd83ca3d13a" - "d8a954d5011aa7861abe3a29ac25b70c4ed5234aff74d34ef5786"_hexbytes; - auto x_sk1 = curve25519::to_curve25519_seckey(to_span(ed_sk1)); - auto x_sk2 = curve25519::to_curve25519_seckey(to_span(ed_sk2)); - - CHECK(oxenc::to_hex(x_sk1.begin(), x_sk1.end()) == - "207e5d97e761300f96c10adc11efdd6d5c15188a9a7682ec05b30ca017e9b447"); - CHECK(oxenc::to_hex(x_sk2.begin(), x_sk2.end()) == - "904943eff27142a8e5cd37c84e2437c9979a560b044bf9a65a8d644b325fe56a"); + "d8a954d5011aa7861abe3a29ac25b70c4ed5234aff74d34ef5786"_hex_b; + auto x_sk1 = ed25519::sk_to_x25519(ed_sk1); + auto x_sk2 = ed25519::sk_to_x25519(ed_sk2); + + CHECK(to_hex(x_sk1) == "207e5d97e761300f96c10adc11efdd6d5c15188a9a7682ec05b30ca017e9b447"); + CHECK(to_hex(x_sk2) == "904943eff27142a8e5cd37c84e2437c9979a560b044bf9a65a8d644b325fe56a"); } diff --git a/tests/test_dm_receive.cpp b/tests/test_dm_receive.cpp new file mode 100644 index 000000000..cb48d79b3 --- /dev/null +++ b/tests/test_dm_receive.cpp @@ -0,0 +1,431 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "session/crypto/ed25519.hpp" +#include "test_helper.hpp" + +using namespace session; +using namespace session::core; +using namespace std::literals; +using namespace oxenc::literals; + +namespace { + +// Fixed sender seed, shared across all test cases. +constexpr auto SENDER_SEED = + "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"_hex_b; + +struct SenderKeys { + b32 ed_pk; + b64 ed_sk; + b33 session_id; // 0x05-prefixed long-term X25519 pubkey + + SenderKeys() { + ed25519::keypair(ed_pk, ed_sk); + ed25519::pk_to_session_id(session_id, ed_pk); + } +}; + +// Bundles an owned data buffer with a SwarmMessage whose data span points into it, +// keeping lifetime correct: the buffer must outlive any SwarmMessage referencing it. +struct OwnedMessage { + std::vector data; + SwarmMessage msg; + + explicit OwnedMessage( + std::span d, + std::string hash = "testhash", + sys_ms ts = from_epoch_ms(1000), + sys_ms exp = from_epoch_ms(9999)) : + data{d.begin(), d.end()}, msg{data, std::move(hash), ts, exp} {} +}; + +} // namespace + +// ── V1 happy path ──────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("_handle_direct_messages: v1 receive", "[core][dm]") { + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + // Minimal valid SessionProtos::Content: field 15 (sigTimestamp) = 1. + constexpr auto plaintext = "7801"_hex_b; + auto encoded = + encode_dm_v1(plaintext, sender.ed_sk, clock_now_ms(), recip_session_id, std::nullopt); + + OwnedMessage om{std::span{encoded}, "hash_v1", from_epoch_ms(1234), from_epoch_ms(9999)}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + + REQUIRE(failures.empty()); + REQUIRE(received.size() == 1); + const auto& msg = received[0]; + CHECK(msg.hash == "hash_v1"); + CHECK(msg.timestamp == from_epoch_ms(1234)); + CHECK(msg.expiry == from_epoch_ms(9999)); + CHECK(msg.version == 1); + CHECK(msg.sender_session_id == sender.session_id); + CHECK(std::ranges::equal(msg.content, plaintext)); + CHECK_FALSE(msg.pro_signature.has_value()); +} + +// ── V2 happy path ──────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("_handle_direct_messages: v2 receive", "[core][dm]") { + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + // Trigger account key generation before querying the pubkeys. + recipient->devices.active_account_keys(); + + auto [x25519_bytes, mlkem_bytes] = TestHelper::active_account_pubkeys(*recipient); + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + constexpr auto content = "deadbeef"_hex_b; + auto ct = encrypt_for_recipient_v2( + sender.ed_sk, recip_session_id, x25519_bytes, mlkem_bytes, content, std::nullopt); + + OwnedMessage om{std::span{ct}, "hash_v2", from_epoch_ms(5678), from_epoch_ms(8888)}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + + REQUIRE(failures.empty()); + REQUIRE(received.size() == 1); + const auto& msg = received[0]; + CHECK(msg.hash == "hash_v2"); + CHECK(msg.timestamp == from_epoch_ms(5678)); + CHECK(msg.expiry == from_epoch_ms(8888)); + CHECK(msg.version == 2); + CHECK(msg.sender_session_id == sender.session_id); + CHECK(std::ranges::equal(msg.content, content)); + CHECK_FALSE(msg.pro_signature.has_value()); + CHECK(msg.pfs_encrypted); +} + +// ── Failure paths ──────────────────────────────────────────────────────────────────────────────── + +TEST_CASE("_handle_direct_messages: failure paths", "[core][dm]") { + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + + auto deliver = [&](std::span data) { + OwnedMessage om{data}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + }; + + SECTION("empty data → bad_format") { + deliver(std::span{}); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::bad_format); + } + + SECTION("0x00 0x03 → unknown_version") { + deliver("0003010203"_hex_b); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::unknown_version); + } + + SECTION("v2 too short for prefix decryption → bad_format") { + // Prefix decryption needs at least version(2) + ki(2) + ephemeral_E(32) = 36 bytes. + deliver("00020102030405060708"_hex_b); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::bad_format); + } + + SECTION("v2 key indicator matches no account key → no_pfs_key") { + // Encrypt for a different Core; the key indicator will be unrecognisable to recipient. + TempCore other; + other->devices.active_account_keys(); + auto [x25519_bytes, mlkem_bytes] = TestHelper::active_account_pubkeys(*other); + b33 other_session_id; + std::ranges::copy(other->globals.session_id(), other_session_id.begin()); + + auto ct = encrypt_for_recipient_v2( + sender.ed_sk, + other_session_id, + x25519_bytes, + mlkem_bytes, + "01"_hex_b, + std::nullopt); + deliver(std::span{ct}); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::no_pfs_key); + } + + SECTION("v2 AEAD MAC corrupted → no_pfs_key") { + // Encrypt a valid v2 message for recipient, then corrupt the xchacha ciphertext tail to + // cause MAC authentication failure on both the PFS key loop and the non-PFS fallback. + // Both paths throw DecryptV2Error, so no_pfs_key is fired (nothing could decrypt it). + recipient->devices.active_account_keys(); + auto [x25519_bytes, mlkem_bytes] = TestHelper::active_account_pubkeys(*recipient); + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + auto ct = encrypt_for_recipient_v2( + sender.ed_sk, + recip_session_id, + x25519_bytes, + mlkem_bytes, + "01"_hex_b, + std::nullopt); + // Wire format: [0,1]=version, [2,3]=ki, [4,35]=E, [36,1123]=mlkem_ct, [1124+]=xchacha. + // Flip the final byte of the xchacha ciphertext to corrupt the AEAD tag. + REQUIRE(ct.size() > 1124 + 16); + ct.back() ^= std::byte{0xff}; + deliver(std::span{ct}); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::no_pfs_key); + } + + SECTION("v1 malformed ciphertext → decrypt_failed") { + deliver("0102030405060708"_hex_b); + CHECK(received.empty()); + REQUIRE(failures.size() == 1); + CHECK(failures[0] == MessageDecryptFailure::decrypt_failed); + } +} + +// ── Non-PFS fallback ───────────────────────────────────────────────────────────────────────────── + +TEST_CASE("_handle_direct_messages: v2 non-PFS fallback receive", "[core][dm]") { + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + // Do NOT call active_account_keys() — sender has no PFS keys for this recipient. + + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + constexpr auto content = "cafebabe"_hex_b; + auto ct = encrypt_for_recipient_v2_nopfs(sender.ed_sk, recip_session_id, content, std::nullopt); + + OwnedMessage om{std::span{ct}, "hash_nopfs", from_epoch_ms(3333), from_epoch_ms(7777)}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + + REQUIRE(failures.empty()); + REQUIRE(received.size() == 1); + const auto& msg = received[0]; + CHECK(msg.hash == "hash_nopfs"); + CHECK(msg.timestamp == from_epoch_ms(3333)); + CHECK(msg.expiry == from_epoch_ms(7777)); + CHECK(msg.version == 2); + CHECK(msg.sender_session_id == sender.session_id); + CHECK(std::ranges::equal(msg.content, content)); + CHECK_FALSE(msg.pro_signature.has_value()); + CHECK_FALSE(msg.pfs_encrypted); +} + +TEST_CASE( + "_handle_direct_messages: v2 non-PFS fallback succeeds when ki collides with PFS key", + "[core][dm]") { + // A non-PFS message whose decrypted ki happens to match the 2-byte ML-KEM prefix of one of + // the recipient's real PFS account keys. The PFS key loop runs but throws DecryptV2Error + // (wrong key derivation); the non-PFS fallback then succeeds. + // + // The ki is XOR-encrypted: wire_ki = plaintext_ki ⊕ kiss, where kiss is derived from the + // ephemeral key pair. decrypt_incoming_v2_prefix recovers plaintext_ki. We construct + // the collision deterministically by patching the wire_ki bytes after encrypting: + // new_wire_ki[i] = wire_ki[i] ⊕ plaintext_ki[i] ⊕ target_ki[i] + // which sets plaintext_ki to target_ki without touching E or the ciphertext body. + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + recipient->devices.active_account_keys(); + + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + auto seed_access = recipient->globals.account_seed(); + auto x25519_sec = seed_access.x25519_key(); + std::span x25519_pub{recip_session_id.data() + 1, 32}; + + // The target ki is the first 2 bytes of the recipient's active ML-KEM public key. + auto [x25519_bytes, mlkem_bytes] = TestHelper::active_account_pubkeys(*recipient); + std::array target_ki{mlkem_bytes[0], mlkem_bytes[1]}; + + constexpr auto content = "deadc0de"_hex_b; + auto ct = encrypt_for_recipient_v2_nopfs(sender.ed_sk, recip_session_id, content, std::nullopt); + + // Recover the current plaintext ki so we can XOR it out and XOR the target in. + auto current_ki = decrypt_incoming_v2_prefix(x25519_sec, x25519_pub, ct); + ct[2] ^= current_ki[0] ^ target_ki[0]; + ct[3] ^= current_ki[1] ^ target_ki[1]; + + // Verify the patch: the decrypted ki should now equal target_ki. + REQUIRE(decrypt_incoming_v2_prefix(x25519_sec, x25519_pub, ct) == target_ki); + + OwnedMessage om{std::span{ct}, "hash_ki_collision"}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + + REQUIRE(failures.empty()); + REQUIRE(received.size() == 1); + CHECK(std::ranges::equal(received[0].content, content)); + CHECK_FALSE(received[0].pfs_encrypted); +} + +// ── PFS ki-prefix collision within the loop ────────────────────────────────────────────────────── + +TEST_CASE( + "_handle_direct_messages: PFS decryption succeeds when ki collides within the PFS loop", + "[core][dm]") { + // Verify that when active_account_keys(ki) returns multiple candidates (because two account + // keys share the same 2-byte ML-KEM prefix), the loop continues past a DecryptV2Error on the + // wrong key and succeeds with the correct key. + // + // We find a colliding pair via the birthday paradox: rotating account keys until any two + // generated keys share the same 2-byte ML-KEM prefix. Expected ~321 rotations on average + // (sqrt(pi * 65536 / 2)), each taking < 1 ms, so the total cost is well under a second. + SenderKeys sender; + + std::vector received; + std::vector failures; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + cbs.message_decrypt_failed = [&](const SwarmMessage&, MessageDecryptFailure r) { + failures.push_back(r); + }; + + TempCore recipient{cbs}; + + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + // Generate the first account key and record (prefix → pubkeys) as we rotate. + recipient->devices.active_account_keys(); + + using Prefix = std::array; + using PubkeyPair = std::pair, std::array>; + std::map seen; + + // Record the current active key; returns the earlier key's pubkeys if its prefix collides. + auto record_active = [&]() -> std::optional { + auto [x, m] = TestHelper::active_account_pubkeys(*recipient); + Prefix pfx{m[0], m[1]}; + if (auto it = seen.find(pfx); it != seen.end()) + return it->second; + seen.emplace(pfx, PubkeyPair{x, m}); + return std::nullopt; + }; + + record_active(); + + PubkeyPair target_pubkeys; + bool found = false; + for (int i = 0; i < 500'000 && !found; ++i) { + recipient->devices.rotate_account_keys(); + if (auto match = record_active()) { + target_pubkeys = *match; + found = true; + } + } + REQUIRE(found); + + // Encrypt with the earlier (now-rotated) key that shares the active key's ki prefix. + // active_account_keys(ki) returns [active_key (wrong), rotated_target (right)], so Core + // tries the wrong key first (DecryptV2Error), then succeeds with the right one. + constexpr auto content = "feedface"_hex_b; + auto ct = encrypt_for_recipient_v2( + sender.ed_sk, + recip_session_id, + target_pubkeys.first, + target_pubkeys.second, + content, + std::nullopt); + + OwnedMessage om{std::span{ct}, "hash_ki_pfs_collision"}; + recipient->receive_messages({&om.msg, 1}, config::Namespace::Default, true); + + REQUIRE(failures.empty()); + REQUIRE(received.size() == 1); + CHECK(std::ranges::equal(received[0].content, content)); + CHECK(received[0].pfs_encrypted); +} + +// ── Callback exception safety ──────────────────────────────────────────────────────────────────── + +TEST_CASE( + "_handle_direct_messages: exception in message_received is swallowed and processing " + "continues", + "[core][dm]") { + SenderKeys sender; + + int call_count = 0; + callbacks cbs; + cbs.message_received = [&](ReceivedMessage&&) { + ++call_count; + throw std::runtime_error("deliberate test exception"); + }; + + TempCore recipient{cbs}; + + b33 recip_session_id; + std::ranges::copy(recipient->globals.session_id(), recip_session_id.begin()); + + // Minimal valid SessionProtos::Content: field 15 (sigTimestamp) = 1. + constexpr auto plaintext = "7801"_hex_b; + auto e1 = encode_dm_v1(plaintext, sender.ed_sk, clock_now_ms(), recip_session_id, std::nullopt); + auto e2 = encode_dm_v1(plaintext, sender.ed_sk, clock_now_ms(), recip_session_id, std::nullopt); + + OwnedMessage om1{std::span{e1}, "h1"}; + OwnedMessage om2{std::span{e2}, "h2"}; + std::array msgs{om1.msg, om2.msg}; + + // The thrown exception must not propagate, and both messages must reach the callback. + CHECK_NOTHROW(recipient->receive_messages(msgs, config::Namespace::Default, true)); + CHECK(call_count == 2); +} diff --git a/tests/test_dm_send.cpp b/tests/test_dm_send.cpp new file mode 100644 index 000000000..b23ea1115 --- /dev/null +++ b/tests/test_dm_send.cpp @@ -0,0 +1,452 @@ +#include + +#include +#include +#include +#include +#include + +#include "test_helper.hpp" + +using namespace session; +using namespace session::core; +using namespace std::literals; +using namespace oxenc::literals; + +namespace { + +// Returns the session_id of a Core as a std::byte array. +std::array sid_bytes(Core& c) { + std::array result; + auto sid = c.globals.session_id(); + std::memcpy(result.data(), sid.data(), 33); + return result; +} + +// Minimal valid SessionProtos::Content protobuf: field 15 (sigTimestamp) = 1. +constexpr auto MINIMAL_CONTENT = "7801"_hex_b; + +// A valid session ID for tests that don't need actual decryption on the other side. +constexpr auto DUMMY_SID = + "05fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; + +std::span content_bytes() { + static const auto bytes = std::as_bytes(std::span{MINIMAL_CONTENT}); + return bytes; +} + +} // namespace + +// ── V2 PFS send + receive round-trip ──────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: v2 PFS round-trip", "[core][send_dm]") { + std::vector received; + std::vector statuses; + + callbacks sender_cbs; + sender_cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { + statuses.push_back(s); + }; + + callbacks recip_cbs; + recip_cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + + TempCore sender{sender_cbs}; + TempCore recipient{recip_cbs}; + auto* net = attach_mock_network(*sender); + + recipient->devices.active_account_keys(); + auto [x25519_pub, mlkem_pub] = TestHelper::active_account_pubkeys(*recipient); + auto recip_sid = sid_bytes(*recipient); + TestHelper::seed_pfs_cache(*sender, recip_sid, x25519_pub, mlkem_pub); + + auto msg_id = sender->send_dm(recip_sid, content_bytes(), clock_now_ms()); + CHECK(msg_id == 1); + + auto sent = stores(*net); + REQUIRE(sent.size() == 1); + CHECK(store_body(*sent[0])["namespace"] == static_cast(config::Namespace::Default)); + auto payload = store_payload(*sent[0]); + REQUIRE(accept_stores(*net) == 1); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[0] == MessageSendStatus::sending); + CHECK(statuses[1] == MessageSendStatus::success); + + // What went onto the wire, fed back in as the recipient's swarm would deliver it. + SwarmMessage sm; + sm.data = payload; + sm.hash = "send_test_hash"; + sm.timestamp = clock_now_ms(); + sm.expiry = clock_now_ms() + 24h; + + recipient->receive_messages({&sm, 1}, config::Namespace::Default, true); + + REQUIRE(received.size() == 1); + CHECK(received[0].version == 2); + CHECK(received[0].pfs_encrypted); + CHECK(std::ranges::equal(received[0].content, MINIMAL_CONTENT)); +} + +// ── V1 fallback (NAK, force_v2=false) ─────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: v1 fallback on NAK", "[core][send_dm]") { + std::vector received; + std::vector statuses; + + callbacks sender_cbs; + sender_cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { + statuses.push_back(s); + }; + + callbacks recip_cbs; + recip_cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + + TempCore sender{sender_cbs}; + TempCore recipient{recip_cbs}; + auto* net = attach_mock_network(*sender); + + auto recip_sid = sid_bytes(*recipient); + TestHelper::seed_pfs_nak(*sender, recip_sid); + + sender->send_dm(recip_sid, content_bytes(), clock_now_ms()); + + auto sent = stores(*net); + REQUIRE(sent.size() == 1); + auto payload = store_payload(*sent[0]); + REQUIRE(accept_stores(*net) == 1); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[0] == MessageSendStatus::sending); + CHECK(statuses[1] == MessageSendStatus::success); + + SwarmMessage sm; + sm.data = payload; + sm.hash = "v1_hash"; + sm.timestamp = clock_now_ms(); + sm.expiry = clock_now_ms() + 24h; + + recipient->receive_messages({&sm, 1}, config::Namespace::Default, true); + + REQUIRE(received.size() == 1); + CHECK(received[0].version == 1); + CHECK_FALSE(received[0].pfs_encrypted); +} + +// ── V2 non-PFS (force_v2=true, NAK) ──────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: v2 non-PFS with force_v2", "[core][send_dm]") { + std::vector received; + std::vector statuses; + + callbacks sender_cbs; + sender_cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { + statuses.push_back(s); + }; + + callbacks recip_cbs; + recip_cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + + TempCore sender{sender_cbs}; + TempCore recipient{recip_cbs}; + auto* net = attach_mock_network(*sender); + + auto recip_sid = sid_bytes(*recipient); + TestHelper::seed_pfs_nak(*sender, recip_sid); + + sender->send_dm( + recip_sid, content_bytes(), clock_now_ms(), std::nullopt, 14 * 24h, /*force_v2=*/true); + + auto sent = stores(*net); + REQUIRE(sent.size() == 1); + auto payload = store_payload(*sent[0]); + REQUIRE(accept_stores(*net) == 1); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[0] == MessageSendStatus::sending); + CHECK(statuses[1] == MessageSendStatus::success); + + SwarmMessage sm; + sm.data = payload; + sm.hash = "nopfs_hash"; + sm.timestamp = clock_now_ms(); + sm.expiry = clock_now_ms() + 24h; + + recipient->receive_messages({&sm, 1}, config::Namespace::Default, true); + + REQUIRE(received.size() == 1); + CHECK(received[0].version == 2); + CHECK_FALSE(received[0].pfs_encrypted); +} + +// ── No network error ──────────────────────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: no_network once the keys are known but nothing can send", "[core][send_dm]") { + std::vector statuses; + + callbacks cbs; + cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { statuses.push_back(s); }; + + TempCore sender{cbs}; + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[0] == MessageSendStatus::sending); + CHECK(statuses[1] == MessageSendStatus::no_network); +} + +// ── No network, no cache → immediate no_network ──────────────────────────────────────────────── + +TEST_CASE("send_dm: no_network when no cache and no network", "[core][send_dm]") { + std::vector statuses; + + callbacks cbs; + cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { statuses.push_back(s); }; + + TempCore sender{cbs}; + + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + // No cache entry and no network → immediate no_network. + REQUIRE(statuses.size() == 1); + CHECK(statuses[0] == MessageSendStatus::no_network); +} + +// ── The store request a send produces ─────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: the store names the recipient, namespace and ttl", "[core][send_dm]") { + TempCore sender{}; + auto* net = attach_mock_network(*sender); + + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + auto custom_ttl = std::chrono::milliseconds{7 * 24h}; + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms(), std::nullopt, custom_ttl); + + auto sent = stores(*net); + REQUIRE(sent.size() == 1); + auto body = store_body(*sent[0]); + CHECK(body["pubkey"] == oxenc::to_hex(DUMMY_SID)); + CHECK(body["namespace"] == static_cast(config::Namespace::Default)); + CHECK(body["ttl"] == custom_ttl.count()); +} + +// ── Network error status ──────────────────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: network_error when store fails", "[core][send_dm]") { + std::vector statuses; + + callbacks cbs; + cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { statuses.push_back(s); }; + + TempCore sender{cbs}; + auto* net = attach_mock_network(*sender); + + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + REQUIRE(answer_stores(*net, false) == 1); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[0] == MessageSendStatus::sending); + CHECK(statuses[1] == MessageSendStatus::network_error); +} + +// ── Monotonic message IDs ─────────────────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: message IDs are monotonically increasing", "[core][send_dm]") { + TempCore sender{}; + auto* net = attach_mock_network(*sender); + + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + auto id1 = sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + auto id2 = sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + auto id3 = sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + CHECK(id1 == 1); + CHECK(id2 == 2); + CHECK(id3 == 3); +} + +// ── Success is the swarm's answer, not the dispatch ───────────────────────────────────────────── + +TEST_CASE("send_dm: success waits for the store to be answered", "[core][send_dm]") { + std::vector statuses; + + callbacks cbs; + cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { statuses.push_back(s); }; + + TempCore sender{cbs}; + auto* net = attach_mock_network(*sender); + + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + // `sending` means dispatched, and stops there: nothing has come back from the swarm yet. + REQUIRE(statuses.size() == 1); + CHECK(statuses[0] == MessageSendStatus::sending); + REQUIRE(stores(*net).size() == 1); + + REQUIRE(accept_stores(*net) == 1); + + REQUIRE(statuses.size() == 2); + CHECK(statuses[1] == MessageSendStatus::success); +} + +// ── Content protobuf overload ─────────────────────────────────────────────────────────────────── + +TEST_CASE("send_dm: Content overload round-trip", "[core][send_dm]") { + std::vector received; + + callbacks recip_cbs; + recip_cbs.message_received = [&](ReceivedMessage&& m) { received.push_back(std::move(m)); }; + + TempCore sender{}; + TempCore recipient{recip_cbs}; + auto* net = attach_mock_network(*sender); + + recipient->devices.active_account_keys(); + auto [x25519_pub, mlkem_pub] = TestHelper::active_account_pubkeys(*recipient); + auto recip_sid = sid_bytes(*recipient); + TestHelper::seed_pfs_cache(*sender, recip_sid, x25519_pub, mlkem_pub); + + auto ts = clock_now_ms(); + SessionProtos::Content content; + content.mutable_datamessage()->set_body("hello from the Content overload"); + + sender->send_dm(recip_sid, content, ts); + + auto sent = stores(*net); + REQUIRE(sent.size() == 1); + auto payload = store_payload(*sent[0]); + + SwarmMessage sm; + sm.data = payload; + sm.hash = "content_overload_hash"; + sm.timestamp = ts; + sm.expiry = ts + 24h; + + recipient->receive_messages({&sm, 1}, config::Namespace::Default, true); + + REQUIRE(received.size() == 1); + SessionProtos::Content decoded; + REQUIRE(decoded.ParseFromArray( + received[0].content.data(), static_cast(received[0].content.size()))); + CHECK(decoded.datamessage().body() == "hello from the Content overload"); + + // An unset sigTimestamp is filled in from sent_timestamp. + CHECK(decoded.sigtimestamp() == static_cast(ts.time_since_epoch().count())); +} + +TEST_CASE("send_dm: Content overload preserves a matching sigTimestamp", "[core][send_dm]") { + TempCore sender{}; + auto* net = attach_mock_network(*sender); + + TestHelper::seed_pfs_nak(*sender, DUMMY_SID); + + auto ts = clock_now_ms(); + SessionProtos::Content content; + content.set_sigtimestamp(static_cast(ts.time_since_epoch().count())); + content.mutable_datamessage()->set_body("explicit timestamp"); + + CHECK_NOTHROW(sender->send_dm(DUMMY_SID, content, ts)); + CHECK(stores(*net).size() == 1); +} + +TEST_CASE("send_dm: Content overload rejects a mismatched sigTimestamp", "[core][send_dm]") { + TempCore sender{}; + + auto ts = clock_now_ms(); + SessionProtos::Content content; + content.set_sigtimestamp(static_cast(ts.time_since_epoch().count()) + 5000); + content.mutable_datamessage()->set_body("mismatched"); + + CHECK_THROWS_AS(sender->send_dm(DUMMY_SID, content, ts), std::invalid_argument); +} + +// ── Sends queued behind a PFS key fetch ───────────────────────────────────────────────────────── + +TEST_CASE( + "send_dm: a send queued behind a key fetch is released when the fetch settles", + "[core][send_dm]") { + std::vector statuses; + std::vector fetches; + + callbacks cbs; + cbs.message_send_status = [&](int64_t, MessageSendStatus s, auto) { statuses.push_back(s); }; + cbs.pfs_keys_fetched = [&](std::span, PfsKeyFetch r) { + fetches.push_back(r); + }; + + TempCore sender{cbs}; + auto* net = attach_mock_network(*sender); + + // Nothing cached for this recipient, so the send is queued behind a key fetch. + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + REQUIRE(statuses.size() == 1); + CHECK(statuses[0] == MessageSendStatus::awaiting_keys); + CHECK(stores(*net).empty()); + REQUIRE(net->sent_requests.size() == 1); + CHECK(net->sent_requests[0].request.endpoint == "retrieve"); + + // A *failed* fetch must still release the send rather than stranding it forever. + REQUIRE(fail_retrieves(*net) == 1); + + REQUIRE(fetches.size() == 1); + CHECK(fetches[0] == PfsKeyFetch::failed); + REQUIRE(accept_stores(*net) == 1); + REQUIRE(statuses.size() >= 2); + CHECK(statuses.back() == MessageSendStatus::success); +} + +TEST_CASE( + "send_dm: every send queued for one recipient is released by a single fetch", + "[core][send_dm]") { + std::vector fetches; + + callbacks cbs; + cbs.pfs_keys_fetched = [&](std::span, PfsKeyFetch r) { + fetches.push_back(r); + }; + + TempCore sender{cbs}; + auto* net = attach_mock_network(*sender); + + for (int i = 0; i < 3; i++) + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + + CHECK(stores(*net).empty()); + REQUIRE(!net->sent_requests.empty()); + + // Settling one fetch drains the whole queue for that recipient, even though each send asked + // for the keys in its own right. + REQUIRE(fail_retrieves(*net, 1) == 1); + + CHECK(stores(*net).size() == 3); + CHECK(fetches.size() == 1); +} + +TEST_CASE( + "send_dm: a throwing pfs_keys_fetched callback does not strand queued sends", + "[core][send_dm]") { + callbacks cbs; + // Callbacks are not permitted to throw, but a buggy one must not take the queued sends with + // it: the flush happens regardless. + cbs.pfs_keys_fetched = [](std::span, PfsKeyFetch) { + throw std::runtime_error{"buggy application callback"}; + }; + + TempCore sender{cbs}; + auto* net = attach_mock_network(*sender); + + sender->send_dm(DUMMY_SID, content_bytes(), clock_now_ms()); + REQUIRE(!net->sent_requests.empty()); + + CHECK_NOTHROW(fail_retrieves(*net)); + CHECK(stores(*net).size() == 1); +} diff --git a/tests/test_ed25519.cpp b/tests/test_ed25519.cpp index e82092af1..e3a1ac0aa 100644 --- a/tests/test_ed25519.cpp +++ b/tests/test_ed25519.cpp @@ -3,12 +3,13 @@ #include #include -#include "session/ed25519.hpp" +#include "session/crypto/ed25519.hpp" +#include "session/pro_backend.hpp" TEST_CASE("Ed25519 key pair generation", "[ed25519][keypair]") { // Generate two random key pairs and make sure they don't match - auto [pk1, sk1] = session::ed25519::ed25519_key_pair(); - auto [pk2, sk2] = session::ed25519::ed25519_key_pair(); + auto [pk1, sk1] = session::ed25519::keypair(); + auto [pk2, sk2] = session::ed25519::keypair(); CHECK(pk1.size() == 32); CHECK(sk1.size() == 64); @@ -20,14 +21,12 @@ TEST_CASE("Ed25519 key pair generation seed", "[ed25519][keypair]") { using namespace session; constexpr auto ed_seed1 = - "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_u; + "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; constexpr auto ed_seed2 = - "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_u; - constexpr auto ed_seed_invalid = "010203040506070809"_hex_u; + "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; - auto [pk1, sk1] = session::ed25519::ed25519_key_pair(ed_seed1); - auto [pk2, sk2] = session::ed25519::ed25519_key_pair(ed_seed2); - CHECK_THROWS(session::ed25519::ed25519_key_pair(ed_seed_invalid)); + auto [pk1, sk1] = session::ed25519::keypair(ed_seed1); + auto [pk2, sk2] = session::ed25519::keypair(ed_seed2); CHECK(pk1.size() == 32); CHECK(sk1.size() == 64); @@ -50,15 +49,13 @@ TEST_CASE("Ed25519 seed for private key", "[ed25519][seed]") { using namespace session; constexpr auto ed_sk1 = - "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab78862834829a" - "87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hex_u; + "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7" + "8862834829a87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hex_b; constexpr auto ed_sk2 = - "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_u; - constexpr auto ed_sk_invalid = "010203040506070809"_hex_u; + "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; - auto seed1 = session::ed25519::seed_for_ed_privkey(ed_sk1); - auto seed2 = session::ed25519::seed_for_ed_privkey(ed_sk2); - CHECK_THROWS(session::ed25519::seed_for_ed_privkey(ed_sk_invalid)); + auto seed1 = session::ed25519::extract_seed(ed_sk1); + auto seed2 = session::ed25519::extract_seed(ed_sk2); CHECK(oxenc::to_hex(seed1) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); @@ -95,13 +92,13 @@ TEST_CASE("Ed25519 pro key pair generation seed", "[ed25519][keypair]") { // // clang-format on - constexpr auto seed1 = "e5481635020d6f7b327e94e6d63e33a431fccabc4d2775845c43a8486a9f2884"_hex_u; - constexpr auto seed2 = "743d646706b6b04b97b752036dd6cf5f2adc4b339fcfdfb4b496f0764bb93a84"_hex_u; - constexpr auto seed_invalid = "010203040506070809"_hex_u; + constexpr auto seed1 = "e5481635020d6f7b327e94e6d63e33a431fccabc4d2775845c43a8486a9f2884"_hex_b; + constexpr auto seed2 = "743d646706b6b04b97b752036dd6cf5f2adc4b339fcfdfb4b496f0764bb93a84"_hex_b; - auto sk1 = session::ed25519::ed25519_pro_privkey_for_ed25519_seed(seed1); - auto sk2 = session::ed25519::ed25519_pro_privkey_for_ed25519_seed(seed2); - CHECK_THROWS(session::ed25519::ed25519_pro_privkey_for_ed25519_seed(seed_invalid)); + auto [pk1, sk1] = + session::ed25519::derive_subkey(seed1, session::pro_backend::pro_subkey_domain); + auto [pk2, sk2] = + session::ed25519::derive_subkey(seed2, session::pro_backend::pro_subkey_domain); CHECK(sk1.size() == 64); CHECK(sk1 != sk2); @@ -121,12 +118,12 @@ TEST_CASE("Ed25519", "[ed25519][signature]") { using namespace session; constexpr auto ed_seed = - "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_u; - constexpr auto ed_pk = "8862834829a87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hex_u; + "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; + constexpr auto ed_pk = "8862834829a87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f"_hex_b; constexpr auto ed_invalid = "010203040506070809"_hex_u; auto sig1 = session::ed25519::sign(ed_seed, to_span("hello")); - CHECK_THROWS(session::ed25519::sign(ed_invalid, to_span("hello"))); + CHECK_THROWS(session::ed25519::sign({ed_invalid.data(), ed_invalid.size()}, to_span("hello"))); auto expected_sig_hex = "e03b6e87a53d83f202f2501e9b52193dbe4a64c6503f88244948dee53271" @@ -134,6 +131,32 @@ TEST_CASE("Ed25519", "[ed25519][signature]") { CHECK(oxenc::to_hex(sig1) == expected_sig_hex); CHECK(session::ed25519::verify(sig1, ed_pk, to_span("hello"))); - CHECK_THROWS(session::ed25519::verify(ed_invalid, ed_pk, to_span("hello"))); - CHECK_THROWS(session::ed25519::verify(ed_pk, ed_invalid, to_span("hello"))); +} + +TEST_CASE("Ed25519 pubkey validity", "[ed25519][pubkey]") { + using namespace session; + + auto valid = [](std::string_view hex) { + auto bytes = oxenc::from_hex(hex); + REQUIRE(bytes.size() == 32); + return ed25519::is_valid_pubkey(to_span(bytes).first<32>()); + }; + + // A real key, and the pubkey of a known seed + CHECK(valid("8862834829a87e0afadfed763fa8785e893dbde7f2c001ff1071aa55005c347f")); + CHECK(valid(oxenc::to_hex(ed25519::keypair().first))); + + // Not a point on the curve. Most 32-byte values are not: only about 6% are points on the + // main subgroup, which is what makes this check worth doing on anything received. + CHECK_FALSE(valid("0123456789abcdef0123456789abcdef00000000000000000000000000000000")); + + // On the curve but of small order, so not a usable key: the identity and one of order 8 + CHECK_FALSE(valid("0000000000000000000000000000000000000000000000000000000000000000")); + CHECK_FALSE(valid("0100000000000000000000000000000000000000000000000000000000000000")); + CHECK_FALSE(valid("26e8958fc2b227b045c3f489f2ef98f0d5dfac05d3c63339b13802886d53fc05")); + + // An X25519 pubkey is not an Ed25519 one, which is the mistake this guards against: these two + // are the same key in its two forms, and only the Ed form is a point here. + CHECK(valid("929e33ded05e653fec04b49645117f51851f102a947e04806791be416ed76602")); + CHECK_FALSE(valid("16d6c60aebb0851de7e6f4dc0a4734671dbf80f73664c008596511454cb6576d")); } diff --git a/tests/test_encrypt.cpp b/tests/test_encrypt.cpp index 873837535..2c304e581 100644 --- a/tests/test_encrypt.cpp +++ b/tests/test_encrypt.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -15,8 +14,8 @@ using namespace session; TEST_CASE("config message encryption", "[config][encrypt]") { auto message1 = "some message 1"_bytes; - auto key1 = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hexbytes; - auto key2 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hexbytes; + auto key1 = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"_hex_b; + auto key2 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"_hex_b; auto enc1 = config::encrypt(message1, key1, "test-suite1"); CHECK(oxenc::to_hex(enc1.begin(), enc1.end()) == "f14f242a26638f3305707d1035e734577f943cd7d28af58e32637e" @@ -25,19 +24,19 @@ TEST_CASE("config message encryption", "[config][encrypt]") { CHECK(to_hex(enc2) != to_hex(enc1)); auto enc3 = config::encrypt(message1, key2, "test-suite1"); CHECK(to_hex(enc3) != to_hex(enc1)); - auto nonce = std::vector{enc1.begin() + (enc1.size() - 24), enc1.end()}; - auto nonce2 = std::vector{enc2.begin() + (enc2.size() - 24), enc2.end()}; - auto nonce3 = std::vector{enc3.begin() + (enc3.size() - 24), enc3.end()}; + auto nonce = std::vector{enc1.begin() + (enc1.size() - 24), enc1.end()}; + auto nonce2 = std::vector{enc2.begin() + (enc2.size() - 24), enc2.end()}; + auto nonce3 = std::vector{enc3.begin() + (enc3.size() - 24), enc3.end()}; CHECK(to_hex(nonce) == "af2f4860cb4d0f8ba7e09d29e31f5e4a18f65847287a54a0"); CHECK(to_hex(nonce2) == "277e639d36ba46470dfff509a68cb73d9a96386c51739bdd"); CHECK(to_hex(nonce3) == to_hex(nonce)); auto plain = config::decrypt(enc1, key1, "test-suite1"); - CHECK(plain == message1); + CHECK(std::ranges::equal(plain, message1)); CHECK_THROWS_AS(config::decrypt(enc1, key1, "test-suite2"), config::decrypt_error); CHECK_THROWS_AS(config::decrypt(enc1, key2, "test-suite1"), config::decrypt_error); - enc1[3] = '\x42'; + enc1[3] = std::byte{0x42}; CHECK_THROWS_AS(config::decrypt(enc1, key1, "test-suite1"), config::decrypt_error); } diff --git a/tests/test_format.cpp b/tests/test_format.cpp new file mode 100644 index 000000000..d712c64c2 --- /dev/null +++ b/tests/test_format.cpp @@ -0,0 +1,146 @@ +#include + +#include +#include +#include +#include + +#include "session/format.hpp" +#include "utils.hpp" + +TEST_CASE("byte span formatting - default hex", "[format]") { + CHECK(fmt::format("{}", "abcd0123"_hex_b) == "abcd0123"); + CHECK(fmt::format("{:x}", "abcd0123"_hex_b) == "abcd0123"); +} + +TEST_CASE("byte span formatting - various types", "[format]") { + auto arr = "deadbeef"_hex_b; + + SECTION("std::span with static extent") { + CHECK(fmt::format("{}", arr) == "deadbeef"); + } + + SECTION("std::span with dynamic extent") { + std::span sp{arr}; + CHECK(fmt::format("{}", sp) == "deadbeef"); + } + + SECTION("std::vector") { + std::vector vec{arr.begin(), arr.end()}; + CHECK(fmt::format("{}", vec) == "deadbeef"); + } + + SECTION("std::array") { + std::array a; + std::copy(arr.begin(), arr.end(), a.begin()); + CHECK(fmt::format("{}", a) == "deadbeef"); + } +} + +TEST_CASE("byte span formatting - empty span", "[format]") { + std::span empty; + CHECK(fmt::format("{}", empty) == ""); + CHECK(fmt::format("{:x}", empty) == ""); + CHECK(fmt::format("{:z}", empty) == "0"); + // Note: empty base64 is skipped due to an oxenc bug producing "=" for empty input + CHECK(fmt::format("{:a}", empty) == ""); + CHECK(fmt::format("{:r}", empty) == ""); +} + +TEST_CASE("byte span formatting - stripped hex", "[format]") { + SECTION("all zeros") { + CHECK(fmt::format("{:z}", "00000000"_hex_b) == "0"); + } + + SECTION("leading zeros stripped") { + CHECK(fmt::format("{:z}", "00001234"_hex_b) == "1234"); + } + + SECTION("leading zero nibble stripped") { + CHECK(fmt::format("{:z}", "000abc"_hex_b) == "abc"); + } + + SECTION("no leading zeros") { + CHECK(fmt::format("{:z}", "ff01"_hex_b) == "ff01"); + } + + SECTION("single non-zero byte with leading nibble zero") { + CHECK(fmt::format("{:z}", "0002"_hex_b) == "2"); + } +} + +TEST_CASE("byte span formatting - base32z", "[format]") { + auto val = "0001020304"_hex_b; + auto hex_result = fmt::format("{:x}", val); + auto b32z_result = fmt::format("{:a}", val); + CHECK(hex_result == "0001020304"); + CHECK(!b32z_result.empty()); + CHECK(b32z_result != hex_result); +} + +TEST_CASE("byte span formatting - base64", "[format]") { + CHECK(fmt::format("{:b}", "00010203"_hex_b) == "AAECAw=="); + CHECK(fmt::format("{:B}", "00010203"_hex_b) == "AAECAw"); +} + +TEST_CASE("byte span formatting - raw", "[format]") { + CHECK(fmt::format("{:r}", "6869"_hex_b) == "hi"); +} + +TEST_CASE("byte span formatting - ellipsis", "[format]") { + // 8 bytes = 16 hex chars: "0123456789abcdef" + auto val = "0123456789abcdef"_hex_b; + CHECK(fmt::format("{}", val) == "0123456789abcdef"); + + SECTION("truncation with tail") { + // 10 display chars: 7 leading + ellipsis + 2 trailing + CHECK(fmt::format("{:10.2}", val) == "0123456…ef"); + } + + SECTION("no truncation when value fits") { + CHECK(fmt::format("{:20.4}", val) == "0123456789abcdef"); + } + + SECTION("ellipsis with explicit mode") { + CHECK(fmt::format("{:10.2x}", val) == "0123456…ef"); + } + + SECTION("tail of zero") { + CHECK(fmt::format("{:5.0}", val) == "0123…"); + } + + SECTION("minimum ellipsis") { + CHECK(fmt::format("{:2.0}", val) == "0…"); + } +} + +TEST_CASE("byte span formatting - 32 byte key ellipsis", "[format]") { + auto key = "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"_hex_b; + auto full = fmt::format("{}", key); + CHECK(full.size() == 64); + + // Ellipsize to 12 display chars with 4-char tail: + // 7 leading + 3 bytes UTF-8 ellipsis + 4 trailing = 14 bytes + auto ellipsized = fmt::format("{:12.4}", key); + CHECK(ellipsized == "0102030…1f20"); + CHECK(ellipsized.size() == 7 + 3 + 4); +} + +TEST_CASE("byte span formatting - format errors", "[format]") { + auto val = "01"_hex_b; + + // Use fmt::runtime() to bypass compile-time format string checking + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:0}"), val), fmt::format_error); + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:5}"), val), fmt::format_error); + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:q}"), val), fmt::format_error); + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:xx}"), val), fmt::format_error); + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:3.3}"), val), fmt::format_error); + CHECK_THROWS_AS(fmt::format(fmt::runtime("{:1.0}"), val), fmt::format_error); +} + +TEST_CASE("byte span formatting - _format UDL", "[format]") { + using namespace session::literals; + auto val = "deadbeef"_hex_b; + CHECK("key: {}"_format(val) == "key: deadbeef"); + CHECK("key: {:z}"_format(val) == "key: deadbeef"); +} diff --git a/tests/test_group_info.cpp b/tests/test_group_info.cpp index 4dfea5c11..8fbc8a073 100644 --- a/tests/test_group_info.cpp +++ b/tests/test_group_info.cpp @@ -1,11 +1,11 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -14,39 +14,40 @@ using namespace std::literals; using namespace oxenc::literals; using namespace session::config; +using namespace session; TEST_CASE("Group Info settings", "[config][groups][info]") { - const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); + const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "cbd569f56fb13ea95a3f0c05c331cc24139c0090feb412069dc49fab34406ece"); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - std::vector> enc_keys{ - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hexbytes}; + std::vector> enc_keys; + enc_keys.push_back( + to_vector("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hex_b)); - groups::Info ginfo1{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Info ginfo1{ed_pk, ed_sk, std::nullopt}; // This is just for testing: normally you don't load keys manually but just make a groups::Keys // object that loads the keys into the Members object for you. for (const auto& k : enc_keys) - ginfo1.add_key(k, false); + ginfo1.add_key(std::span{k}.first<32>(), false); enc_keys.insert( enc_keys.begin(), - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hexbytes); - enc_keys.push_back("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hexbytes); - enc_keys.push_back("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"_hexbytes); - groups::Info ginfo2{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + to_vector("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hex_b)); + enc_keys.push_back( + to_vector("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hex_b)); + enc_keys.push_back( + to_vector("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"_hex_b)); + groups::Info ginfo2{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys) // Just for testing, as above. - ginfo2.add_key(k, false); + ginfo2.add_key(std::span{k}.first<32>(), false); ginfo1.set_name("GROUP Name"); CHECK(ginfo1.is_dirty()); @@ -64,7 +65,7 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { CHECK(ginfo1.needs_dump()); CHECK_FALSE(ginfo1.needs_push()); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash1", p1[0]); CHECK(ginfo2.merge(merge_configs) == std::unordered_set{{"fakehash1"s}}); CHECK_FALSE(ginfo2.needs_push()); @@ -73,7 +74,7 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { ginfo2.set_profile_pic( "http://example.com/12345", - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes); + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b); ginfo2.set_expiry_timer(1h); constexpr int64_t create_time{1682529839}; ginfo2.set_created(create_time); @@ -96,9 +97,9 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { // This fails because ginfo1 doesn't yet have the new key that ginfo2 used (bbb...) CHECK(ginfo1.merge(merge_configs) == std::unordered_set{}); - ginfo1.add_key("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hexbytes); + ginfo1.add_key("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hex_b); ginfo1.add_key( - "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hexbytes, + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hex_b, /*prepend=*/false); CHECK(ginfo1.merge(merge_configs) == std::unordered_set{{"fakehash2"s}}); @@ -108,8 +109,9 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { CHECK(ginfo1.get_name() == "Better name!"); CHECK(ginfo1.get_profile_pic().url == "http://example.com/12345"); - CHECK(ginfo1.get_profile_pic().key == - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes); + CHECK(std::ranges::equal( + ginfo1.get_profile_pic().key, + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b)); CHECK(ginfo1.get_expiry_timer() == 1h); CHECK(ginfo1.get_created() == create_time); CHECK(ginfo1.get_delete_before() == create_time + 50 * 86400); @@ -123,8 +125,9 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { CHECK(ginfo2.merge(merge_configs) == std::unordered_set{{"fakehash3"s}}); CHECK(ginfo2.get_name() == "Better name!"); CHECK(ginfo2.get_profile_pic().url == "http://example.com/12345"); - CHECK(ginfo2.get_profile_pic().key == - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes); + CHECK(std::ranges::equal( + ginfo2.get_profile_pic().key, + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b)); CHECK(ginfo2.get_expiry_timer() == 1h); CHECK(ginfo2.get_created() == create_time); CHECK(ginfo2.get_delete_before() == create_time + 50 * 86400); @@ -144,31 +147,28 @@ TEST_CASE("Group Info settings", "[config][groups][info]") { TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { - const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); + const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "cbd569f56fb13ea95a3f0c05c331cc24139c0090feb412069dc49fab34406ece"); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - std::vector> enc_keys1; + std::vector> enc_keys1; enc_keys1.push_back( - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hexbytes); - std::vector> enc_keys2; + to_vector("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hex_b)); + std::vector> enc_keys2; enc_keys2.push_back( - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hexbytes); + to_vector("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hex_b)); enc_keys2.push_back( - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hexbytes); + to_vector("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hex_b)); // This Info object has only the public key, not the priv key, and so cannot modify things: - groups::Info ginfo{session::to_span(ed_pk), std::nullopt, std::nullopt}; + groups::Info ginfo{ed_pk, std::nullopt, std::nullopt}; for (const auto& k : enc_keys1) // Just for testing, as above. - ginfo.add_key(k, false); + ginfo.add_key(std::span{k}.first<32>(), false); REQUIRE_THROWS_WITH( ginfo.set_name("Super Group!"), "Unable to make changes to a read-only config object"); @@ -177,10 +177,10 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { CHECK(!ginfo.is_dirty()); // This one is good and has the right signature: - groups::Info ginfo_rw{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Info ginfo_rw{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys1) // Just for testing, as above. - ginfo_rw.add_key(k, false); + ginfo_rw.add_key(std::span{k}.first<32>(), false); ginfo_rw.set_name("Super Group!!"); CHECK(ginfo_rw.is_dirty()); @@ -195,15 +195,15 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { CHECK(ginfo_rw.needs_dump()); CHECK_FALSE(ginfo_rw.needs_push()); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash1", to_push.at(0)); CHECK(ginfo.merge(merge_configs) == std::unordered_set{{"fakehash1"s}}); CHECK_FALSE(ginfo.needs_push()); - groups::Info ginfo_rw2{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Info ginfo_rw2{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys1) // Just for testing, as above. - ginfo_rw2.add_key(k, false); + ginfo_rw2.add_key(std::span{k}.first<32>(), false); CHECK(ginfo_rw2.merge(merge_configs) == std::unordered_set{{"fakehash1"s}}); CHECK_FALSE(ginfo.needs_push()); @@ -218,22 +218,16 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { // Deliberately use the wrong signing key so that what we produce encrypts successfully but // doesn't verify - const auto seed_bad1 = - "0023456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - std::array ed_pk_bad1; - std::array ed_sk_bad1; - crypto_sign_ed25519_seed_keypair( - ed_pk_bad1.data(), - ed_sk_bad1.data(), - reinterpret_cast(seed_bad1.data())); + const auto seed_bad1 = "0023456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + auto [ed_pk_bad1, ed_sk_bad1] = ed25519::keypair(seed_bad1); - groups::Info ginfo_bad1{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Info ginfo_bad1{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys1) // Just for testing, as above. - ginfo_bad1.add_key(k, false); + ginfo_bad1.add_key(std::span{k}.first<32>(), false); ginfo_bad1.merge(merge_configs); - ginfo_bad1.set_sig_keys(session::to_span(ed_sk_bad1)); + ginfo_bad1.set_sig_keys(ed_sk_bad1); ginfo_bad1.set_name("Bad name, BAD!"); auto [s_bad, p_bad, o_bad] = ginfo_bad1.push(); @@ -310,10 +304,10 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { CHECK(ginfo.needs_dump()); auto dump = ginfo.dump(); - groups::Info ginfo2{session::to_span(ed_pk), std::nullopt, dump}; + groups::Info ginfo2{ed_pk, std::nullopt, dump}; for (const auto& k : enc_keys1) // Just for testing, as above. - ginfo2.add_key(k, false); + ginfo2.add_key(std::span{k}.first<32>(), false); CHECK(!ginfo.needs_dump()); CHECK(!ginfo2.needs_dump()); @@ -328,10 +322,10 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { CHECK(o5.empty()); // This account has a different primary decryption key - groups::Info ginfo_rw3{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Info ginfo_rw3{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys2) // Just for testing, as above. - ginfo_rw3.add_key(k, false); + ginfo_rw3.add_key(std::span{k}.first<32>(), false); CHECK(ginfo_rw3.merge(merge_configs) == std::unordered_set{{"fakehash23"s}}); CHECK(ginfo_rw3.get_name() == "Super Group 2"); @@ -348,7 +342,7 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { ginfo_rw3.set_profile_pic( "http://example.com/12345", - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes); + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b); CHECK(ginfo_rw3.needs_push()); auto [s7, t7, o7] = ginfo_rw3.push(); CHECK(s7 == s6 + 1); @@ -360,11 +354,12 @@ TEST_CASE("Verify-only Group Info", "[config][groups][verify-only]") { // If we don't have the new "bbb" key loaded yet, this will fail: CHECK(ginfo.merge(merge_configs) == std::unordered_set{}); - ginfo.add_key(enc_keys2.front()); + ginfo.add_key(std::span{enc_keys2.front()}.first<32>()); CHECK(ginfo.merge(merge_configs) == std::unordered_set{{"fakehash7"s}}); auto pic = ginfo.get_profile_pic(); CHECK_FALSE(pic.empty()); CHECK(pic.url == "http://example.com/12345"); - CHECK(pic.key == "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes); + CHECK(std::ranges::equal( + pic.key, "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b)); } diff --git a/tests/test_group_keys.cpp b/tests/test_group_keys.cpp index 0af36f311..6981905f1 100644 --- a/tests/test_group_keys.cpp +++ b/tests/test_group_keys.cpp @@ -19,29 +19,12 @@ #include #include +#include "session/crypto/ed25519.hpp" #include "utils.hpp" using namespace std::literals; using namespace session::config; -static std::array sk_from_seed(std::span seed) { - std::array ignore; - std::array sk; - crypto_sign_ed25519_seed_keypair(ignore.data(), sk.data(), seed.data()); - return sk; -} - -static std::string session_id_from_ed(std::span ed_pk) { - std::string sid; - std::array xpk; - int rc = crypto_sign_ed25519_pk_to_curve25519(xpk.data(), ed_pk.data()); - REQUIRE(rc == 0); - sid.reserve(66); - sid += "05"; - oxenc::to_hex(xpk.begin(), xpk.end(), std::back_inserter(sid)); - return sid; -} - // Hacky little class that implements `[n]` on a std::list. This is inefficient (since it access // has to iterate n times through the list) but we only use it on small lists in this test code so // convenience wins over efficiency. (Why not just use a vector? Because vectors requires `T` to @@ -52,38 +35,26 @@ struct hacky_list : std::list { }; struct pseudo_client { - std::array secret_key; - const std::span public_key{secret_key.data() + 32, 32}; - std::string session_id{session_id_from_ed(public_key)}; + cleared_b64 secret_key; + const std::span public_key{secret_key.data() + 32, 32}; + std::string session_id = oxenc::to_hex(ed25519::pk_to_session_id(public_key)); groups::Info info; groups::Members members; groups::Keys keys; pseudo_client( - std::span seed, - bool admin, - const unsigned char* gpk, - std::optional gsk, - std::optional> info_dump = std::nullopt, - std::optional> members_dump = std::nullopt, - std::optional> keys_dump = std::nullopt) : - secret_key{sk_from_seed(seed)}, - info{std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) - : std::nullopt, - info_dump}, - members{std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) - : std::nullopt, - members_dump}, - keys{session::to_span(secret_key), - std::span{gpk, 32}, - admin ? std::make_optional>({*gsk, 64}) - : std::nullopt, - keys_dump, - info, - members} { + std::span seed, + bool /*admin*/, + std::span gpk, + const ed25519::OptionalPrivKeySpan& gsk, + std::optional> info_dump = std::nullopt, + std::optional> members_dump = std::nullopt, + std::optional> keys_dump = std::nullopt) : + secret_key{ed25519::keypair(seed).second}, + info{gpk, gsk, info_dump}, + members{gpk, gsk, members_dump}, + keys{secret_key, gpk, gsk, keys_dump, info, members} { if (gsk) keys.rekey(info, members); } @@ -91,25 +62,22 @@ struct pseudo_client { TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { - const std::vector group_seed = - "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hexbytes; - const std::vector admin1_seed = - "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - const std::vector admin2_seed = - "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hexbytes; - const std::array member_seeds = { - "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hexbytes, // member1 - "00011122435111155566677788811263446552465222efff0123456789abcdef"_hexbytes, // member2 - "00011129824754185548239498168169316979583253efff0123456789abcdef"_hexbytes, // member3 - "0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff"_hexbytes, // member4 - "3333333333333333333333333333333333333333333333333333333333333333"_hexbytes, // member3b - "4444444444444444444444444444444444444444444444444444444444444444"_hexbytes, // member4b + constexpr auto group_seed = + "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hex_b; + constexpr auto admin1_seed = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + constexpr auto admin2_seed = + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hex_b; + constexpr std::array member_seeds = { + "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hex_b, // member1 + "00011122435111155566677788811263446552465222efff0123456789abcdef"_hex_b, // member2 + "00011129824754185548239498168169316979583253efff0123456789abcdef"_hex_b, // member3 + "0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff"_hex_b, // member4 + "3333333333333333333333333333333333333333333333333333333333333333"_hex_b, // member3b + "4444444444444444444444444444444444444444444444444444444444444444"_hex_b, // member4b }; - std::array group_pk; - std::array group_sk; - - crypto_sign_ed25519_seed_keypair(group_pk.data(), group_sk.data(), group_seed.data()); + auto [group_pk, group_sk] = ed25519::keypair(group_seed); REQUIRE(oxenc::to_hex(group_seed.begin(), group_seed.end()) == oxenc::to_hex(group_sk.begin(), group_sk.begin() + 32)); @@ -120,11 +88,11 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { hacky_list members; // Initialize admin and member objects - admins.emplace_back(admin1_seed, true, group_pk.data(), group_sk.data()); - admins.emplace_back(admin2_seed, true, group_pk.data(), group_sk.data()); + admins.emplace_back(admin1_seed, true, group_pk, group_sk); + admins.emplace_back(admin2_seed, true, group_pk, group_sk); for (int i = 0; i < 4; ++i) - members.emplace_back(member_seeds[i], false, group_pk.data(), std::nullopt); + members.emplace_back(member_seeds[i], false, group_pk, std::nullopt); REQUIRE(admins[0].session_id == "05f1e8b64bbf761edf8f7b47e3a1f369985644cce0a62adb8e21604474bdd49627"); @@ -144,8 +112,8 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { for (const auto& m : members) REQUIRE(m.members.size() == 0); - std::vector>> info_configs; - std::vector>> mem_configs; + std::vector>> info_configs; + std::vector>> mem_configs; // add admin account, re-key, distribute auto& admin1 = admins[0]; @@ -296,7 +264,7 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { CHECK(admin1.members.needs_push()); - std::vector old_key = session::to_vector(admin1.keys.group_enc_key()); + std::vector old_key = session::to_vector(admin1.keys.group_enc_key()); auto new_keys_config4 = admin1.keys.rekey(admin1.info, admin1.members); CHECK(not new_keys_config4.empty()); @@ -363,7 +331,7 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { // Add two new members and send them supplemental keys for (int i = 0; i < 2; ++i) { - auto& m = members.emplace_back(member_seeds[4 + i], false, group_pk.data(), std::nullopt); + auto& m = members.emplace_back(member_seeds[4 + i], false, group_pk, std::nullopt); auto memb = admin1.members.get_or_construct(m.session_id); memb.set_invite_sent(); @@ -431,7 +399,7 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { CHECK(m.keys.active_hashes() == std::unordered_set{{"keyhash5"s}}); } - std::pair> decrypted1, decrypted2; + std::pair> decrypted1, decrypted2; REQUIRE_NOTHROW(decrypted1 = members.back().keys.decrypt_message(compressed)); CHECK(decrypted1.first == admin1.session_id); CHECK(session::to_string(decrypted1.second) == msg); @@ -441,16 +409,16 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { CHECK(session::to_string(decrypted2.second) == msg); auto bad_compressed = compressed; - bad_compressed.back() ^= 0b100; + bad_compressed.back() ^= std::byte{0b100}; CHECK_THROWS_WITH( members.back().keys.decrypt_message(bad_compressed), - "unable to decrypt ciphertext with any current group keys; tried 4"); + "unable to decrypt ciphertext with any current group keys"); // Duplicate members[1] from dumps auto& m1b = members.emplace_back( member_seeds[1], false, - group_pk.data(), + group_pk, std::nullopt, members[1].info.dump(), members[1].members.dump(), @@ -469,7 +437,7 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { // get dropped as stale. info_configs.clear(); mem_configs.clear(); - std::vector new_keys_config6 = + std::vector new_keys_config6 = session::to_vector(admin1.keys.rekey(admin1.info, admin1.members)); auto [iseq6, ipush6, iobs6] = admin1.info.push(); info_configs.emplace_back("ifakehash6", ipush6[0]); @@ -499,7 +467,7 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { "keyhash6"s}}); } - std::vector new_keys_config7 = + std::vector new_keys_config7 = session::to_vector(admin1.keys.rekey(admin1.info, admin1.members)); // Make sure we can encrypt & decrypt even if the rekey is still pending: @@ -553,8 +521,8 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { pseudo_client admin1b{ admin1_seed, true, - group_pk.data(), - group_sk.data(), + group_pk, + group_sk, admin1.info.dump(), admin1.members.dump(), admin1.keys.dump()}; @@ -577,21 +545,21 @@ TEST_CASE("Group Keys - C++ API", "[config][groups][keys][cpp]") { } TEST_CASE("Group Keys - C API", "[config][groups][keys][c]") { - struct pseudo_client { - std::array secret_key; - const std::span public_key{secret_key.data() + 32, 32}; - std::string session_id{session_id_from_ed(public_key)}; + struct pseudo_client_c { + b64 secret_key; + std::span public_key{secret_key.data() + 32, 32}; + std::string session_id = oxenc::to_hex(ed25519::pk_to_session_id(public_key)); config_group_keys* keys; config_object* info; config_object* members; - pseudo_client( - std::vector seed, + pseudo_client_c( + std::span seed, bool is_admin, unsigned char* gpk, std::optional gsk) : - secret_key{sk_from_seed(seed)} { + secret_key{ed25519::keypair(seed).second} { int rv = groups_members_init(&members, gpk, is_admin ? *gsk : NULL, NULL, 0, NULL); REQUIRE(rv == 0); @@ -600,7 +568,7 @@ TEST_CASE("Group Keys - C API", "[config][groups][keys][c]") { rv = groups_keys_init( &keys, - secret_key.data(), + to_unsigned(secret_key.data()), gpk, is_admin ? *gsk : NULL, info, @@ -614,44 +582,41 @@ TEST_CASE("Group Keys - C API", "[config][groups][keys][c]") { REQUIRE(groups_keys_rekey(keys, info, members, nullptr, nullptr)); } - ~pseudo_client() { + ~pseudo_client_c() { config_free(info); config_free(members); } }; - const std::vector group_seed = - "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hexbytes; - const std::vector admin1_seed = - "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - const std::vector admin2_seed = - "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hexbytes; - const std::array member_seeds = { - "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hexbytes, // member1 - "00011122435111155566677788811263446552465222efff0123456789abcdef"_hexbytes, // member2 - "00011129824754185548239498168169316979583253efff0123456789abcdef"_hexbytes, // member3 - "0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff"_hexbytes // member4 + constexpr auto group_seed = + "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hex_b; + constexpr auto admin1_seed = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + constexpr auto admin2_seed = + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hex_b; + constexpr std::array member_seeds = { + "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hex_b, // member1 + "00011122435111155566677788811263446552465222efff0123456789abcdef"_hex_b, // member2 + "00011129824754185548239498168169316979583253efff0123456789abcdef"_hex_b, // member3 + "0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff"_hex_b // member4 }; - std::array group_pk; - std::array group_sk; + auto [group_pk, group_sk] = ed25519::keypair(group_seed); - crypto_sign_ed25519_seed_keypair( - group_pk.data(), - group_sk.data(), - reinterpret_cast(group_seed.data())); REQUIRE(oxenc::to_hex(group_seed.begin(), group_seed.end()) == oxenc::to_hex(group_sk.begin(), group_sk.begin() + 32)); - hacky_list admins; - hacky_list members; + hacky_list admins; + hacky_list members; // Initialize admin and member objects - admins.emplace_back(admin1_seed, true, group_pk.data(), group_sk.data()); - admins.emplace_back(admin2_seed, true, group_pk.data(), group_sk.data()); + auto* gpk = to_unsigned(group_pk.data()); + auto* gsk = to_unsigned(group_sk.data()); + admins.emplace_back(admin1_seed, true, gpk, gsk); + admins.emplace_back(admin2_seed, true, gpk, gsk); for (int i = 0; i < 4; ++i) - members.emplace_back(member_seeds[i], false, group_pk.data(), std::nullopt); + members.emplace_back(member_seeds[i], false, gpk, std::nullopt); REQUIRE(admins[0].session_id == "05f1e8b64bbf761edf8f7b47e3a1f369985644cce0a62adb8e21604474bdd49627"); @@ -851,25 +816,22 @@ TEST_CASE("Group Keys - C API", "[config][groups][keys][c]") { TEST_CASE("Group Keys - swarm authentication", "[config][groups][keys][swarm]") { - const std::vector group_seed = - "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hexbytes; - const std::vector admin_seed = - "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - const std::vector member_seed = - "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hexbytes; + constexpr auto group_seed = + "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hex_b; + constexpr auto admin_seed = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + constexpr auto member_seed = + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"_hex_b; - std::array group_pk; - std::array group_sk; - - crypto_sign_ed25519_seed_keypair(group_pk.data(), group_sk.data(), group_seed.data()); + auto [group_pk, group_sk] = ed25519::keypair(group_seed); REQUIRE(oxenc::to_hex(group_seed.begin(), group_seed.end()) == oxenc::to_hex(group_sk.begin(), group_sk.begin() + 32)); CHECK(oxenc::to_hex(group_pk.begin(), group_pk.end()) == "c50cb3ae956947a8de19135b5be2685ff348afc63fc34a837aca12bc5c1f5625"); - pseudo_client admin{admin_seed, true, group_pk.data(), group_sk.data()}; - pseudo_client member{member_seed, false, group_pk.data(), std::nullopt}; + pseudo_client admin{admin_seed, true, group_pk, group_sk}; + pseudo_client member{member_seed, false, group_pk, std::nullopt}; session::config::UserGroups member_groups{member_seed, std::nullopt}; CHECK(admin.session_id == "05f1e8b64bbf761edf8f7b47e3a1f369985644cce0a62adb8e21604474bdd49627"); @@ -893,7 +855,7 @@ TEST_CASE("Group Keys - swarm authentication", "[config][groups][keys][swarm]") REQUIRE(push.size() == 1); - std::vector>> gr_conf; + std::vector>> gr_conf; gr_conf.emplace_back("fakehash1", push[0]); member_gr2.merge(gr_conf); @@ -919,16 +881,14 @@ TEST_CASE("Group Keys - swarm authentication", "[config][groups][keys][swarm]") CHECK(oxenc::to_base64(subauth.subaccount_sig) == subauth_b64.subaccount_sig); CHECK(oxenc::to_base64(subauth.signature) == subauth_b64.signature); - CHECK(0 == - crypto_sign_ed25519_verify_detached( - reinterpret_cast(subauth.signature.data()), - to_sign.data(), - to_sign.size(), - reinterpret_cast(subauth.subaccount.substr(4).data()))); + CHECK(ed25519::verify( + to_span(subauth.signature).first<64>(), + to_span(subauth.subaccount.substr(4)).first<32>(), + to_sign)); CHECK(member.keys.swarm_verify_subaccount(auth_data)); CHECK(session::config::groups::Keys::swarm_verify_subaccount( - member.info.id, session::to_span(member.secret_key), auth_data)); + member.info.id, member.secret_key, auth_data)); // Try flipping a bit in each position of the auth data and make sure it fails to validate: for (size_t i = 0; i < auth_data.size(); i++) { @@ -938,33 +898,30 @@ TEST_CASE("Group Keys - swarm authentication", "[config][groups][keys][swarm]") // sign bit, so won't actually change anything if it flips. continue; auto auth_data2 = auth_data; - auth_data2[i] ^= 1 << b; + auth_data2[i] ^= static_cast(1 << b); CHECK_FALSE(session::config::groups::Keys::swarm_verify_subaccount( - member.info.id, session::to_span(member.secret_key), auth_data2)); + member.info.id, member.secret_key, auth_data2)); } } } TEST_CASE("Group Keys promotion", "[config][groups][keys][promotion]") { - const std::vector group_seed = - "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hexbytes; - const std::vector admin1_seed = - "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - const std::vector member1_seed = - "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hexbytes; - - std::array group_pk; - std::array group_sk; + constexpr auto group_seed = + "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"_hex_b; + constexpr auto admin1_seed = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + constexpr auto member1_seed = + "000111222333444555666777888999aaabbbcccdddeeefff0123456789abcdef"_hex_b; - crypto_sign_ed25519_seed_keypair(group_pk.data(), group_sk.data(), group_seed.data()); + auto [group_pk, group_sk] = ed25519::keypair(group_seed); REQUIRE(oxenc::to_hex(group_seed.begin(), group_seed.end()) == oxenc::to_hex(group_sk.begin(), group_sk.begin() + 32)); - pseudo_client admin{admin1_seed, true, group_pk.data(), group_sk.data()}; - pseudo_client member{member1_seed, false, group_pk.data(), std::nullopt}; + pseudo_client admin{admin1_seed, true, group_pk, group_sk}; + pseudo_client member{member1_seed, false, group_pk, std::nullopt}; - std::vector>> configs; + std::vector>> configs; { auto m = admin.members.get_or_construct(admin.session_id); m.admin = true; @@ -1018,7 +975,7 @@ TEST_CASE("Group Keys promotion", "[config][groups][keys][promotion]") { REQUIRE(member.info.is_readonly()); REQUIRE(member.members.is_readonly()); - member.keys.load_admin_key(session::to_span(group_sk), member.info, member.members); + member.keys.load_admin_key(group_sk, member.info, member.members); CHECK(member.keys.admin()); CHECK_FALSE(member.members.is_readonly()); diff --git a/tests/test_group_members.cpp b/tests/test_group_members.cpp index 017abe75a..1ed2bcb6e 100644 --- a/tests/test_group_members.cpp +++ b/tests/test_group_members.cpp @@ -1,11 +1,11 @@ #include #include -#include #include #include #include #include +#include #include #include "utils.hpp" @@ -24,43 +24,42 @@ constexpr bool is_prime100(int i) { TEST_CASE("Group Members", "[config][groups][members]") { - const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); + const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "cbd569f56fb13ea95a3f0c05c331cc24139c0090feb412069dc49fab34406ece"); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - std::vector> enc_keys{ - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hexbytes}; + std::vector> enc_keys{ + to_vector("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"_hex_b)}; - groups::Members gmem1{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Members gmem1{ed_pk, ed_sk, std::nullopt}; // This is just for testing: normally you don't load keys manually but just make a groups::Keys // object that loads the keys into the Members object for you. for (const auto& k : enc_keys) - gmem1.add_key(k, false); + gmem1.add_key(std::span{k}.first<32>(), false); enc_keys.insert( enc_keys.begin(), - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hexbytes); - enc_keys.push_back("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hexbytes); - enc_keys.push_back("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"_hexbytes); - groups::Members gmem2{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + to_vector("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hex_b)); + enc_keys.push_back( + to_vector("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"_hex_b)); + enc_keys.push_back( + to_vector("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"_hex_b)); + groups::Members gmem2{ed_pk, ed_sk, std::nullopt}; for (const auto& k : enc_keys) // Just for testing, as above. - gmem2.add_key(k, false); + gmem2.add_key(std::span{k}.first<32>(), false); std::vector sids; while (sids.size() < 256) { - std::array sid; + b33 sid; for (auto& s : sid) - s = sids.size(); - sid[0] = 0x05; + s = static_cast(sids.size()); + sid[0] = std::byte{0x05}; sids.push_back(oxenc::to_hex(sid.begin(), sid.end())); } @@ -71,7 +70,7 @@ TEST_CASE("Group Members", "[config][groups][members]") { m.name = "Admin {}"_format(i); m.profile_picture.url = "http://example.com/{}"_format(i); m.profile_picture.key = - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes; + to_vector("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b); m.profile_updated = std::chrono::sys_seconds{1s}; gmem1.set(m); } @@ -81,7 +80,7 @@ TEST_CASE("Group Members", "[config][groups][members]") { m.set_name("Member {}"_format(i)); m.profile_picture.url = "http://example.com/{}"_format(i); m.profile_picture.key = - "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes; + to_vector("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b); m.profile_updated = session::to_sys_seconds(2); gmem1.set(m); } @@ -102,7 +101,7 @@ TEST_CASE("Group Members", "[config][groups][members]") { CHECK(gmem1.needs_dump()); CHECK_FALSE(gmem1.needs_push()); - std::vector>> merge_configs; + std::vector>> merge_configs; merge_configs.emplace_back("fakehash1", p1.at(0)); CHECK(gmem2.merge(merge_configs) == std::unordered_set{{"fakehash1"s}}); CHECK_FALSE(gmem2.needs_push()); @@ -205,7 +204,7 @@ TEST_CASE("Group Members", "[config][groups][members]") { gmem2.confirm_pushed(s2, {"fakehash2"}); merge_configs.emplace_back("fakehash2", p2.at(0)); // not clearing it first! CHECK(gmem1.merge(merge_configs) == std::unordered_set{{"fakehash1"s}}); - gmem1.add_key("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hexbytes); + gmem1.add_key("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"_hex_b); CHECK(gmem1.merge(merge_configs) == std::unordered_set{{"fakehash1"s, "fakehash2"s}}); CHECK(gmem1.get(sids[23]).value().name == "Member 23"); @@ -218,9 +217,12 @@ TEST_CASE("Group Members", "[config][groups][members]") { CHECK(m.name == ((i == 20 || i == 21 || i >= 50) ? "" : "{} {}"_format(i < 10 ? "Admin" : "Member", i))); - CHECK(m.profile_picture.key == - (i < 20 ? "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes - : ""_hexbytes)); + if (i < 20) + CHECK(std::ranges::equal( + m.profile_picture.key, + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b)); + else + CHECK(m.profile_picture.key.empty()); CHECK(m.profile_picture.url == (i < 20 ? "http://example.com/{}"_format(i) : "")); if (i < 5) CHECK(m.profile_updated.time_since_epoch() == 1s); @@ -302,9 +304,12 @@ TEST_CASE("Group Members", "[config][groups][members]") { CHECK(m.name == ((i == 20 || i == 21 || i >= 50) ? "" : "{} {}"_format(i < 10 ? "Admin" : "Member", i))); - CHECK(m.profile_picture.key == - (i < 20 ? "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hexbytes - : ""_hexbytes)); + if (i < 20) + CHECK(std::ranges::equal( + m.profile_picture.key, + "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"_hex_b)); + else + CHECK(m.profile_picture.key.empty()); CHECK(m.profile_picture.url == (i < 20 ? "http://example.com/{}"_format(i) : "")); if (i < 5) CHECK(m.profile_updated.time_since_epoch() == 1s); @@ -371,18 +376,15 @@ TEST_CASE("Group Members", "[config][groups][members]") { TEST_CASE("Group Members restores extra data", "[config][groups][members]") { - const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair( - ed_pk.data(), ed_sk.data(), reinterpret_cast(seed.data())); + const auto seed = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == "cbd569f56fb13ea95a3f0c05c331cc24139c0090feb412069dc49fab34406ece"); CHECK(oxenc::to_hex(seed.begin(), seed.end()) == oxenc::to_hex(ed_sk.begin(), ed_sk.begin() + 32)); - groups::Members gmem1{session::to_span(ed_pk), session::to_span(ed_sk), std::nullopt}; + groups::Members gmem1{ed_pk, ed_sk, std::nullopt}; auto memberId1 = "050000000000000000000000000000000000000000000000000000000000000000"; auto memberId2 = "051111111111111111111111111111111111111111111111111111111111111111"; @@ -401,7 +403,7 @@ TEST_CASE("Group Members restores extra data", "[config][groups][members]") { auto dumped = gmem1.dump(); - groups::Members gmem2{session::to_span(ed_pk), session::to_span(ed_sk), dumped}; + groups::Members gmem2{ed_pk, ed_sk, dumped}; CHECK(gmem2.get_status(gmem1.get_or_construct(memberId1)) == groups::member::Status::invite_sending); diff --git a/tests/test_hash.cpp b/tests/test_hash.cpp index ab1e2bc15..9e02b45a3 100644 --- a/tests/test_hash.cpp +++ b/tests/test_hash.cpp @@ -1,3 +1,5 @@ +#include + #include #include "session/hash.h" @@ -5,7 +7,12 @@ #include "session/util.hpp" #include "utils.hpp" +using namespace session::literals; + TEST_CASE("Hash generation", "[hash][hash]") { + // Intentionally exercising the deprecated hash::hash() to verify it still works. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" auto hash1 = session::hash::hash(32, session::to_span("TestMessage"), std::nullopt); auto hash2 = session::hash::hash(32, session::to_span("TestMessage"), std::nullopt); auto hash3 = @@ -23,6 +30,7 @@ TEST_CASE("Hash generation", "[hash][hash]") { session::to_span("KeyThatIsTooLongKeyThatIsTooLongKeyThatIsTooLongKeyThatIsTooLongKeyTh" "atIsTooLon" "g"))); +#pragma GCC diagnostic pop CHECK(hash1.size() == 32); CHECK(hash2.size() == 32); @@ -49,3 +57,220 @@ TEST_CASE("Hash generation", "[hash][hash]") { CHECK(to_hex(hash5) == expected_hash5); CHECK(to_hex(hash6) == expected_hash6); } + +TEST_CASE("blake2b_hasher", "[hash][blake2b]") { + using session::b32; + using session::hash::blake2b_hasher; + using session::hash::nullkey; + + // The deprecated hash::hash calls libsodium directly (no blake2b_hasher involvement) and serves + // as the independent reference for the no-pers cases below. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + + // ── No-key, no-pers ────────────────────────────────────────────────────────────────────── + // KAT value cross-checks against deprecated hash::hash (independent libsodium path). + + { + auto out = blake2b_hasher<32>{}.update("TestMessage"_bytes).finalize(); + auto ref = session::hash::hash(32, session::to_span("TestMessage"), std::nullopt); + CHECK(std::ranges::equal(out, ref)); + CHECK(to_hex(out) == "2a48a12262e4548afb97fe2b04a912a02297d451169ee7ef2d01a28ea20286ab"); + } + + { + auto out = blake2b_hasher<64>{}.update("TestMessage"_bytes).finalize(); + auto ref = session::hash::hash(64, session::to_span("TestMessage"), std::nullopt); + CHECK(std::ranges::equal(out, ref)); + CHECK(to_hex(out) == + "9d9085ac026fe3542abbeb2ea2ec05f5c37aecd7695f6cc41e9ccf39014196a3" + "9c02db69c4416d5c45acc2e9469b7f274992b2858f3bb2746becb48c8b56ce4b"); + } + + // ── Keyed, no-pers ─────────────────────────────────────────────────────────────────────── + + { + auto out = blake2b_hasher<32>{"TestKey"_bytes, std::nullopt} + .update("TestMessage"_bytes) + .finalize(); + auto ref = session::hash::hash( + 32, session::to_span("TestMessage"), session::to_span("TestKey")); + CHECK(std::ranges::equal(out, ref)); + CHECK(to_hex(out) == "3d643e479b626bb2907476e32ccf7bdbd1ac3efa0da6e2c335255c48dcc216b6"); + } + + { + auto out = blake2b_hasher<64>{"TestKey"_bytes, std::nullopt} + .update("TestMessage"_bytes) + .finalize(); + auto ref = session::hash::hash( + 64, session::to_span("TestMessage"), session::to_span("TestKey")); + CHECK(std::ranges::equal(out, ref)); + CHECK(to_hex(out) == + "6a2faad89cf9010a4270cba07cc96cfb36688106e080b15fef66bb03c68e8778" + "74c9059edf53d03c1330b2655efdad6e4aa259118b6ea88698ea038efb9d52ce"); + } + +#pragma GCC diagnostic pop + + // ── Multi-update consistency ────────────────────────────────────────────────────────────── + // Splitting the input across calls must yield the same hash. + + { + auto single = blake2b_hasher<32>{}.update("TestMessage"_bytes).finalize(); + auto multi = blake2b_hasher<32>{} + .update("Test"_bytes) // split across two calls + .update("Message"_bytes) + .finalize(); + CHECK(single == multi); + + b32 out_write; + blake2b_hasher<32>{}.update("TestMes"_bytes, "sage"_bytes).finalize(out_write); + CHECK(single == out_write); + } + + // ── Return-value vs write-to-output finalize ────────────────────────────────────────────── + + { + b32 out_write; + blake2b_hasher<32>{}.update("TestMessage"_bytes).finalize(out_write); + auto out_rv = blake2b_hasher<32>{}.update("TestMessage"_bytes).finalize(); + CHECK(out_write == out_rv); + } + + // ── Personalisation string changes output ───────────────────────────────────────────────── + + constexpr auto pers = "TestPers1234567!"_b2b_pers; + + b32 no_pers_out, pers_out; + blake2b_hasher<32>{}.update("TestMessage"_bytes).finalize(no_pers_out); + blake2b_hasher<32>{nullkey, pers}.update("TestMessage"_bytes).finalize(pers_out); + CHECK(no_pers_out != pers_out); + + // Pers is deterministic: same config and input → same output. + b32 pers_out2; + blake2b_hasher<32>{nullkey, pers}.update("TestMessage"_bytes).finalize(pers_out2); + CHECK(pers_out == pers_out2); + + // Pers + multi-update consistency. + b32 pers_multi; + blake2b_hasher<32>{nullkey, pers} + .update("Test"_bytes) + .update("Message"_bytes) + .finalize(pers_multi); + CHECK(pers_out == pers_multi); + + // Different pers → different output. + constexpr auto pers2 = "OtherPers123456!"_b2b_pers; + b32 pers2_out; + blake2b_hasher<32>{nullkey, pers2}.update("TestMessage"_bytes).finalize(pers2_out); + CHECK(pers_out != pers2_out); + + // ── Key + pers ──────────────────────────────────────────────────────────────────────────── + + b32 key_pers_out; + blake2b_hasher<32>{"TestKey"_bytes, pers}.update("TestMessage"_bytes).finalize(key_pers_out); + // Distinct from keyed-only, pers-only, and no-key/no-pers outputs. + CHECK(key_pers_out != pers_out); + CHECK(key_pers_out != no_pers_out); + // Consistent across repeated construction. + b32 key_pers_out2; + blake2b_hasher<32>{"TestKey"_bytes, pers}.update("TestMessage"_bytes).finalize(key_pers_out2); + CHECK(key_pers_out == key_pers_out2); +} + +TEST_CASE("SHA3-256 and SHAKE-256 known-answer tests", "[hash][sha3_256][shake256]") { + // This test case serves two purposes: + // 1. Verify SHA3-256 against NIST FIPS 202 known-answer test vectors. + // 2. Verify SHAKE-256 against NIST FIPS 202 known-answer test vectors. + // 3. Confirm that SHA3-256 and SHAKE-256 produce different output on identical input, + // verifying that the domain suffix byte (0x06 vs 0x1F) is actually applied. + // + // SHA3-256 KATs: + // https://csrc.nist.gov/csrc/media/projects/cryptographic-algorithm-validation-program/documents/sha3/sha-3bittestvectors.zip + // SHAKE-256 KATs: NIST FIPS 202, Appendix A / CAVS test data + + using session::b32; + using session::hash::sha3_256; + using session::hash::shake256; + + b32 sha3_out, shake_out; + + // --- SHA3-256 NIST vectors --- + + // Empty input + sha3_256(sha3_out, ""_bytes); + CHECK(oxenc::to_hex(sha3_out) == + "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"); + + // "abc" (24 bits) + sha3_256(sha3_out, "abc"_bytes); + CHECK(oxenc::to_hex(sha3_out) == + "3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532"); + + // 448-bit message + sha3_256(sha3_out, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"_bytes); + CHECK(oxenc::to_hex(sha3_out) == + "41c0dba2a9d6240849100376a8235e2c82e1b9998a999e21db32dd97496d3376"); + + // 896-bit message + sha3_256( + sha3_out, + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklm" + "nopqklmnopqrlmnopqrsmnopqrstnopqrstu"_bytes); + CHECK(oxenc::to_hex(sha3_out) == + "916f6061fe879741ca6469b43971dfdb28b1a32dc36cb3254e812be27aad1d18"); + + // --- SHAKE-256 NIST vectors (32-byte output) --- + + // Empty input; first 32 bytes from FIPS 202 Appendix B.2 sample output + shake256(""_bytes)(shake_out); + CHECK(oxenc::to_hex(shake_out) == + "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f"); + + // "abc" (24 bits) + shake256("abc"_bytes)(shake_out); + CHECK(oxenc::to_hex(shake_out) == + "483366601360a8771c6863080cc4114d8db44530f8f1e1ee4f94ea37e78b5739"); + + // --- Cross-check: same input must produce different output --- + sha3_256(sha3_out, "abc"_bytes); + shake256("abc"_bytes)(shake_out); + CHECK(sha3_out != shake_out); +} + +TEST_CASE("blake2b_pers integer args are little-endian", "[hash][blake2b][endian]") { + using session::hash::blake2b_pers; + + // make_hashable() must serialize integer arguments as fixed-width little-endian, independent of + // host endianness — the byte encoding underpinning every Session Pro signed digest. Pin the + // contract by requiring an integer to hash identically to its explicit little-endian bytes. On + // a little-endian host this guards the direct-reinterpret path; on a big-endian host (run via + // utils/test-bigendian.sh) it is the only thing exercising make_hashable's byte-swap branch. + constexpr auto pers = "EndianTestPers!!"_b2b_pers; + + { + uint16_t v = 0x0102; + std::array le{std::byte{0x02}, std::byte{0x01}}; + CHECK(blake2b_pers<32>(pers, v) == blake2b_pers<32>(pers, le)); + } + { + uint32_t v = 0x01020304; + std::array le{ + std::byte{0x04}, std::byte{0x03}, std::byte{0x02}, std::byte{0x01}}; + CHECK(blake2b_pers<32>(pers, v) == blake2b_pers<32>(pers, le)); + } + { + uint64_t v = 0x0102030405060708ULL; + std::array le{ + std::byte{0x08}, + std::byte{0x07}, + std::byte{0x06}, + std::byte{0x05}, + std::byte{0x04}, + std::byte{0x03}, + std::byte{0x02}, + std::byte{0x01}}; + CHECK(blake2b_pers<32>(pers, v) == blake2b_pers<32>(pers, le)); + } +} diff --git a/tests/test_helper.hpp b/tests/test_helper.hpp new file mode 100644 index 000000000..284095c33 --- /dev/null +++ b/tests/test_helper.hpp @@ -0,0 +1,497 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace session { + +// nlohmann can't parse a std::byte range directly: libc++'s std::char_traits has no std::byte +// specialization. Parse request/response bodies via a char view instead. +inline nlohmann::json parse_json(std::span body) { + return nlohmann::json::parse(to_string_view(body)); +} + +/// A minimal in-process mock of network::Network for unit tests. +/// Tests can set `current_node` to control which node get_swarm returns, and inspect +/// `sent_requests` to observe outgoing requests and fire their callbacks. +class MockNetwork : public network::Network { + public: + MockNetwork() : network::Network(network::config::Config{}) {} + + struct SentRequest { + network::Request request; + network::network_response_callback_t callback; + }; + std::vector sent_requests; + + // The node returned by get_swarm; tests can change this to simulate swarm-member switches. + network::service_node current_node; + + void send_request( + network::Request request, network::network_response_callback_t callback) override { + sent_requests.push_back({std::move(request), std::move(callback)}); + } + + void get_swarm( + network::x25519_pubkey /*swarm_pubkey*/, + bool /*ignore_strike_count*/, + std::function< + void(network::swarm_id_t swarm_id, std::vector swarm)> + callback) override { + callback(0, {current_node}); + } + + std::vector downloads; + + void download(network::DownloadRequest request) override { + downloads.push_back(std::move(request)); + } + + // A file server, in as much as anything needs one: uploads are encrypted and kept here, and a + // download of one serves it back. Enough to round-trip an attachment through the code that + // sends and saves it without a network, which is the only part of a file server that is + // interesting to test against. + // + // `served` is keyed by the id in the download url, so a url built by generate_download_url from + // what upload_file returned finds its way back to the right bytes. + std::map> served; + int next_file_id = 1000; + + void upload_file(network::FileUploadRequest request, std::span seed) override { + // Encrypted the same way the routers do it, so what a save reads back is what a real + // upload would have left on the server: same scheme, same padding, same key derivation. + std::vector plain; + { + std::ifstream in{request.file, std::ios::binary}; + std::string bytes{std::istreambuf_iterator{in}, {}}; + plain = to_vector(bytes); + } + auto [ciphertext, key] = + attachment::encrypt(seed, plain, request.domain, request.allow_large); + + auto id = std::to_string(next_file_id++); + served.emplace(id, std::move(ciphertext)); + + network::file_metadata meta{id, static_cast(served.at(id).size()), {}, {}}; + if (request.on_progress) + request.on_progress(meta.size, meta.size); + if (request.on_complete) + request.on_complete(std::pair{meta, key}, false); + } +}; + +/// Gives `core` a fresh MockNetwork and hands back a non-owning pointer to it. A Core owns its +/// Network outright -- nothing else may hold it alive -- so a test that goes on poking at the mock +/// keeps a raw pointer rather than a second reference. +inline MockNetwork* attach_mock_network(core::Core& core) { + return &core.make_network(); +} + +/// Answers every captured download with `data`, delivered in chunks as a transport would rather +/// than in one piece -- a decryptor that only works when handed the whole file at once is a bug +/// this is meant to catch. Returns how many there were. +inline size_t serve_downloads( + MockNetwork& net, std::span data, size_t chunk = 4096) { + auto pending = std::exchange(net.downloads, {}); + for (auto& r : pending) { + network::file_metadata meta{"served", static_cast(data.size()), {}, {}}; + for (size_t at = 0; at < data.size(); at += chunk) + r.on_data(meta, data.subspan(at, std::min(chunk, data.size() - at))); + r.on_complete(meta, false); + } + return pending.size(); +} + +/// Answers every captured download from what was uploaded, looked up by the file id in its url -- +/// so a message whose attachment this Client uploaded can be saved back through the same object. +/// A url naming something never uploaded is answered as the file server answers a missing file. +inline size_t serve_downloads(MockNetwork& net, size_t chunk = 4096) { + auto pending = std::exchange(net.downloads, {}); + for (auto& r : pending) { + auto info = network::file_server::parse_download_url(r.download_url); + auto found = info ? net.served.find(info->file_id) : net.served.end(); + if (found == net.served.end()) { + r.on_complete(static_cast(404), false); + continue; + } + + std::span data{found->second}; + network::file_metadata meta{info->file_id, static_cast(data.size()), {}, {}}; + for (size_t at = 0; at < data.size(); at += chunk) + r.on_data(meta, data.subspan(at, std::min(chunk, data.size() - at))); + r.on_complete(meta, false); + } + return pending.size(); +} + +/// Fails every captured download, as a file server that no longer holds the file would. +inline size_t fail_downloads(MockNetwork& net, int16_t status = 404) { + auto pending = std::exchange(net.downloads, {}); + for (auto& r : pending) + r.on_complete(status, false); + return pending.size(); +} + +/// The store requests a MockNetwork has captured, in the order they were sent. Filtered rather +/// than taken wholesale because a Core with a network attached also fetches PFS keys, so a test +/// that asked for a send finds retrieves in the list it never asked for. +inline std::vector stores(MockNetwork& net) { + std::vector found; + for (auto& r : net.sent_requests) + if (r.request.endpoint == "store") + found.push_back(&r); + return found; +} + +/// The JSON a store request carries, which is where the namespace and the payload are. +inline nlohmann::json store_body(const MockNetwork::SentRequest& r) { + if (!r.request.body) + throw std::logic_error{"store request has no body"}; + return parse_json(*r.request.body); +} + +/// The encrypted message a store request is depositing, decoded back out of its base64. +inline std::vector store_payload(const MockNetwork::SentRequest& r) { + return to_vector(oxenc::from_base64(store_body(r)["data"].get())); +} + +/// The hash answer_stores has the swarm assign a store. Distinct per destination swarm, so that a +/// test can tell the copy of an outgoing message left in our own swarm from the recipient's. +inline std::string store_hash_for(std::string_view pubkey_hex) { + return "hash-for-{}"_format(pubkey_hex); +} + +/// Hands the first `max` captured requests for `endpoint` to `respond`, leaving everything else +/// pending, and returns how many were answered. +/// +/// The pending list is taken away before any of it runs, and must be: a response can prompt Core to +/// send something new, and that push_back would otherwise reallocate `sent_requests` out from under +/// the very callback being invoked. +inline size_t answer_requests( + MockNetwork& net, + std::string_view endpoint, + const std::function& respond, + size_t max = std::numeric_limits::max()) { + auto pending = std::exchange(net.sent_requests, {}); + size_t answered = 0; + std::vector others; + + for (auto& r : pending) { + if (r.request.endpoint == endpoint && answered < max) { + respond(r); + answered++; + } else + others.push_back(std::move(r)); + } + + // Appended rather than assigned: whatever the responses prompted belongs in the list too. + for (auto& r : others) + net.sent_requests.push_back(std::move(r)); + + return answered; +} + +/// Answers every captured store, and returns how many there were -- which is itself worth asserting +/// on, since an outgoing message is two stores and a note to self is one. +inline size_t answer_stores(MockNetwork& net, bool accepted) { + return answer_requests(net, "store", [accepted](MockNetwork::SentRequest& r) { + if (accepted) { + nlohmann::json resp = { + {"hash", store_hash_for(store_body(r)["pubkey"].get())}}; + r.callback(true, false, 200, {}, resp.dump()); + } else + r.callback(false, false, 500, {}, "nope"); + }); +} + +inline size_t accept_stores(MockNetwork& net) { + return answer_stores(net, true); +} + +/// Times out captured key fetches, which is what releases a send queued behind one. +inline size_t fail_retrieves(MockNetwork& net, size_t max = std::numeric_limits::max()) { + return answer_requests( + net, + "retrieve", + [](MockNetwork::SentRequest& r) { r.callback(false, true, 0, {}, std::nullopt); }, + max); +} + +// Smart-pointer-like RAII wrapper around a Core backed by a unique temporary DB file. +// The DB file is removed on destruction. Default encryption uses a zeroed raw_key. +// If `extra_dir` is set, that directory tree is also removed recursively on destruction (used by +// make_live_core to clean up the network's cache directory). +struct TempCore { + std::filesystem::path path; + std::optional extra_dir; + std::unique_ptr core; + + template + explicit TempCore(Opts&&... opts) : + path{std::filesystem::temp_directory_path() / + fmt::format("{}.db", session::random::unique_id("test_core", 7))}, + core{std::make_unique(path, std::forward(opts)...)} {} + + TempCore(TempCore&&) = default; + TempCore& operator=(TempCore&&) = default; + + ~TempCore() { + core.reset(); // close DB before removing the file + std::error_code ec; + std::filesystem::remove(path, ec); + if (extra_dir) + std::filesystem::remove_all(*extra_dir, ec); + } + + core::Core* operator->() { return core.get(); } + core::Core& operator*() { return *core; } +}; + +/// Stands in for a real router so that Network's own logic -- which sits *above* routing, and which +/// MockNetwork's send_request override skips entirely -- can be driven without a network. +/// +/// Every request is recorded and answered from `replies`, keyed by the destination node's pubkey, +/// so a test says "this node is unreachable, that one answers" and then asserts on which were tried +/// and in what order. Everything else IRouter requires is a no-op: routing strategy is not what is +/// under test here. +class FakeRouter : public network::IRouter { + public: + struct Reply { + bool success = true; + bool timeout = false; + int16_t status = 200; + std::optional body = "{}"; + }; + + // Destination pubkey -> how that node answers. Anything not named here answers as unreachable, + // which makes "only this node works" the short thing to write. + std::map replies; + Reply default_reply{false, false, network::ERROR_INVALID_DESTINATION, "Node is not reachable"}; + + // Every destination tried, in order. The point of most assertions. + std::vector tried; + std::vector timeouts; + + void send_request( + network::Request request, network::network_response_callback_t callback) override { + auto* node = std::get_if(&request.destination); + if (!node) + return callback(false, false, network::ERROR_INVALID_DESTINATION, {}, "not a node"); + + tried.push_back(node->remote_pubkey); + timeouts.push_back(request.request_timeout); + + auto found = replies.find(node->remote_pubkey); + const auto& reply = found != replies.end() ? found->second : default_reply; + callback(reply.success, reply.timeout, reply.status, {}, reply.body); + } + + void suspend() override {} + void resume(bool) override {} + void close_connections() override {} + void clear_cache() override {} + network::ConnectionStatus get_status() const override { + return network::ConnectionStatus::connected; + } + void upload(network::UploadRequest) override {} + void upload_file(network::FileUploadRequest, std::span) override {} + void download(network::DownloadRequest) override {} +}; + +class TestHelper { + public: + static void poll(core::Core& core) { core._poll(); } + + /// Puts a swarm straight into the pool's cache. get_swarm consults it first and answers from + /// it without touching the network, which is what lets swarm-level behaviour be tested at all: + /// a test pool has no seed nodes, so nothing would ever resolve otherwise. + static void seed_swarm( + network::SnodePool& pool, + const network::x25519_pubkey& swarm_pubkey, + std::vector nodes) { + pool._swarm_cache[swarm_pubkey] = {0, std::move(nodes)}; + } + + /// Substitutes the router beneath a Network, so its own logic can be exercised against + /// scripted answers. Also hands back the pool, which a test has to seed for anything + /// swarm-addressed to resolve. + static void set_router(network::Network& net, std::shared_ptr router) { + net._router = std::move(router); + } + static network::SnodePool& snode_pool(network::Network& net) { return *net._snode_pool; } + + static sqlite::Connection db_conn(core::Core& core) { return core.db.conn(); } + + /// Drives the database-to-config direction directly, which is what makes the round-trip + /// assertable: applying a config and then deriving one back has to be the identity, and only a + /// test can ask for the second half in isolation. + /// + /// A template so that this header need not know the client types; it is only ever instantiated + /// where they are complete. + template + static void sync_contact(Client& c, const Id& id) { + c._sync_contact(id); + } + + template + static void sync_convo_volatile(Client& c, const Id& id) { + c._sync_convo_volatile(id); + } + + /// The push debounce, driven by hand. A test that waited out real intervals would be both slow + /// and racy -- the timer fires on the event loop while the test reads from its own thread -- so + /// what is exercised here is the decision the timer makes, with the clock supplied. + static bool push_scheduled(core::Configs& configs) { return configs._push_scheduled; } + static void push_if_due(core::Configs& configs) { configs._push_if_due(); } + static void backdate_push_state( + core::Configs& configs, + std::chrono::milliseconds since_last_change, + std::chrono::milliseconds since_first_change) { + auto now = std::chrono::steady_clock::now(); + configs._last_change = now - since_last_change; + configs._burst_started = now - since_first_change; + } + + // Returns whether the given migration name is recorded as applied. + static bool migration_applied(core::Core& core, std::string_view name) { + return core.db.conn() + .prepared_maybe_get( + "SELECT name FROM migrations_applied WHERE name = ?", name) + .has_value(); + } + + // The cursor a retrieve from this namespace+node would send: the newest hash that node handed + // us and still holds. Derived rather than stored, so this asks the same question the poll + // does. + static std::optional namespace_last_hash( + core::Core& core, int16_t ns, const network::ed25519_pubkey& sn_pubkey) { + return core.db.conn().prepared_maybe_get( + R"( +SELECT h.hash FROM swarm_hashes h JOIN swarm_nodes n ON n.id = h.node + WHERE h.namespace = ? AND n.pubkey = ? AND (h.expiry IS NULL OR h.expiry > ?) + ORDER BY h.id DESC LIMIT 1 +)", + ns, + sn_pubkey, + epoch_ms(clock_now_ms())); + } + + // Device group payload encryption/decryption. These are private to Devices and currently have + // no production caller (nothing yet builds or pushes a device group message), so tests are the + // only thing exercising them. + static std::vector encrypt_device_data( + core::Devices& d, const core::device::map& devices) { + return d.encrypt_device_data(devices); + } + static std::vector decrypt_device_data( + core::Devices& d, std::span data) { + return d.decrypt_device_data(data); + } + + // Feeds an encrypted device group message through the receive path, as a poll would. + static void receive_device_group_message(core::Devices& d, std::span data) { + d.receive_device_group_message(data); + } + + // Returns the raw 32-byte seed for the account key identified by the given x25519 public key. + static cleared_b32 account_key_seed( + core::Devices& d, std::span x25519_pub) { + cleared_b32 seed; + auto c = d.conn(); + auto blob = c.prepared_get>>( + "SELECT seed FROM device_account_keys WHERE pubkey_x25519 = ?", + std::as_bytes(x25519_pub)); + std::ranges::copy(blob, seed.begin()); + return seed; + } + + // Returns the {pubkey_x25519, pubkey_mlkem768} of the active (unrotated) account key. + static std::pair, std::array> active_account_pubkeys( + core::Core& core) { + auto [x25519, mlkem768] = + core.db.conn() + .prepared_get< + sqlite::blob_guts>, + sqlite::blob_guts>>( + "SELECT pubkey_x25519, pubkey_mlkem768" + " FROM device_account_keys WHERE rotated IS NULL"); + return {x25519, mlkem768}; + } + + // Cached PFS key entry as stored in the pfs_key_cache table. + // fetched_at and pubkeys are nullopt when the entry is a NAK (no valid keys). + struct PfsCacheEntry { + std::optional fetched_at; + std::optional nak_at; + std::optional> pubkey_x25519; + std::optional> pubkey_mlkem768; + }; + + // Returns true if any swarm node has handed us a hash in the given namespace. Used by live + // tests to detect that a poll completed and delivered at least one message. + static bool has_any_namespace_sync(core::Core& core, config::Namespace ns) { + auto count = core.db.conn().prepared_get( + "SELECT COUNT(*) FROM swarm_hashes WHERE namespace = ?", static_cast(ns)); + return count > 0; + } + + // Seeds the pfs_key_cache with PFS keys for a remote session_id. + static void seed_pfs_cache( + core::Core& core, + std::span remote_session_id, + std::span x25519_pub, + std::span mlkem768_pub) { + core._store_pfs_keys(remote_session_id, x25519_pub, mlkem768_pub); + } + + // Seeds a NAK entry in the pfs_key_cache (remote has no published PFS keys). + static void seed_pfs_nak(core::Core& core, std::span remote_session_id) { + core._store_pfs_nak(remote_session_id); + } + + // Returns the pfs_key_cache entry for the given session_id, or nullopt if absent. + static std::optional pfs_cache_entry( + core::Core& core, std::span session_id) { + using X = sqlite::blob_guts>; + using M = sqlite::blob_guts>; + auto row = core.db.conn() + .prepared_maybe_get< + std::optional, + std::optional, + std::optional, + std::optional>( + "SELECT fetched_at, nak_at, pubkey_x25519, pubkey_mlkem768" + " FROM pfs_key_cache WHERE session_id = ?", + session_id); + if (!row) + return std::nullopt; + auto [fetched_at, nak_at, pk_x25519, pk_mlkem768] = *row; + return PfsCacheEntry{ + fetched_at, + nak_at, + pk_x25519 ? std::optional{(std::array)*pk_x25519} : std::nullopt, + pk_mlkem768 ? std::optional{(std::array)*pk_mlkem768} + : std::nullopt}; + } +}; + +} // namespace session diff --git a/tests/test_ip_country.cpp b/tests/test_ip_country.cpp new file mode 100644 index 000000000..aea8fd17a --- /dev/null +++ b/tests/test_ip_country.cpp @@ -0,0 +1,162 @@ +#include +#include +#include +#include + +#include "../src/network/ip_country/data.hpp" + +using namespace session::ip_country; +using namespace oxen::log::literals; + +namespace { + +// The country a range's code index means, i.e. what a lookup anywhere in that range must return. +std::optional country_of(uint8_t code) { + if (code == 0) + return std::nullopt; + return detail::country_codes()[code]; +} + +} // namespace + +TEST_CASE("ip-to-country database shape", "[ip_country]") { + auto starts = detail::range_starts(); + auto codes = detail::range_codes(); + auto table = detail::country_codes(); + + REQUIRE(starts.size() == codes.size()); + REQUIRE(available() == !starts.empty()); + + if (!available()) { + // Built without WITH_IP_GEOLOCATION, so there is nothing to check the shape of beyond its + // being consistently empty; the lookups themselves are exercised below either way. + CHECK(table.empty()); + CHECK(attribution().empty()); + CHECK(database_version().empty()); + return; + } + + CHECK_FALSE(attribution().empty()); + CHECK_FALSE(database_version().empty()); + + // Index 0 is the unknown slot rather than a country; the rest are alpha-2 codes. + REQUIRE(table.size() >= 2); + CHECK(table[0].empty()); + + for (size_t i = 1; i < table.size(); i++) { + auto cc = table[i]; + if (cc.size() != 2 || + !std::ranges::all_of(cc, [](char c) { return c >= 'A' && c <= 'Z'; })) { + FAIL("code table entry " << i << " (" << cc << ") is not an alpha-2 country code"); + break; + } + } + + // A lookup finds the range a search lands in and stops, so the table has to start at 0.0.0.0 + // and ascend; anything else silently mislabels the addresses below the first entry. + CHECK(starts.front() == ipv4{0, 0, 0, 0}); + + size_t out_of_order = 0, bad_code = 0, unmerged = 0; + for (size_t i = 0; i < starts.size(); i++) { + if (i > 0 && !(starts[i - 1] < starts[i])) + out_of_order++; + if (codes[i] >= table.size()) + bad_code++; + // Not a correctness requirement, but the generator merges neighbours with the same country, + // so a run of them means it stopped doing its job. + if (i > 0 && codes[i - 1] == codes[i]) + unmerged++; + } + CHECK(out_of_order == 0); + CHECK(bad_code == 0); + CHECK(unmerged == 0); + + // The codes are numbered by descending range count (ties alphabetical), which is what keeps the + // generated source small and its month-to-month diff shallow. Recount them and check the + // numbering still follows, since a generator that quietly stopped sorting would cost both. + std::vector ranges_per_country(table.size(), 0); + for (auto code : codes) + ranges_per_country[code]++; + + for (size_t i = 2; i < table.size(); i++) { + auto prev = ranges_per_country[i - 1], cur = ranges_per_country[i]; + if (prev < cur || (prev == cur && !(table[i - 1] < table[i]))) { + FAIL("country " << table[i] << " (" << cur << " ranges) is numbered after " + << table[i - 1] << " (" << prev << " ranges)"); + break; + } + } +} + +TEST_CASE("ip-to-country range boundaries", "[ip_country]") { + auto starts = detail::range_starts(); + auto codes = detail::range_codes(); + + if (!available()) { + // Every lookup misses, which is the whole point of the empty database: a client needs no + // #ifdef of its own. + CHECK_FALSE(lookup(ipv4{1, 1, 1, 1})); + CHECK_FALSE(lookup(ipv4{"95.216.0.0"})); + CHECK_FALSE(lookup(ipv4{0, 0, 0, 0})); + CHECK_FALSE(lookup(ipv4{255, 255, 255, 255})); + return; + } + + // Walk a sample of ranges spread across the table, checking each one's first and last address + // and the first address of the next range: an off-by-one in the search shows up as a range + // bleeding into its neighbour. + size_t step = std::max(1, starts.size() / 500); + size_t mismatches = 0; + std::string first_failure; + auto check = [&](ipv4 ip, std::optional expected) { + auto got = lookup(ip); + if (got == expected) + return; + mismatches++; + if (first_failure.empty()) + first_failure = "{} gave {} rather than {}"_format( + ip.to_string(), got.value_or("(unknown)"), expected.value_or("(unknown)")); + }; + + for (size_t i = 0; i < starts.size(); i += step) { + auto expected = country_of(codes[i]); + check(starts[i], expected); + + // The range runs until the next one starts, or to the top of the address space for the + // last one. + ipv4 last = i + 1 < starts.size() ? ipv4{starts[i + 1].addr - 1} : ipv4{255, 255, 255, 255}; + check(last, expected); + if (i + 1 < starts.size()) + check(starts[i + 1], country_of(codes[i + 1])); + } + + INFO(first_failure); + CHECK(mismatches == 0); +} + +TEST_CASE("ip-to-country reserved space", "[ip_country]") { + if (!available()) + return; + + // DB-IP labels space that belongs to no country -- 0.0.0.0/8, the RFC1918 blocks, loopback, + // link-local, multicast and up -- with its ZZ marker, which the generator folds into the + // unknown code. This is the code == 0 path. + CHECK_FALSE(lookup(ipv4{"0.0.0.0"})); + CHECK_FALSE(lookup(ipv4{"10.0.0.1"})); + CHECK_FALSE(lookup(ipv4{"127.0.0.1"})); + CHECK_FALSE(lookup(ipv4{"169.254.1.1"})); + CHECK_FALSE(lookup(ipv4{"192.168.1.1"})); + CHECK_FALSE(lookup(ipv4{"255.255.255.255"})); +} + +TEST_CASE("ip-to-country smoke test against the bundled snapshot", "[ip_country]") { + if (!available()) + return; + + // Unlike everything above, this asserts what the data says rather than how the lookup works, + // so it can legitimately fail after a refresh: Hetzner's Helsinki space is about as stable an + // anchor as free geo data offers, but if this is what breaks, check the new snapshot and move + // the anchor rather than treating it as a bug. + CHECK(lookup(ipv4{"95.216.0.0"}) == "FI"); + CHECK(lookup(ipv4{"95.216.33.113"}) == "FI"); +} diff --git a/tests/test_logging.cpp b/tests/test_logging.cpp index 061874056..2ff5f38c3 100644 --- a/tests/test_logging.cpp +++ b/tests/test_logging.cpp @@ -3,22 +3,19 @@ #include #include #include +#include #include #include #include "utils.hpp" -#ifndef DISABLE_NETWORKING -#include -#endif - using namespace session; using namespace oxen; using namespace oxen::log::literals; std::regex timestamp_re{R"(\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\] \[\+[\d.hms]+\])"}; // Clears timestamps out of a log statement for testing reproducibility -std::string fixup_log(std::string_view log) { +static std::string fixup_log(std::string_view log) { std::string fixed; std::regex_replace( std::back_inserter(fixed), @@ -89,7 +86,6 @@ TEST_CASE("Logging callbacks", "[logging]") { line1)); } -#ifndef DISABLE_NETWORKING TEST_CASE("Logging callbacks with quic::Network", "[logging][network]") { oxen::log::clear_sinks(); simple_logs.clear(); @@ -107,5 +103,4 @@ TEST_CASE("Logging callbacks with quic::Network", "[logging][network]") { CHECK(std::any_of(simple_logs.begin(), simple_logs.end(), [](const std::string& s) { return s.find("[quic:") != std::string::npos; })); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/tests/test_mnemonics.cpp b/tests/test_mnemonics.cpp new file mode 100644 index 000000000..1e8638350 --- /dev/null +++ b/tests/test_mnemonics.cpp @@ -0,0 +1,662 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "utils.hpp" + +using namespace session::mnemonics; +using namespace oxenc::literals; + +// Test vectors: SHA-512("libsession-util mnemonic test vector") encoded as 48 words per language. +// These pin the exact word list contents and ordering; if any word list changes this test fails. +TEST_CASE("Mnemonic word list test vectors", "[mnemonics]") { + // seed = SHA-512("libsession-util mnemonic test vector") + auto seed = + "0dd5d9bc3d68c25a396f4aacd922a4d620a19cf3c9054cb825dd8a2c5420f4f3" + "ca314c582ffef5388df36e2546cc9103dd1776a634f676e1e631289b8d280b2e"_hex_b; + + // clang-format off + const std::pair> expected[] = { + {"English", { + "threaten", "efficient", "wives", "skirting", "repent", "ashtray", + "rural", "ammo", "reunion", "yoyo", "already", "tucks", + "attire", "waxing", "uphill", "template", "ghetto", "anxiety", + "utensils", "newt", "safety", "paper", "quote", "pebbles", + "album", "gnaw", "puppy", "tidy", "foxes", "menu", + "evenings", "spying", "wallets", "plotting", "fuselage", "geometry", + "toilet", "cylinder", "swagger", "eels", "when", "tether", + "cowl", "saga", "gossip", "vats", "bias", "federal", + }}, + {"Chinese (simplified)", { + "忆", "众", "瓷", "坡", "残", "合", "麻", "度", "综", "淀", "得", "炭", + "四", "弃", "隆", "违", "亚", "物", "博", "娘", "缓", "薄", "纤", "暗", + "方", "减", "爷", "浆", "官", "乐", "称", "阀", "蜡", "予", "谈", "落", + "罚", "志", "蓝", "音", "浅", "森", "百", "净", "波", "灌", "无", "格", + }}, + {"Dutch", { + "tray", "erna", "zetbaas", "spijgat", "rits", "bedwelmd", + "salade", "arubaan", "rodijk", "zottebol", "aorta", "vanmiddag", + "belboei", "worp", "voip", "tosti", "glaasje", "auping", + "waas", "neuzelaar", "saus", "pacht", "ramselaar", "pauze", + "amnestie", "goeierd", "puzzelaar", "treur", "gegraaid", "mantel", + "feilbaar", "tabak", "witmaker", "plausibel", "gemiddeld", "giepmans", + "tyfoon", "derf", "ticket", "ermitage", "zalig", "trabant", + "danenberg", "scampi", "groosman", "warklomp", "bolvormig", "formule", + }}, + {"Esperanto", { + "sizifa", "ebena", "viskoza", "rapida", "optimisto", "anjono", + "pezoforto", "alfabeto", "orfino", "zeto", "akselo", "stomako", + "aplikado", "vazaro", "timida", "sidejo", "fermi", "alzaca", + "tosti", "latitudo", "pilkoludo", "moskito", "ofsajdo", "muro", + "akademio", "fimensa", "oblikva", "sklavo", "eskapi", "kisi", + "elektro", "rojo", "vampiro", "neulo", "etullernejo", "feino", + "sodakvo", "cigaredo", "safario", "duzo", "veziko", "simpla", + "cemento", "pimento", "flirti", "tunelo", "babili", "enciklopedio", + }}, + {"French", { + "skier", "devoir", "vingt", "rideau", "profond", "aucun", + "rail", "angoisse", "propre", "vous", "amener", "star", + "avant", "vampire", "tenter", "sigle", "fixe", "ardeur", + "toge", "navrer", "rang", "parmi", "pompier", "pause", + "album", "focus", "poids", "socle", "faible", "miel", + "eaux", "rustre", "vague", "pilote", "faveur", "final", + "songeur", "chiot", "sauge", "devin", "version", "sinon", + "chasse", "rapace", "fosse", "tour", "billet", "enlever", + }}, + {"German", { + "Salz", "Dezibel", "Wind", "plündern", "Mund", "Anker", + "Oberarzt", "Alter", "Nabel", "Zielfoto", "Almosen", "Skikurs", + "Anrecht", "Wahlen", "Tempo", "Rüstung", "Espe", "Amulett", + "Topmodel", "Kampagne", "Ofenholz", "Lavasee", "Milchkuh", "Lerche", + "Aktfoto", "Exil", "melden", "Sanftmut", "Erde", "Hufeisen", + "Edelweiß", "Rapsöl", "Vorrat", "Luxus", "erkalten", "Erzeuger", + "Schulbus", "Bogen", "Respekt", "Detektiv", "wechseln", "Sack", + "Blauwal", "öffnen", "Fakultät", "Trödel", "Bach", "Einzug", + }}, + {"Italian", { + "spegnere", "comune", "vigilare", "sartoria", "pulire", "arachidi", + "retorica", "amnistia", "quaderno", "zainetto", "amante", "subire", + "armonia", "velluto", "tirare", "sospiro", "enigma", "anello", + "trattore", "moglie", "ricambio", "panino", "porzione", "parodia", + "allarme", "esaltare", "polimero", "spezzare", "dorso", "madama", + "cupola", "seme", "vegetale", "pianeta", "eclissi", "emisfero", + "stadio", "cannone", "silicone", "compagna", "vertebra", "spalla", + "calzone", "ricetta", "estrarre", "tulipano", "bagaglio", "dialogo", + }}, + {"Japanese", { + "なさけ", "きかく", "はらう", "でこぼこ", "たんとう", "いとこ", + "ちゃんこなべ", "いさましい", "たんぴん", "ひかく", "いきもの", "にっさん", + "いふく", "はせる", "ねっしん", "どんぶり", "けちゃっぷ", "いそがしい", + "ねんかん", "せいよう", "ちらみ", "そめる", "たぼう", "そんぞく", + "あんてい", "けとばす", "だったい", "ななおし", "くめる", "しょっけん", + "きまる", "てんぷら", "はこぶ", "たいめん", "けいけん", "けしき", + "なれる", "おじさん", "とくしゅう", "きかい", "はったつ", "ないそう", + "おくる", "ちりがみ", "けみかる", "のがす", "うせつ", "くうき", + }}, + {"Lojban", { + "vasxu", "ferti", "rarbau", "tadji", "sisku", "cando", + "sobde", "bloti", "skami", "faumlu", "birka", "xabju", + "carna", "jbogu'e", "xruki", "tutci", "jicmu", "briju", + "zbabu", "panje", "sombo", "ransu", "senpi", "rekto", + "bifce", "jinku", "savru", "vensa", "jarco", "murta", + "gapru", "temse", "jbocre", "rupnu", "jdini", "jgira", + "viska", "dansu", "toldi", "fepri", "reisku", "vamji", + "dacti", "sonci", "jivbu", "zifre", "cinza", "grake", + }}, + {"Portuguese", { + "sonso", "druso", "vontade", "riacho", "paxa", "arlequim", + "porvir", "alvura", "pegaso", "xodo", "alhures", "tavola", + "ascorbico", "vetusto", "trovoar", "slide", "feto", "anotar", + "ufologo", "mausoleu", "prezar", "nouveau", "otite", "nutritivo", + "ajudante", "fiorde", "orla", "sossego", "exaustor", "lele", + "emulsao", "rural", "veja", "ojeriza", "faixas", "feltro", + "suor", "cluster", "seara", "dropes", "viquingue", "soerguer", + "cinzento", "privilegios", "foco", "unheiro", "bemol", "ereto", + }}, + {"Russian", { + "уровень", "древний", "эмблема", "тайна", "сельский", "бегство", + "согласие", "арсенал", "сечение", "язык", "аптека", "фишка", + "бивень", "шрам", "центр", "умолять", "исходить", "атрибут", + "чепуха", "отбор", "сонный", "пуля", "рюкзак", "пшеница", + "анкета", "капитан", "рыба", "ускорять", "иголка", "область", + "женщина", "трибуна", "шорох", "ресурс", "изоляция", "ипподром", + "ушко", "гамма", "тянуть", "драка", "щель", "уплата", + "выходить", "сообщать", "кенгуру", "чужой", "быстрый", "зачет", + }}, + {"Spanish", { + "pasta", "chiste", "relieve", "obtener", "mito", "anillo", + "músculo", "aleta", "moho", "riego", "alambre", "pésimo", + "añejo", "reacción", "pompa", "parcela", "diente", "altura", + "previo", "intuir", "nación", "llanto", "mensaje", "loción", + "aguja", "divino", "mecha", "pausa", "curva", "héroe", + "collar", "óptica", "rasgo", "mamut", "dejar", "diamante", + "pellejo", "brote", "otoño", "chico", "reflejo", "párrafo", + "bozal", "nadar", "droga", "pronto", "astro", "crear", + }}, + }; + // clang-format on + + for (auto& [lang_name, exp_words] : expected) { + SECTION(std::string(lang_name)) { + auto* lang = find_language(lang_name); + REQUIRE(lang); + auto mnemonic = bytes_to_words(seed, *lang, false); + REQUIRE(mnemonic.size() == 48); + auto wspan = mnemonic.open(); + for (size_t i = 0; i < 48; i++) + CHECK(wspan[i] == exp_words[i]); + } + } +} + +TEST_CASE("Mnemonic round-trip tests", "[mnemonics]") { + std::vector data_128(16); + std::vector data_256(32); + + std::mt19937 gen(42); + std::uniform_int_distribution dist(0, 255); + + auto fill_random = [&](std::vector& v) { + for (auto& b : v) + b = static_cast(dist(gen)); + }; + + fill_random(data_128); + fill_random(data_256); + + for (auto lang : get_languages()) { + SECTION("Language: " + std::string(lang->english_name)) { + // 128-bit -> 12 words -> 128-bit + auto words12 = bytes_to_words(data_128, *lang, false); + CHECK(words12.size() == 12); + auto back12 = words_to_bytes(words12.open().words, *lang); + CHECK(std::ranges::equal(back12.access().buf, data_128)); + + // 128-bit -> 13 words (with checksum) -> 128-bit + auto words13 = bytes_to_words(data_128, *lang); + CHECK(words13.size() == 13); + auto back13 = words_to_bytes(words13.open().words, *lang); + CHECK(std::ranges::equal(back13.access().buf, data_128)); + + // 256-bit -> 24 words -> 256-bit + auto words24 = bytes_to_words(data_256, *lang, false); + CHECK(words24.size() == 24); + auto back24 = words_to_bytes(words24.open().words, *lang); + CHECK(std::ranges::equal(back24.access().buf, data_256)); + + // 256-bit -> 25 words (with checksum) -> 256-bit + auto words25 = bytes_to_words(data_256, *lang); + CHECK(words25.size() == 25); + auto back25 = words_to_bytes(words25.open().words, *lang); + CHECK(std::ranges::equal(back25.access().buf, data_256)); + } + } +} + +TEST_CASE("Mnemonic case-insensitivity and prefix matching", "[mnemonics]") { + auto english = find_language("English"); + REQUIRE(english); + + // 4 bytes: [0x01, 0x02, 0x03, 0x04] + // V = 0x04030201 = 67305985 + // A = 67305985 % 1626 = 1443 + // B = (67305985 / 1626 + 1443) % 1626 = (41393 + 1443) % 1626 = 42836 % 1626 = 180 + // C = (67305985 / 1626 / 1626 + 180) % 1626 = (25 + 180) % 1626 = 205 + + // Words for English at indices 1443, 180, 205 + std::vector data = { + std::byte{0x01}, std::byte{0x02}, std::byte{0x03}, std::byte{0x04}}; + auto words = bytes_to_words(data, *english, false); + REQUIRE(words.size() == 3); + + SECTION("Exact match") { + auto back = words_to_bytes(words.open().words, *english); + CHECK(std::ranges::equal(back.access().buf, data)); + } + + SECTION("Case-insensitive match (ASCII)") { + std::vector upper_words; + std::vector storage; + for (auto w : words.open()) { + std::string upper(w); + for (auto& c : upper) + c = std::toupper(static_cast(c)); + storage.push_back(upper); + } + for (const auto& s : storage) + upper_words.push_back(s); + + auto back = words_to_bytes(upper_words, *english); + CHECK(std::ranges::equal(back.access().buf, data)); + } + + SECTION("Prefix match") { + std::vector prefix_words; + std::vector storage; + for (auto w : words.open()) { + storage.push_back(std::string(w.substr(0, english->prefix_len))); + } + for (const auto& s : storage) + prefix_words.push_back(s); + + auto back = words_to_bytes(prefix_words, *english); + CHECK(std::ranges::equal(back.access().buf, data)); + } + + SECTION("Prefix match with typo after prefix") { + std::vector typo_words; + std::vector storage; + for (auto w : words.open()) { + storage.push_back(std::string(w.substr(0, english->prefix_len)) + "xyz"); + } + for (const auto& s : storage) + typo_words.push_back(s); + + auto back = words_to_bytes(typo_words, *english); + CHECK(std::ranges::equal(back.access().buf, data)); + } +} + +TEST_CASE("Mnemonic language lookup", "[mnemonics]") { + CHECK(find_language("English") != nullptr); + CHECK(find_language("German") != nullptr); + CHECK(find_language("Deutsch") != nullptr); + CHECK(find_language("русский язык") != nullptr); + CHECK(find_language("NonExistent") == nullptr); +} + +TEST_CASE("Mnemonic checksum", "[mnemonics]") { + auto english = find_language("English"); + REQUIRE(english); + + std::vector data = { + std::byte{0x01}, std::byte{0x02}, std::byte{0x03}, std::byte{0x04}}; + auto words3 = bytes_to_words(data, *english, false); + REQUIRE(words3.size() == 3); + auto words4 = bytes_to_words(data, *english); + REQUIRE(words4.size() == 4); + + SECTION("Checksum word duplicates one of the seed words") { + // Which one is chosen is pinned by the reference vectors below; the invariant here is that + // it is always a repeat of a seed word rather than an independent thirteenth word. + auto s3 = words3.open(); + auto s4 = words4.open(); + CHECK((s4[3] == s3[0] || s4[3] == s3[1] || s4[3] == s3[2])); + } + + SECTION("Checksum round-trip") { + auto back = words_to_bytes(words4.open().words, *english); + CHECK(std::ranges::equal(back.access().buf, data)); + } + + SECTION("Bad checksum throws checksum_error") { + // A known word that is not any of the seed words cannot be the checksum word, whichever of + // them the CRC selects. + auto s4 = words4.open(); + std::string_view other; + for (auto w : english->words) + if (w != s4[0] && w != s4[1] && w != s4[2]) { + other = w; + break; + } + REQUIRE(!other.empty()); + std::vector bad = {s4[0], s4[1], s4[2], other}; + CHECK_THROWS_AS(words_to_bytes(bad, *english), checksum_error); + } + + SECTION("Unknown checksum word throws unknown_word_error") { + auto s4 = words4.open(); + std::vector bad = {s4[0], s4[1], s4[2], "ZZZunknown"}; + CHECK_THROWS_AS(words_to_bytes(bad, *english), unknown_word_error); + } +} + +TEST_CASE("Mnemonic error handling", "[mnemonics]") { + auto english = find_language("English"); + + SECTION("Invalid byte length") { + std::vector data(15); + CHECK_THROWS_AS(bytes_to_words(data, *english), std::invalid_argument); + } + + SECTION("Invalid word count") { + std::vector words = {"abbey", "abducts"}; + CHECK_THROWS_AS(words_to_bytes(words, *english), std::invalid_argument); + } + + SECTION("Unknown word") { + // Use mixed case to verify word() returns the original input, not a lowercased prefix. + // "ZZZ..." has prefix "zzz" which is not in the English word list. + std::vector words = {"abbey", "abducts", "ZZZunknown"}; + CHECK_THROWS_AS(words_to_bytes(words, *english), unknown_word_error); + try { + words_to_bytes(words, *english); + } catch (const unknown_word_error& e) { + CHECK(e.word() == "ZZZunknown"); + } + } + + SECTION("Overflow word triplet") { + // a=0 (abbey), b=0 (abbey), c=1625 (zoom): + // 0 + 0 + 1625*1626² = 4,296,298,500 > UINT32_MAX — must be rejected + std::vector words = {"abbey", "abbey", "zoom"}; + CHECK_THROWS_AS(words_to_bytes(words, *english), std::invalid_argument); + } +} + +// ── Checksum word ─────────────────────────────────────────────────────────────────────────────── +// +// The checksum word repeats one of the seed words, selected by a CRC-32 over their concatenated +// prefixes modulo the word count. The vectors below were produced from that algorithm as it is +// implemented in oxen-core's electrum-words.cpp (boost::crc_32_type over the trimmed words) and, +// independently, session-ios' Mnemonic.swift -- the two shipping implementations libsession has to +// interoperate with. +TEST_CASE("mnemonics: checksum word matches the reference algorithm", "[mnemonics][checksum]") { + constexpr auto seed16 = "000102030405060708090a0b0c0d0e0f"_hex_b; + constexpr auto seed32 = + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"_hex_b; + + auto words_of = [](std::span seed) { + auto m = session::mnemonics::bytes_to_words(seed, "English"); + auto opened = m.open(); + return std::vector{opened.begin(), opened.end()}; + }; + + SECTION("13 words") { + auto w = words_of(seed16); + REQUIRE(w.size() == 13); + CHECK(fmt::format("{}", fmt::join(w, " ")) == + "amaze buffet cake entrance symptoms tiger lamb maze nestle python dusted faxed " + "faxed"); + // The checksum word is a repeat of one of the seed words, not a thirteenth distinct one. + CHECK(w.back() == w[11]); + } + + SECTION("25 words") { + auto w = words_of(seed32); + REQUIRE(w.size() == 25); + CHECK(fmt::format("{}", fmt::join(w, " ")) == + "amaze buffet cake entrance symptoms tiger lamb maze nestle python dusted faxed " + "update vague zinger boxes ornament renting glass gained island nabbing afield " + "calamity boxes"); + CHECK(w.back() == w[15]); + } + + SECTION("an all-zero seed still checksums") { + auto w = words_of("00000000000000000000000000000000"_hex_b); + REQUIRE(w.size() == 13); + for (const auto& word : w) + CHECK(word == "abbey"); + } +} + +TEST_CASE("mnemonics: phrases round-trip through the checksum", "[mnemonics][checksum]") { + for (auto hex : + {"000102030405060708090a0b0c0d0e0f", + "ffffffffffffffffffffffffffffffff", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"}) { + auto seed = oxenc::from_hex(std::string_view{hex}); + auto bytes = std::as_bytes(std::span{seed}); + + auto m = session::mnemonics::bytes_to_words(bytes, "English"); + auto back = session::mnemonics::words_to_bytes( + m.open().words, session::mnemonics::get_language("English")); + auto acc = back.access(); + CHECK(std::ranges::equal(std::span{acc.buf}, bytes)); + } +} + +TEST_CASE("mnemonics: only the significant prefix matters", "[mnemonics][checksum]") { + // English uses a 3-codepoint prefix, so anything past the third letter is decoration: a phrase + // stays valid when the tail of a word is mistyped, or truncated to just the prefix. This is + // the property that makes the word list usable at all, and it has to hold for the checksum word + // as well as the seed words, since the checksum is computed over prefixes too. + constexpr auto seed = "000102030405060708090a0b0c0d0e0f"_hex_b; + auto expected = std::vector{seed.begin(), seed.end()}; + + auto decodes_to_seed = [&](std::vector w) { + auto back = + session::mnemonics::words_to_bytes(w, session::mnemonics::get_language("English")); + auto acc = back.access(); + return std::ranges::equal(std::span{acc.buf}, expected); + }; + + // amaze buffet cake entrance symptoms tiger lamb maze nestle python dusted faxed | faxed + SECTION("typos past the third letter") { + CHECK(decodes_to_seed( + {"amazing", + "buffalo", + "cakewalk", + "entropy", + "symbol", + "tigger", + "lambda", + "mazurka", + "nestling", + "pythons", + "dustpan", + "faxing", + "faxing"})); + } + + SECTION("words truncated to their prefix") { + CHECK(decodes_to_seed( + {"ama", + "buf", + "cak", + "ent", + "sym", + "tig", + "lam", + "maz", + "nes", + "pyt", + "dus", + "fax", + "fax"})); + } + + SECTION("a mistyped tail on the checksum word alone") { + CHECK(decodes_to_seed( + {"amaze", + "buffet", + "cake", + "entrance", + "symptoms", + "tiger", + "lamb", + "maze", + "nestle", + "python", + "dusted", + "faxed", + "faxidermy"})); + } +} + +TEST_CASE("mnemonics: rejects bad phrases", "[mnemonics][checksum]") { + auto lang = std::cref(session::mnemonics::get_language("English")); + auto decode = [&](std::vector w) { + return session::mnemonics::words_to_bytes(w, lang.get()); + }; + + // Correct phrase, for reference: + // amaze buffet cake entrance symptoms tiger lamb maze nestle python dusted faxed | faxed + SECTION("wrong checksum word") { + CHECK_THROWS_AS( + decode({"amaze", + "buffet", + "cake", + "entrance", + "symptoms", + "tiger", + "lamb", + "maze", + "nestle", + "python", + "dusted", + "faxed", + "amaze"}), + session::mnemonics::checksum_error); + } + + SECTION("two seed words transposed") { + // Same words, so a sum-of-indices checksum would not notice; a CRC over the ordered + // prefixes does. + CHECK_THROWS_AS( + decode({"buffet", + "amaze", + "cake", + "entrance", + "symptoms", + "tiger", + "lamb", + "maze", + "nestle", + "python", + "dusted", + "faxed", + "faxed"}), + session::mnemonics::checksum_error); + } + + SECTION("one seed word altered before the prefix boundary") { + CHECK_THROWS_AS( + decode({"amaze", + "buffet", + "cake", + "entrance", + "symptoms", + "tiger", + "lamb", + "maze", + "nestle", + "python", + "dusted", + "gagged", + "faxed"}), + session::mnemonics::checksum_error); + } + + SECTION("a word that is not in the list at all") { + CHECK_THROWS_AS( + decode({"amaze", + "buffet", + "cake", + "entrance", + "symptoms", + "tiger", + "lamb", + "maze", + "nestle", + "python", + "dusted", + "faxed", + "zzzzz"}), + session::mnemonics::unknown_word_error); + } +} + +// ── Input canonicalisation ────────────────────────────────────────────────────────────────────── +// +// The word lists are NFC. A user whose input path produces NFD -- macOS filesystem APIs, several +// IMEs, some PDF and web copy paths -- types something that renders identically but is a different +// byte sequence. For someone restoring an account that is a lockout, so both forms have to reach +// the same seed. +// +// The decomposed spellings below use \u escapes rather than literal combining sequences, so that no +// editor or tool can quietly normalise them into the composed form and leave these tests comparing +// a string with itself. The static_asserts fail the build if that happens regardless. +namespace { +// "mögen": ö is U+00F6 composed, o + U+0308 decomposed. +constexpr std::string_view moegen_nfc = "mögen"; +constexpr std::string_view moegen_nfd = "mo\u0308gen"; +static_assert(moegen_nfc != moegen_nfd, "the decomposed spelling has been normalised away"); + +// "тайна": й is U+0439 composed, и + U+0306 decomposed. German's prefix is 4 codepoints and +// Russian's is 3, so in both cases the combining mark falls outside the prefix window. +constexpr std::string_view tajna_nfc = "тайна"; +constexpr std::string_view tajna_nfd = "таи\u0306на"; +static_assert(tajna_nfc != tajna_nfd, "the decomposed spelling has been normalised away"); + +constexpr std::string_view oestlich_lower = "östlich"; +constexpr std::string_view oestlich_upper = "ÖSTLICH"; +static_assert(oestlich_lower != oestlich_upper); +} // namespace + +TEST_CASE("mnemonics: a phrase decodes the same however it was typed", "[mnemonics][unicode]") { + // Crash mögen Muster, repeated -- chosen because it contains a decomposable word. + constexpr auto seed = "04040404040404040404040404040404"_hex_b; + auto& german = session::mnemonics::get_language("German"); + + auto expected = std::vector{seed.begin(), seed.end()}; + auto decode = [&](std::vector w) { + auto back = session::mnemonics::words_to_bytes(w, german); + auto acc = back.access(); + return std::vector{acc.buf.begin(), acc.buf.end()}; + }; + + auto m = session::mnemonics::bytes_to_words(seed, german); + auto opened = m.open(); + std::vector composed{opened.begin(), opened.end()}; + REQUIRE(decode(composed) == expected); + REQUIRE(std::ranges::count(composed, moegen_nfc) > 0); + + SECTION("every occurrence decomposed") { + auto w = composed; + std::ranges::replace(w, moegen_nfc, moegen_nfd); + CHECK(decode(w) == expected); + } + + SECTION("decomposed in the checksum word too") { + auto w = composed; + w.back() = w.back() == moegen_nfc ? moegen_nfd : w.back(); + CHECK(decode(w) == expected); + } +} + +TEST_CASE("mnemonics: case folds beyond ASCII", "[mnemonics][unicode]") { + // tolower() only folds single bytes, so an all-caps non-ASCII word used to be unrecognisable. + // towlower() would fold it, but only under a UTF-8 locale -- LC_ALL=C would silently disable + // it. + constexpr auto seed = "04040404040404040404040404040404"_hex_b; + auto& german = session::mnemonics::get_language("German"); + + auto m = session::mnemonics::bytes_to_words(seed, german); + auto opened = m.open(); + std::vector w{opened.begin(), opened.end()}; + std::ranges::replace(w, moegen_nfc, std::string_view{"MÖGEN"}); + + auto back = session::mnemonics::words_to_bytes(w, german); + auto acc = back.access(); + CHECK(std::ranges::equal(acc.buf, std::span{seed})); +} + +TEST_CASE("mnemonics: decomposed input does not match a different word", "[mnemonics][unicode]") { + // The dangerous case: й decomposes to и + U+0306, the mark falls outside Russian's 3-codepoint + // window, and "тайна" truncates to "таи" -- which is "таинство". Without normalisation the + // lookup succeeds against the wrong word and silently decodes to a different seed, which is + // worse than a rejection: nothing tells the user their phrase was misread. + auto& russian = session::mnemonics::get_language("Russian"); + + auto decode = [&](std::string_view word) { + std::vector phrase(12, word); + auto back = session::mnemonics::words_to_bytes(phrase, russian); + auto acc = back.access(); + return std::vector{acc.buf.begin(), acc.buf.end()}; + }; + + // The word the truncation would collide with really is a different word with a different seed, + // so the check below is meaningful rather than vacuous. + auto composed = decode(tajna_nfc); + CHECK(decode("таинство") != composed); + + // Typed decomposed, it must still be тайна. + CHECK(decode(tajna_nfd) == composed); +} diff --git a/tests/test_multi_encrypt.cpp b/tests/test_multi_encrypt.cpp index 6d8491fab..dca3da43a 100644 --- a/tests/test_multi_encrypt.cpp +++ b/tests/test_multi_encrypt.cpp @@ -1,69 +1,64 @@ #include #include -#include -#include #include +#include +#include #include #include #include "utils.hpp" -using x_pair = std::pair, std::array>; - -// Returns X25519 privkey, pubkey from an Ed25519 seed -x_pair to_x_keys(std::span ed_seed) { - std::array ed_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair(ed_pk.data(), ed_sk.data(), ed_seed.data()); - x_pair ret; - auto& [x_priv, x_pub] = ret; - [[maybe_unused]] int rc = crypto_sign_ed25519_pk_to_curve25519(x_pub.data(), ed_pk.data()); - assert(rc == 0); - crypto_sign_ed25519_sk_to_curve25519(x_priv.data(), ed_sk.data()); - return ret; +using namespace session; + +using x_pair = std::pair; + +// Returns X25519 {privkey, pubkey} from an Ed25519 seed +static x_pair to_x_keys(std::span ed_seed) { + auto [ed_pk, ed_sk] = ed25519::keypair(ed_seed); + return {ed25519::sk_to_x25519(ed_sk), ed25519::pk_to_x25519(ed_pk)}; } TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { const std::array seeds = { - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes, - "0123456789abcdef000000000000000000000000000000000000000000000000"_hexbytes, - "0123456789abcdef111111111111111100000000000000000000000000000000"_hexbytes, - "0123456789abcdef222222222222222200000000000000000000000000000000"_hexbytes, - "0123456789abcdef333333333333333300000000000000000000000000000000"_hexbytes}; + "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b, + "0123456789abcdef000000000000000000000000000000000000000000000000"_hex_b, + "0123456789abcdef111111111111111100000000000000000000000000000000"_hex_b, + "0123456789abcdef222222222222222200000000000000000000000000000000"_hex_b, + "0123456789abcdef333333333333333300000000000000000000000000000000"_hex_b}; std::array x_keys; for (size_t i = 0; i < seeds.size(); i++) x_keys[i] = to_x_keys(seeds[i]); - CHECK(oxenc::to_hex(session::to_span(x_keys[0].second)) == + CHECK(oxenc::to_hex(x_keys[0].second) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - CHECK(oxenc::to_hex(session::to_span(x_keys[1].second)) == + CHECK(oxenc::to_hex(x_keys[1].second) == "d673a8fb4800d2a252d2fc4e3342a88cdfa9412853934e8993d12d593be13371"); - CHECK(oxenc::to_hex(session::to_span(x_keys[2].second)) == + CHECK(oxenc::to_hex(x_keys[2].second) == "afd9716ea69ab8c7f475e1b250c86a6539e260804faecf2a803e9281a4160738"); - CHECK(oxenc::to_hex(session::to_span(x_keys[3].second)) == + CHECK(oxenc::to_hex(x_keys[3].second) == "03be14feabd59122349614b88bdc90db1d1af4c230e9a73c898beec833d51f11"); - CHECK(oxenc::to_hex(session::to_span(x_keys[4].second)) == + CHECK(oxenc::to_hex(x_keys[4].second) == "27b5c1ea87cef76284c752fa6ee1b9186b1a95e74e8f5b88f8b47e5191ce6f08"); - auto nonce = "32ab4bb45d6df5cc14e1c330fb1a8b68ea3826a8c2213a49"_hexbytes; + auto nonce = "32ab4bb45d6df5cc14e1c330fb1a8b68ea3826a8c2213a49"_hex_b; - std::vector> recipients; + std::vector> recipients; for (auto& [_, pubkey] : x_keys) recipients.emplace_back(pubkey.data(), pubkey.size()); std::vector msgs{{"hello", "cruel", "world"}}; - std::vector> encrypted; + std::vector> encrypted; session::encrypt_for_multiple( msgs[0], - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), nonce, - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + x_keys[0].first, + x_keys[0].second, "test suite", - [&](std::span enc) { + [&](std::span enc) { encrypted.emplace_back(session::to_vector(enc)); }); @@ -73,39 +68,39 @@ TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { CHECK(to_hex(encrypted[2]) == "01c4fc2156327735f3fb5063b11ea95f6ebcc5b6cc"); auto m1 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[1].first), - session::to_span(x_keys[1].second), - session::to_span(x_keys[0].second), + x_keys[1].first, + x_keys[1].second, + x_keys[0].second, "test suite"); auto m2 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[2].first), - session::to_span(x_keys[2].second), - session::to_span(x_keys[0].second), + x_keys[2].first, + x_keys[2].second, + x_keys[0].second, "test suite"); auto m3 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), + x_keys[3].first, + x_keys[3].second, + x_keys[0].second, "test suite"); auto m3b = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), + x_keys[3].first, + x_keys[3].second, + x_keys[0].second, "not test suite"); auto m4 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[4].first), - session::to_span(x_keys[4].second), - session::to_span(x_keys[0].second), + x_keys[4].first, + x_keys[4].second, + x_keys[0].second, "test suite"); REQUIRE(m1); @@ -120,13 +115,13 @@ TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { encrypted.clear(); session::encrypt_for_multiple( - session::to_view_vector(msgs.begin(), msgs.end()), - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + to_view_vector(msgs.begin(), msgs.end()), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), nonce, - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + x_keys[0].first, + x_keys[0].second, "test suite", - [&](std::span enc) { + [&](std::span enc) { encrypted.emplace_back(session::to_vector(enc)); }); @@ -136,39 +131,39 @@ TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { CHECK(to_hex(encrypted[2]) == "1ecee2215d226817edfdb097f05037eb799309103a"); m1 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[1].first), - session::to_span(x_keys[1].second), - session::to_span(x_keys[0].second), + x_keys[1].first, + x_keys[1].second, + x_keys[0].second, "test suite"); m2 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[2].first), - session::to_span(x_keys[2].second), - session::to_span(x_keys[0].second), + x_keys[2].first, + x_keys[2].second, + x_keys[0].second, "test suite"); m3 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), + x_keys[3].first, + x_keys[3].second, + x_keys[0].second, "test suite"); m3b = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), + x_keys[3].first, + x_keys[3].second, + x_keys[0].second, "not test suite"); m4 = session::decrypt_for_multiple( - session::to_view_vector(encrypted), + to_view_vector(encrypted), nonce, - session::to_span(x_keys[4].first), - session::to_span(x_keys[4].second), - session::to_span(x_keys[0].second), + x_keys[4].first, + x_keys[4].second, + x_keys[0].second, "test suite"); REQUIRE(m1); @@ -183,13 +178,13 @@ TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { // Mismatch messages & recipients size throws: CHECK_THROWS(session::encrypt_for_multiple( - session::to_view_vector(msgs.begin(), std::prev(msgs.end())), - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + to_view_vector(msgs.begin(), std::prev(msgs.end())), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), nonce, - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + x_keys[0].first, + x_keys[0].second, "test suite", - [&](std::span enc) { + [&](std::span enc) { encrypted.emplace_back(session::to_vector(enc)); })); } @@ -197,62 +192,61 @@ TEST_CASE("Multi-recipient encryption", "[encrypt][multi]") { TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][simple]") { const std::array seeds = { - "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes, - "0123456789abcdef000000000000000000000000000000000000000000000000"_hexbytes, - "0123456789abcdef111111111111111100000000000000000000000000000000"_hexbytes, - "0123456789abcdef222222222222222200000000000000000000000000000000"_hexbytes, - "0123456789abcdef333333333333333300000000000000000000000000000000"_hexbytes}; + "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b, + "0123456789abcdef000000000000000000000000000000000000000000000000"_hex_b, + "0123456789abcdef111111111111111100000000000000000000000000000000"_hex_b, + "0123456789abcdef222222222222222200000000000000000000000000000000"_hex_b, + "0123456789abcdef333333333333333300000000000000000000000000000000"_hex_b}; std::array x_keys; for (size_t i = 0; i < seeds.size(); i++) x_keys[i] = to_x_keys(seeds[i]); - CHECK(oxenc::to_hex(session::to_span(x_keys[0].second)) == + CHECK(oxenc::to_hex(x_keys[0].second) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - CHECK(oxenc::to_hex(session::to_span(x_keys[1].second)) == + CHECK(oxenc::to_hex(x_keys[1].second) == "d673a8fb4800d2a252d2fc4e3342a88cdfa9412853934e8993d12d593be13371"); - CHECK(oxenc::to_hex(session::to_span(x_keys[2].second)) == + CHECK(oxenc::to_hex(x_keys[2].second) == "afd9716ea69ab8c7f475e1b250c86a6539e260804faecf2a803e9281a4160738"); - CHECK(oxenc::to_hex(session::to_span(x_keys[3].second)) == + CHECK(oxenc::to_hex(x_keys[3].second) == "03be14feabd59122349614b88bdc90db1d1af4c230e9a73c898beec833d51f11"); - CHECK(oxenc::to_hex(session::to_span(x_keys[4].second)) == + CHECK(oxenc::to_hex(x_keys[4].second) == "27b5c1ea87cef76284c752fa6ee1b9186b1a95e74e8f5b88f8b47e5191ce6f08"); - auto nonce = "32ab4bb45d6df5cc14e1c330fb1a8b68ea3826a8c2213a49"_hexbytes; + auto nonce = "32ab4bb45d6df5cc14e1c330fb1a8b68ea3826a8c2213a49"_hex_b; - std::vector> recipients; + std::vector> recipients; for (auto& [_, pubkey] : x_keys) recipients.emplace_back(pubkey.data(), pubkey.size()); std::vector msgs{{"hello", "cruel", "world"}}; - std::vector encrypted = session::encrypt_for_multiple_simple( + std::vector encrypted = encrypt_for_multiple_simple( msgs[0], - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + x_keys[0].first, + x_keys[0].second, "test suite"); REQUIRE(encrypted.size() == /* de */ 2 + /* 1:# 24:...nonce... */ 3 + 27 + /* 1:e le */ 3 + 2 + - /* XX: then data with overhead */ 3 * - (3 + 5 + crypto_aead_xchacha20poly1305_ietf_ABYTES)); + /* XX: then data with overhead */ 3 * (3 + 5 + encryption::XCHACHA20_ABYTES)); // If we encrypt again the value should be different (because of the default randomized nonce): - CHECK(encrypted != session::encrypt_for_multiple_simple( - msgs[0], - session::to_view_vector( - std::next(recipients.begin()), std::prev(recipients.end())), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), - "test suite")); - - auto padded = session::encrypt_for_multiple_simple( + CHECK(encrypted != + encrypt_for_multiple_simple( + msgs[0], + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + x_keys[0].first, + x_keys[0].second, + "test suite")); + + auto padded = encrypt_for_multiple_simple( msgs[0], - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + x_keys[0].first, + x_keys[0].second, "test suite", nonce, 4); @@ -261,26 +255,20 @@ TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][sim auto padded_list = padded_dict.require("e"); std::vector padded_sizes; while (!padded_list.is_finished()) - padded_sizes.push_back(padded_list.consume>().size()); + padded_sizes.push_back(padded_list.consume>().size()); - CHECK(padded_sizes == - std::vector(4, msgs[0].size() + crypto_aead_xchacha20poly1305_ietf_ABYTES)); + CHECK(padded_sizes == std::vector(4, msgs[0].size() + encryption::XCHACHA20_ABYTES)); - auto padded_message = session::decrypt_for_multiple_simple( - padded, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), - "test suite"); + auto padded_message = decrypt_for_multiple_simple( + padded, x_keys[3].first, x_keys[3].second, x_keys[0].second, "test suite"); REQUIRE(padded_message); CHECK(session::to_string(*padded_message) == msgs[0]); - auto empty_padded = session::encrypt_for_multiple_simple( + auto empty_padded = encrypt_for_multiple_simple( std::string_view{}, - session::to_view_vector( - std::next(recipients.begin()), std::next(recipients.begin(), 2)), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + to_view_vector(std::next(recipients.begin()), std::next(recipients.begin(), 2)), + x_keys[0].first, + x_keys[0].second, "test suite", nonce, 4); @@ -290,49 +278,25 @@ TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][sim std::vector empty_padded_sizes; while (!empty_padded_list.is_finished()) empty_padded_sizes.push_back( - empty_padded_list.consume>().size()); + empty_padded_list.consume>().size()); - CHECK(empty_padded_sizes == std::vector(4, crypto_aead_xchacha20poly1305_ietf_ABYTES)); + CHECK(empty_padded_sizes == std::vector(4, encryption::XCHACHA20_ABYTES)); - auto empty_message = session::decrypt_for_multiple_simple( - empty_padded, - session::to_span(x_keys[1].first), - session::to_span(x_keys[1].second), - session::to_span(x_keys[0].second), - "test suite"); + auto empty_message = decrypt_for_multiple_simple( + empty_padded, x_keys[1].first, x_keys[1].second, x_keys[0].second, "test suite"); REQUIRE(empty_message); CHECK(empty_message->empty()); - auto m1 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[1].first), - session::to_span(x_keys[1].second), - session::to_span(x_keys[0].second), - "test suite"); - auto m2 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[2].first), - session::to_span(x_keys[2].second), - session::to_span(x_keys[0].second), - "test suite"); - auto m3 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), - "test suite"); - auto m3b = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), - "not test suite"); - auto m4 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[4].first), - session::to_span(x_keys[4].second), - session::to_span(x_keys[0].second), - "test suite"); + auto m1 = decrypt_for_multiple_simple( + encrypted, x_keys[1].first, x_keys[1].second, x_keys[0].second, "test suite"); + auto m2 = decrypt_for_multiple_simple( + encrypted, x_keys[2].first, x_keys[2].second, x_keys[0].second, "test suite"); + auto m3 = decrypt_for_multiple_simple( + encrypted, x_keys[3].first, x_keys[3].second, x_keys[0].second, "test suite"); + auto m3b = decrypt_for_multiple_simple( + encrypted, x_keys[3].first, x_keys[3].second, x_keys[0].second, "not test suite"); + auto m4 = decrypt_for_multiple_simple( + encrypted, x_keys[4].first, x_keys[4].second, x_keys[0].second, "test suite"); REQUIRE(m1); REQUIRE(m2); @@ -344,11 +308,11 @@ TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][sim CHECK(session::to_string(*m2) == "hello"); CHECK(session::to_string(*m3) == "hello"); - encrypted = session::encrypt_for_multiple_simple( - session::to_view_vector(msgs), - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + encrypted = encrypt_for_multiple_simple( + to_view_vector(msgs), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + x_keys[0].first, + x_keys[0].second, "test suite", nonce); @@ -359,36 +323,16 @@ TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][sim "bcb642c49c6da03f70cdaab2ed6666721318afd631"_hex, "1ecee2215d226817edfdb097f05037eb799309103a"_hex)); - m1 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[1].first), - session::to_span(x_keys[1].second), - session::to_span(x_keys[0].second), - "test suite"); - m2 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[2].first), - session::to_span(x_keys[2].second), - session::to_span(x_keys[0].second), - "test suite"); - m3 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), - "test suite"); - m3b = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[3].first), - session::to_span(x_keys[3].second), - session::to_span(x_keys[0].second), - "not test suite"); - m4 = session::decrypt_for_multiple_simple( - encrypted, - session::to_span(x_keys[4].first), - session::to_span(x_keys[4].second), - session::to_span(x_keys[0].second), - "test suite"); + m1 = decrypt_for_multiple_simple( + encrypted, x_keys[1].first, x_keys[1].second, x_keys[0].second, "test suite"); + m2 = decrypt_for_multiple_simple( + encrypted, x_keys[2].first, x_keys[2].second, x_keys[0].second, "test suite"); + m3 = decrypt_for_multiple_simple( + encrypted, x_keys[3].first, x_keys[3].second, x_keys[0].second, "test suite"); + m3b = decrypt_for_multiple_simple( + encrypted, x_keys[3].first, x_keys[3].second, x_keys[0].second, "not test suite"); + m4 = decrypt_for_multiple_simple( + encrypted, x_keys[4].first, x_keys[4].second, x_keys[0].second, "test suite"); REQUIRE(m1); REQUIRE(m2); @@ -400,10 +344,10 @@ TEST_CASE("Multi-recipient encryption, simpler interface", "[encrypt][multi][sim CHECK(session::to_string(*m2) == "cruel"); CHECK(session::to_string(*m3) == "world"); - CHECK_THROWS(session::encrypt_for_multiple_simple( - session::to_view_vector(msgs.begin(), std::prev(msgs.end())), - session::to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), - session::to_span(x_keys[0].first), - session::to_span(x_keys[0].second), + CHECK_THROWS(encrypt_for_multiple_simple( + to_view_vector(msgs.begin(), std::prev(msgs.end())), + to_view_vector(std::next(recipients.begin()), std::prev(recipients.end())), + x_keys[0].first, + x_keys[0].second, "test suite")); } diff --git a/tests/test_network_swarm.cpp b/tests/test_network_swarm.cpp index 887ea581e..d4f41e48d 100644 --- a/tests/test_network_swarm.cpp +++ b/tests/test_network_swarm.cpp @@ -10,7 +10,7 @@ using namespace session; using namespace session::network; using namespace session::network::swarm; -swarm_id_t get_swarm_id( +static swarm_id_t get_swarm_id( std::string swarm_pubkey_hex, std::vector>> swarms) { if (swarm_pubkey_hex.size() == 66) diff --git a/tests/test_onion_request_router.cpp b/tests/test_onion_request_router.cpp index 8913315d8..9ea9d3541 100644 --- a/tests/test_onion_request_router.cpp +++ b/tests/test_onion_request_router.cpp @@ -5,8 +5,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -89,7 +89,7 @@ class TestOnionRequestRouter { namespace detail { class TestRequestQueue : public detail::RequestQueue, public CallTracker { public: - TestRequestQueue(std::shared_ptr loop) : detail::RequestQueue(loop) {}; + TestRequestQueue(oxen::quic::Loop& loop) : detail::RequestQueue(loop) {}; void add(Request request, network_response_callback_t callback) override { if (check_should_ignore_and_log_call("add")) @@ -131,14 +131,10 @@ namespace { TestSnodePool( config::SnodePool config, - std::shared_ptr loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, network_fetcher_t direct_fetcher = [](Request, network_response_callback_t) {}) : - SnodePool( - std::move(config), - std::move(loop), - std::move(disk_loop), - std::move(direct_fetcher)) {} + SnodePool(std::move(config), loop, disk_loop, std::move(direct_fetcher)) {} void record_node_failure(const service_node& node, bool permanent = false) override { if (check_should_ignore_and_log_call("record_node_failure(node)")) @@ -188,23 +184,22 @@ namespace { ConnectionStatus get_status() const override { return ConnectionStatus::unknown; }; void verify_connectivity( - service_node /*node*/, + service_node, std::chrono::milliseconds /*timeout*/, const std::string& /*request_id*/, - const RequestCategory /*category*/, + const RequestCategory, std::function error_code)> /*callback*/) override { func_called("verify_connectivity"); } - void add_failure_listener( - const ed25519_pubkey& /*pubkey*/, std::function /*listener*/) override { + void add_failure_listener(const ed25519_pubkey&, std::function) override { func_called("add_failure_listener"); } - void remove_failure_listeners(const ed25519_pubkey& /*pubkey*/) override { + void remove_failure_listeners(const ed25519_pubkey&) override { func_called("remove_failure_listeners"); } - void send_request(Request /*request*/, network_response_callback_t /*callback*/) override { + void send_request(Request, network_response_callback_t) override { func_called("send_request"); } }; @@ -249,10 +244,10 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { true, true, {{PathCategory::standard, 1}}}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; - auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hexbytes; - auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hexbytes; - auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; + auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; + auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hex_b; + auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hex_b; auto target = service_node{ ed25519_pubkey::from_bytes(ed_pk), oxen::quic::ipv4{"127.0.0.1"}, @@ -288,7 +283,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); auto transport = std::make_shared(); std::shared_ptr router; @@ -296,7 +291,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { snode_pool->clear_node_strikes(); snode_pool->reset_calls(); path.emplace(OnionPath{"Test", {target2, target3, target4}}); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {*path}); TestOnionRequestRouter::handle_transport_response( router, @@ -331,7 +326,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { snode_pool->clear_node_strikes(); snode_pool->reset_calls(); path.emplace(OnionPath{"Test", {target2, target3, target4}}); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {*path}); TestOnionRequestRouter::handle_transport_response( router, @@ -366,7 +361,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { REQUIRE(snode_pool->node_strike_count(target2) == 0); snode_pool->reset_calls(); path.emplace(OnionPath{"Test", {target2, target3, target4}}); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths( router, PathCategory::standard, @@ -410,7 +405,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { snode_pool->reset_calls(); snode_pool->mock_unused_nodes = {target}; path.emplace(OnionPath{"Test", {target2, target3, target4}}); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {*path}); TestOnionRequestRouter::handle_transport_response( router, @@ -460,7 +455,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { snode_pool->clear_node_strikes(); snode_pool->reset_calls(); path.emplace(OnionPath{"Test", {target2, target3, target4}}); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {*path}); TestOnionRequestRouter::handle_transport_response( router, @@ -498,7 +493,7 @@ TEST_CASE("Network", "[network][onion_request_router][handle_errors]") { snode_pool->reset_calls(); path.emplace(OnionPath{"Test", {target2, target3, target4}}); router = std::make_shared( - config, loop, disk_loop, snode_pool, transport); + config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {*path}); TestOnionRequestRouter::handle_transport_response( router, @@ -562,20 +557,20 @@ TEST_CASE("Network", "[network][onion_request_router][build_path]") { {{PathCategory::standard, 1}}}; auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); auto transport = std::make_shared(); std::shared_ptr router; // Nothing should happen if the network is suspended snode_pool->reset_calls(); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); router->suspend(); TestOnionRequestRouter::build_path(router, PathCategory::standard); CHECK(snode_pool->did_not_call("get_unused_nodes")); // If the unused nodes are empty it refreshes them snode_pool->reset_calls(); - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::build_path(router, PathCategory::standard); CHECK(snode_pool->called("get_unused_nodes")); CHECK(snode_pool->called("refresh_if_needed")); @@ -611,10 +606,10 @@ TEST_CASE("Network", "[network][onion_request_router][find_valid_path]") { true, false, {{PathCategory::standard, 1}}}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; - auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hexbytes; - auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hexbytes; - auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; + auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; + auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hex_b; + auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hex_b; auto target = service_node{ ed25519_pubkey::from_bytes(ed_pk), oxen::quic::ipv4{"127.0.0.1"}, @@ -650,22 +645,22 @@ TEST_CASE("Network", "[network][onion_request_router][find_valid_path]") { auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); auto transport = std::make_shared(); std::shared_ptr router; // It returns nothing when given no path options - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {}); CHECK(TestOnionRequestRouter::find_valid_path(router, request) == nullptr); // It excludes paths which include the IP of the target - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {path1}); CHECK(TestOnionRequestRouter::find_valid_path(router, request) == nullptr); // It returns a path when there is a valid one - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {path2}); CHECK(TestOnionRequestRouter::find_valid_path(router, request) != nullptr); @@ -686,7 +681,7 @@ TEST_CASE("Network", "[network][onion_request_router][find_valid_path]") { true, true, // single path mode {{PathCategory::standard, 1}}}; - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); TestOnionRequestRouter::set_paths(router, PathCategory::standard, {path1}); CHECK(TestOnionRequestRouter::find_valid_path(router, request) != nullptr); } @@ -721,7 +716,7 @@ TEST_CASE("Network", "[network][onion_request_router][check_request_queue_timeou true, false, {{PathCategory::standard, 1}}}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; auto target = service_node{ ed25519_pubkey::from_bytes(ed_pk), oxen::quic::ipv4{"127.0.0.1"}, @@ -757,9 +752,9 @@ TEST_CASE("Network", "[network][onion_request_router][check_request_queue_timeou auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); auto transport = std::make_shared(); - auto queue = std::make_shared(loop); + auto queue = std::make_shared(*loop); std::shared_ptr router; // Test that it doesn't start checking for timeouts when the request doesn't have an overall @@ -772,8 +767,8 @@ TEST_CASE("Network", "[network][onion_request_router][check_request_queue_timeou RequestCategory::standard, 1000ms, std::nullopt}; - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); - queue = std::make_shared(loop); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); + queue = std::make_shared(*loop); TestOnionRequestRouter::set_request_queues(router, {{PathCategory::standard, queue}}); router->send_request( request, @@ -791,8 +786,8 @@ TEST_CASE("Network", "[network][onion_request_router][check_request_queue_timeou // `check_timeouts` at the timeout rather than poll) request = Request{ "AAAA", target, "info", to_vector("test"), RequestCategory::standard, 1000ms, 100ms}; - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); - queue = std::make_shared(loop); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); + queue = std::make_shared(*loop); TestOnionRequestRouter::set_request_queues(router, {{PathCategory::standard, queue}}); router->send_request( request, @@ -812,8 +807,8 @@ TEST_CASE("Network", "[network][onion_request_router][check_request_queue_timeou std::promise prom; request = Request{ "AAAA", target, "info", to_vector("test"), RequestCategory::standard, 1000ms, 200ms}; - router = std::make_shared(config, loop, disk_loop, snode_pool, transport); - queue = std::make_shared(loop); + router = std::make_shared(config, *loop, *disk_loop, snode_pool, transport); + queue = std::make_shared(*loop); TestOnionRequestRouter::set_request_queues(router, {{PathCategory::standard, queue}}); router->send_request( request, diff --git a/tests/test_onionreq.cpp b/tests/test_onionreq.cpp index 79f3bfd78..6509267a0 100644 --- a/tests/test_onionreq.cpp +++ b/tests/test_onionreq.cpp @@ -10,28 +10,28 @@ using namespace session::network; TEST_CASE("Onion request encryption", "[encryption][onionreq]") { - auto A = "bbdfc83022d0aff084a6f0c529a93d1c4d728bf7e41199afed0e01ae70d20540"_hexbytes; - auto B = "caea52c5b0c316d85ffb53ea536826618b13dee40685f166f632653114526a78"_hexbytes; - auto b = "8fcd8ad3a15c76f76f1c56dff0c529999f8c59b4acda79e05666e54d5727dca1"_hexbytes; + auto A = "bbdfc83022d0aff084a6f0c529a93d1c4d728bf7e41199afed0e01ae70d20540"_hex_b; + auto B = "caea52c5b0c316d85ffb53ea536826618b13dee40685f166f632653114526a78"_hex_b; + auto b = "8fcd8ad3a15c76f76f1c56dff0c529999f8c59b4acda79e05666e54d5727dca1"_hex_b; auto enc_gcm = "1eb6ae1cd72f60999486365749bd5dc15cc0b6a2a44d7d063daa5e93722f0c025fd00306403b61" - ""_hexbytes; + ""_hex_b; auto enc_gcm_broken1 = "1eb6ae1cd72f60999486365759bd5dc15cc0b6a2a44d7d063daa5e93722f0c025fd00306403b61" - ""_hexbytes; + ""_hex_b; auto enc_gcm_broken2 = "1eb6ae1cd72f60999486365749bd5dc15cc0b6a2a44d7d063daa5e93722f0c025fd00306403b69" - ""_hexbytes; + ""_hex_b; auto enc_xchacha20 = "9e1a3abe60eff3ea5c23556cc7e225b6f94355315f7281f66ecf4dbb06e7899a52b863e03cde3b28" - "7d1638d765db75de02b032"_hexbytes; + "7d1638d765db75de02b032"_hex_b; auto enc_xchacha20_broken1 = "9e1a3abe60eff3ea5c23556cc7e225b6f94355315f7281f66ecf4dbb06e7899a52b863e03cde3b28" - "7d1638d765db75de02b033"_hexbytes; + "7d1638d765db75de02b033"_hex_b; auto enc_xchacha20_broken2 = "9e1a3abe60eff3ea5c23556ccfe225b6f94355315f7281f66ecf4dbb06e7899a52b863e03cde3b28" - "7d1638d765db75de02b032"_hexbytes; + "7d1638d765db75de02b032"_hex_b; HopEncryption e{x25519_seckey::from_bytes(b), x25519_pubkey::from_bytes(B), true}; @@ -46,21 +46,21 @@ TEST_CASE("Onion request encryption", "[encryption][onionreq]") { TEST_CASE("Onion request parser", "[onionreq][parser]") { - auto A = "8167e97672005c669a48858c69895f395ca235219ac3f7a4210022b1f910e652"_hexbytes; - auto a = "d2ee09e1a557a077d385fcb69a11ffb6909ecdcc8348def3e0e4172c8a1431c1"_hexbytes; - auto B = "8388de69bc0d4b6196133233ad9a46ba0473474bc67718aad96a3a33c257f726"_hexbytes; - auto b = "2f4d1c0d28e137777ec0a316e9f4f763e3e66662a6c51994c6315c9ef34b6deb"_hexbytes; + auto A = "8167e97672005c669a48858c69895f395ca235219ac3f7a4210022b1f910e652"_hex_b; + auto a = "d2ee09e1a557a077d385fcb69a11ffb6909ecdcc8348def3e0e4172c8a1431c1"_hex_b; + auto B = "8388de69bc0d4b6196133233ad9a46ba0473474bc67718aad96a3a33c257f726"_hex_b; + auto b = "2f4d1c0d28e137777ec0a316e9f4f763e3e66662a6c51994c6315c9ef34b6deb"_hex_b; auto enc_gcm = "270000009525d587d188c92a966eef0e7162bef99a6171a124575b998072a8ee7eb265e0b6f0930ed96504" "7b22656e635f74797065223a20226165732d67636d222c2022657068656d6572616c5f6b6579223a202238" "31363765393736373230303563363639613438383538633639383935663339356361323335323139616333" - "6637613432313030323262316639313065363532227d"_hexbytes; + "6637613432313030323262316639313065363532227d"_hex_b; auto enc_xchacha20 = "33000000e440bc244ddcafd947b86fc5a964aa58de54a6d75cc0f0f3840db14b6c1176a8e2e0a04d5fbdf9" "8f23adee1edc8362ab99b10b7b22656e635f74797065223a2022786368616368613230222c202265706865" "6d6572616c5f6b6579223a2022383136376539373637323030356336363961343838353863363938393566" - "33393563613233353231396163336637613432313030323262316639313065363532227d"_hexbytes; + "33393563613233353231396163336637613432313030323262316639313065363532227d"_hex_b; OnionReqParser parser_gcm{B, b, enc_gcm}; CHECK(to_string(parser_gcm.payload()) == "Hello world"); diff --git a/tests/test_pfs_key_cache.cpp b/tests/test_pfs_key_cache.cpp new file mode 100644 index 000000000..5331d4555 --- /dev/null +++ b/tests/test_pfs_key_cache.cpp @@ -0,0 +1,259 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "test_helper.hpp" +#include "utils.hpp" + +using namespace session; +using namespace std::literals; + +// Wraps a single bt-dict message payload as a mock AccountPubkeys retrieve response. +// prefetch_pfs_keys() sends a plain "retrieve" and expects {messages: [...]} at the top level. +static nlohmann::json make_pubkey_response(std::span msg_data) { + nlohmann::json msg_item; + msg_item["data"] = oxenc::to_base64( + std::string_view{reinterpret_cast(msg_data.data()), msg_data.size()}); + nlohmann::json resp; + resp["messages"] = nlohmann::json::array({std::move(msg_item)}); + return resp; +} + +// Returns an AccountPubkeys response body with no messages. +static nlohmann::json make_empty_response() { + nlohmann::json resp; + resp["messages"] = nlohmann::json::array(); + return resp; +} + +TEST_CASE("prefetch_pfs_keys throws without network", "[core][pfs]") { + TempCore c; + TempCore remote; + auto session_id = remote->globals.session_id(); + b33 sid; + std::ranges::copy(session_id, sid.begin()); + CHECK_THROWS_AS(c->prefetch_pfs_keys(sid), std::logic_error); +} + +TEST_CASE("prefetch_pfs_keys fetches and caches remote account pubkeys", "[core][pfs]") { + TempCore c; + auto* mock_net = attach_mock_network(*c); + + // Build a "remote" account whose pubkeys we want to fetch. + TempCore remote; + auto remote_msg = remote->devices.build_account_pubkey_message(); + + auto session_id_span = remote->globals.session_id(); + b33 sid; + std::ranges::copy(session_id_span, sid.begin()); + + SECTION("Fetches and stores pubkeys when cache is absent") { + c->prefetch_pfs_keys(sid); + + REQUIRE(mock_net->sent_requests.size() == 1); + CHECK(mock_net->sent_requests[0].request.endpoint == "retrieve"); + auto req = parse_json(*mock_net->sent_requests[0].request.body); + CHECK(req["pubkey"] == oxenc::to_hex(sid)); + CHECK(req["namespace"] == static_cast(config::Namespace::AccountPubkeys)); + + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(remote_msg).dump()); + + auto entry = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(entry.has_value()); + REQUIRE(entry->fetched_at.has_value()); + // fetched_at should be close to now. + auto age = clock_now_s() - from_epoch_s(*entry->fetched_at); + CHECK(age >= 0s); + CHECK(age < 5s); + CHECK_FALSE(entry->nak_at.has_value()); + + // The stored pubkeys must match those from the remote's active account key. + auto [expected_x25519, expected_mlkem768] = TestHelper::active_account_pubkeys(*remote); + REQUIRE(entry->pubkey_x25519.has_value()); + REQUIRE(entry->pubkey_mlkem768.has_value()); + CHECK(*entry->pubkey_x25519 == expected_x25519); + CHECK(*entry->pubkey_mlkem768 == expected_mlkem768); + } + + SECTION("Skips fetch when cache is fresh") { + // First fetch: populates the cache. + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(remote_msg).dump()); + REQUIRE(TestHelper::pfs_cache_entry(*c, sid).has_value()); + mock_net->sent_requests.clear(); + + // Second fetch within PFS_KEY_FRESH_DURATION: must not send another request. + c->prefetch_pfs_keys(sid); + CHECK(mock_net->sent_requests.empty()); + } + + SECTION("Re-fetches when cache is stale (older than PFS_KEY_FRESH_DURATION)") { + // First fetch. + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(remote_msg).dump()); + mock_net->sent_requests.clear(); + + // Advance clock past the fresh threshold. + ScopedClockOffset advance_past_fresh{core::Core::PFS_KEY_FRESH_DURATION + 1s}; + c->prefetch_pfs_keys(sid); + CHECK(mock_net->sent_requests.size() == 1); + } +} + +TEST_CASE("prefetch_pfs_keys NAK handling", "[core][pfs]") { + TempCore c; + auto* mock_net = attach_mock_network(*c); + + TempCore remote; + auto session_id_span = remote->globals.session_id(); + b33 sid; + std::ranges::copy(session_id_span, sid.begin()); + + // Helper: fire the pending request with an empty-messages response (NAK condition). + auto fire_nak = [&] { + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback(true, false, 200, {}, make_empty_response().dump()); + mock_net->sent_requests.clear(); + }; + + SECTION("Records NAK when fetch succeeds but returns no keys") { + c->prefetch_pfs_keys(sid); + fire_nak(); + + auto entry = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(entry.has_value()); + CHECK_FALSE(entry->fetched_at.has_value()); + REQUIRE(entry->nak_at.has_value()); + auto nak_age = clock_now_s() - from_epoch_s(*entry->nak_at); + CHECK(nak_age >= 0s); + CHECK(nak_age < 5s); + CHECK_FALSE(entry->pubkey_x25519.has_value()); + CHECK_FALSE(entry->pubkey_mlkem768.has_value()); + } + + SECTION("NAK suppresses re-fetch within PFS_KEY_NAK_DURATION") { + c->prefetch_pfs_keys(sid); + fire_nak(); + + // Should not issue another request while the NAK is fresh. + c->prefetch_pfs_keys(sid); + CHECK(mock_net->sent_requests.empty()); + } + + SECTION("NAK allows re-fetch after PFS_KEY_NAK_DURATION expires") { + c->prefetch_pfs_keys(sid); + fire_nak(); + + ScopedClockOffset advance_past_nak_expiry{core::Core::PFS_KEY_NAK_DURATION + 1s}; + c->prefetch_pfs_keys(sid); + CHECK(mock_net->sent_requests.size() == 1); + } + + SECTION("NAK does not overwrite an existing valid entry") { + // Populate the cache with a valid entry. + auto remote_msg = remote->devices.build_account_pubkey_message(); + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(remote_msg).dump()); + mock_net->sent_requests.clear(); + + auto before = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(before.has_value()); + REQUIRE(before->pubkey_x25519.has_value()); + + // Advance clock to make the entry stale, then fire a re-fetch that returns nothing. + ScopedClockOffset advance_past_fresh{core::Core::PFS_KEY_FRESH_DURATION + 1s}; + c->prefetch_pfs_keys(sid); + fire_nak(); + + // Valid pubkeys must still be present; nak_at is also set. + auto after = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(after.has_value()); + CHECK(after->pubkey_x25519 == before->pubkey_x25519); + CHECK(after->pubkey_mlkem768 == before->pubkey_mlkem768); + CHECK(after->fetched_at == before->fetched_at); + REQUIRE(after->nak_at.has_value()); + } + + SECTION("Stale valid entry is not gated by a concurrent NAK") { + // Populate the cache with a valid entry. + auto remote_msg = remote->devices.build_account_pubkey_message(); + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(remote_msg).dump()); + mock_net->sent_requests.clear(); + + // Make stale and fire a NAK. + { + ScopedClockOffset advance_past_fresh{core::Core::PFS_KEY_FRESH_DURATION + 1s}; + c->prefetch_pfs_keys(sid); + fire_nak(); + + // With a fresh NAK and stale valid entry, a new call should still re-fetch because + // the NAK only gates the no-valid-keys path. + c->prefetch_pfs_keys(sid); + CHECK(mock_net->sent_requests.size() == 1); + } + } +} + +TEST_CASE("prefetch_pfs_keys handles malformed responses gracefully", "[core][pfs]") { + TempCore c; + auto* mock_net = attach_mock_network(*c); + + TempCore remote; + auto session_id_span = remote->globals.session_id(); + b33 sid; + std::ranges::copy(session_id_span, sid.begin()); + + SECTION("Garbage bt-dict data: NAK written, no valid pubkeys stored") { + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + + nlohmann::json msg_item; + msg_item["data"] = oxenc::to_base64("not a bt-dict"); + nlohmann::json bad_resp; + bad_resp["messages"] = nlohmann::json::array({std::move(msg_item)}); + + mock_net->sent_requests[0].callback(true, false, 200, {}, bad_resp.dump()); + auto entry = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(entry.has_value()); + CHECK(entry->nak_at.has_value()); + CHECK_FALSE(entry->pubkey_x25519.has_value()); + } + + SECTION("Bad signature: NAK written, no valid pubkeys stored") { + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + + // A message signed with the wrong key (our own account instead of the remote's). + auto wrong_msg = c->devices.build_account_pubkey_message(); + mock_net->sent_requests[0].callback( + true, false, 200, {}, make_pubkey_response(wrong_msg).dump()); + auto entry = TestHelper::pfs_cache_entry(*c, sid); + REQUIRE(entry.has_value()); + CHECK(entry->nak_at.has_value()); + CHECK_FALSE(entry->pubkey_x25519.has_value()); + } + + SECTION("Network failure: nothing written") { + c->prefetch_pfs_keys(sid); + REQUIRE(mock_net->sent_requests.size() == 1); + + mock_net->sent_requests[0].callback(false, false, 0, {}, std::nullopt); + CHECK_FALSE(TestHelper::pfs_cache_entry(*c, sid).has_value()); + } +} diff --git a/tests/test_poll.cpp b/tests/test_poll.cpp new file mode 100644 index 000000000..14e125c60 --- /dev/null +++ b/tests/test_poll.cpp @@ -0,0 +1,336 @@ +#include +#include + +#include +#include +#include +#include + +#include "test_helper.hpp" + +using namespace session; + +// The batch _poll() sends is one subrequest per polled namespace, and results are matched to it by +// position. Both helpers below therefore work from the *request*: which namespaces it asked for, +// and in what order. That is what the real storage server does, and it means these tests do not +// have to be rewritten each time Core learns to poll another namespace -- a positional assertion +// breaks on every such change and says nothing about why. + +// The parameters of the subrequest asking for `ns`. +// Note each of these binds the parsed json to a local before iterating it: ranging directly over +// `parse_json(body)["requests"]` takes a reference *into* a temporary that dies before the loop +// body runs, which reads as an empty batch rather than as an error. +static nlohmann::json params_for(std::span request_body, int16_t ns) { + auto batch = parse_json(request_body); + for (const auto& r : batch["requests"]) + if (r["params"]["namespace"] == ns) + return r["params"]; + throw std::runtime_error{"batch contains no subrequest for namespace {}"_format(ns)}; +} + +// The namespaces a batch asked for, sorted, for asserting the whole set at once. +static std::vector namespaces_in(std::span request_body) { + auto batch = parse_json(request_body); + std::vector out; + for (const auto& r : batch["requests"]) + out.push_back(r["params"]["namespace"].get()); + std::ranges::sort(out); + return out; +} + +// A batch response shaped to the request: one result per subrequest, carrying a single message in +// the one that asked for `ns` and nothing in the rest. +static nlohmann::json make_response( + std::span request_body, + int16_t ns, + std::vector msg_data, + std::string hash) { + auto batch = parse_json(request_body); + auto results = nlohmann::json::array(); + for (const auto& r : batch["requests"]) { + nlohmann::json body; + body["messages"] = nlohmann::json::array(); + if (r["params"]["namespace"] == ns) { + nlohmann::json item; + item["data"] = oxenc::to_base64(msg_data); + item["hash"] = hash; + body["messages"].push_back(std::move(item)); + } + results.push_back({{"code", 200}, {"body", std::move(body)}}); + } + return nlohmann::json{{"results", std::move(results)}}; +} + +TEST_CASE("Core automatic polling", "[core][poll]") { + bool received = false; + core::callbacks cbs; + cbs.device_link_request = [&](int, + const core::device::Info&, + std::span) { received = true; }; + + TempCore core{cbs}; + auto* mock_net = attach_mock_network(*core); + // Use a fixed non-zero pubkey for the node. + mock_net->current_node.remote_pubkey[0] = std::byte{0x01}; + + // Trigger poll via TestHelper + TestHelper::poll(*core); + + REQUIRE(mock_net->sent_requests.size() == 1); + auto& sent = mock_net->sent_requests[0]; + + CHECK(sent.request.endpoint == "batch"); + auto& body = *sent.request.body; + + // Every namespace the account polls: the four user configs, one-to-one messages, and the two + // the device group needs. + CHECK(namespaces_in(body) == std::vector{-21, 0, 2, 3, 4, 5, 21}); + + for (const auto& r : parse_json(body)["requests"]) + CHECK(r["method"] == "retrieve"); + + // Devices (ns 21) requires auth, and is the namespace the rest of this test uses. + auto params = params_for(body, 21); + CHECK(params["pubkey"] == oxenc::to_hex(core->globals.session_id())); + CHECK(params.contains("pubkey_ed25519")); + CHECK(params.contains("timestamp")); + CHECK(params.contains("signature")); + // No prior hash for this node yet, so no last_hash in any subrequest. + CHECK_FALSE(params.contains("last_hash")); + + // The config namespaces are owner-writable, so retrieving from them is signed too. + CHECK(params_for(body, 2).contains("signature")); + CHECK(params_for(body, 3).contains("signature")); + + // Default (ns 0) is signed; AccountPubkeys (ns -21) is public and is not. + CHECK(params_for(body, 0).contains("signature")); + CHECK_FALSE(params_for(body, -21).contains("signature")); + + // Build a valid link request from a second device sharing the same account seed. + cleared_b32 seed_bytes; + { + auto seed_acc = core->globals.account_seed(); + std::ranges::copy(std::as_bytes(seed_acc.seed()), seed_bytes.begin()); + } + TempCore linker{core::predefined_seed{std::span{seed_bytes}}}; + auto outer_msg = linker->devices.build_link_request().message; + + sent.callback( + true, false, 200, {}, make_response(*sent.request.body, 21, outer_msg, "hash1").dump()); + + // Verify last_hash was stored under this specific node's pubkey. + CHECK(TestHelper::namespace_last_hash(*core, 21, mock_net->current_node.remote_pubkey) == + "hash1"); + CHECK(received); + + // Poll again with the same node — should include last_hash in the Devices subrequest. + mock_net->sent_requests.clear(); + TestHelper::poll(*core); + + REQUIRE(mock_net->sent_requests.size() == 1); + CHECK(params_for(*mock_net->sent_requests[0].request.body, 21)["last_hash"] == "hash1"); +} + +TEST_CASE( + "Polling uses per-node last_hash to avoid missing messages on swarm-member switch", + "[core][poll]") { + TempCore c; + auto* mock_net = attach_mock_network(*c); + + // Two distinct service nodes with different pubkeys. + network::service_node node_a, node_b; + node_a.remote_pubkey[0] = std::byte{0xAA}; + node_b.remote_pubkey[0] = std::byte{0xBB}; + + // ── First poll: node A, no prior state ────────────────────────────────────── + mock_net->current_node = node_a; + TestHelper::poll(*c); + REQUIRE(mock_net->sent_requests.size() == 1); + { + auto p = params_for(*mock_net->sent_requests[0].request.body, 21); + // No prior hash for any node — must not send last_hash. + CHECK_FALSE(p.contains("last_hash")); + } + // Respond with hash "xyz" from node A. + mock_net->sent_requests[0].callback( + true, + false, + 200, + {}, + make_response(*mock_net->sent_requests[0].request.body, 21, {std::byte{0x01}}, "xyz") + .dump()); + CHECK(TestHelper::namespace_last_hash(*c, 21, node_a.remote_pubkey) == "xyz"); + CHECK_FALSE(TestHelper::namespace_last_hash(*c, 21, node_b.remote_pubkey).has_value()); + + // ── Second poll: still node A — must use A's stored hash ──────────────────── + mock_net->sent_requests.clear(); + TestHelper::poll(*c); + REQUIRE(mock_net->sent_requests.size() == 1); + { + auto p = params_for(*mock_net->sent_requests[0].request.body, 21); + CHECK(p["last_hash"] == "xyz"); + } + + // ── Third poll: switch to node B — no stored hash for B, so request everything ── + mock_net->sent_requests.clear(); + mock_net->current_node = node_b; + TestHelper::poll(*c); + REQUIRE(mock_net->sent_requests.size() == 1); + { + auto p = params_for(*mock_net->sent_requests[0].request.body, 21); + // B has no recorded hash — must not send last_hash so we get everything. + CHECK_FALSE(p.contains("last_hash")); + } + // Respond with hash "zyx" from node B (the message that B happens to have seen first). + mock_net->sent_requests[0].callback( + true, + false, + 200, + {}, + make_response(*mock_net->sent_requests[0].request.body, 21, {std::byte{0x02}}, "zyx") + .dump()); + CHECK(TestHelper::namespace_last_hash(*c, 21, node_b.remote_pubkey) == "zyx"); + // A's hash is untouched. + CHECK(TestHelper::namespace_last_hash(*c, 21, node_a.remote_pubkey) == "xyz"); + + // ── Fourth poll: back to node A — must still use A's hash, not B's ────────── + mock_net->sent_requests.clear(); + mock_net->current_node = node_a; + TestHelper::poll(*c); + REQUIRE(mock_net->sent_requests.size() == 1); + { + auto p = params_for(*mock_net->sent_requests[0].request.body, 21); + CHECK(p["last_hash"] == "xyz"); + } +} + +TEST_CASE("Poll: the sync cursor advances only after the batch is handled", "[core][poll]") { + // Attached below, once there is a Core to own it; the callback that reads it does not run + // until then either. + MockNetwork* mock_net = nullptr; + + core::Core* core_ptr = nullptr; + std::optional hash_during_callback; + bool called = false; + + // Observe the stored cursor from inside the handler. If it has already advanced by the time + // the batch is being handled, then a handler that fails -- or a crash at that moment -- loses + // the batch permanently, because the swarm filters on last_hash. + core::callbacks cbs; + cbs.device_link_request = + [&](int, const core::device::Info&, std::span) { + called = true; + hash_during_callback = TestHelper::namespace_last_hash( + *core_ptr, 21, mock_net->current_node.remote_pubkey); + }; + + TempCore core{cbs}; + core_ptr = &*core; + mock_net = attach_mock_network(*core); + mock_net->current_node.remote_pubkey[0] = std::byte{0x01}; + + cleared_b32 seed_bytes; + { + auto seed_acc = core->globals.account_seed(); + std::ranges::copy(std::as_bytes(seed_acc.seed()), seed_bytes.begin()); + } + TempCore linker{core::predefined_seed{std::span{seed_bytes}}}; + auto outer_msg = linker->devices.build_link_request().message; + + TestHelper::poll(*core); + REQUIRE(mock_net->sent_requests.size() == 1); + mock_net->sent_requests[0].callback( + true, + false, + 200, + {}, + make_response(*mock_net->sent_requests[0].request.body, 21, outer_msg, "hash1").dump()); + + REQUIRE(called); + CHECK(!hash_during_callback); + CHECK(TestHelper::namespace_last_hash(*core, 21, mock_net->current_node.remote_pubkey) == + "hash1"); +} + +// A batch response where every namespace answered successfully and returned nothing. +static nlohmann::json make_empty_response(std::span request_body) { + auto batch = parse_json(request_body); + auto results = nlohmann::json::array(); + for (size_t i = 0; i < batch["requests"].size(); i++) + results.push_back({{"code", 200}, {"body", {{"messages", nlohmann::json::array()}}}}); + return nlohmann::json{{"results", std::move(results)}}; +} + +// Marks the result for `ns` as having more behind it, found by the position its subrequest occupies +// -- which is how Core matches results to requests too. +static void set_more( + nlohmann::json& response, std::span request_body, int16_t ns) { + auto batch = parse_json(request_body); + for (size_t i = 0; i < batch["requests"].size(); i++) + if (batch["requests"][i]["params"]["namespace"] == ns) + response["results"][i]["body"]["more"] = true; +} + +TEST_CASE("Poll: a truncated namespace is continued before it is reported final", "[core][poll]") { + int calls = 0; + core::callbacks cbs; + cbs.device_link_request = + [&](int, const core::device::Info&, std::span) { calls++; }; + + TempCore core{cbs}; + auto* mock_net = attach_mock_network(*core); + mock_net->current_node.remote_pubkey[0] = std::byte{0x01}; + + cleared_b32 seed_bytes; + { + auto seed_acc = core->globals.account_seed(); + std::ranges::copy(std::as_bytes(seed_acc.seed()), seed_bytes.begin()); + } + TempCore linker{core::predefined_seed{std::span{seed_bytes}}}; + auto outer_msg = linker->devices.build_link_request().message; + + TestHelper::poll(*core); + REQUIRE(mock_net->sent_requests.size() == 1); + + auto first = *mock_net->sent_requests[0].request.body; + auto resp = make_response(first, 21, outer_msg, "hash1"); + set_more(resp, first, 21); + + // Copied out before invoking: the continuation is sent from inside this call, which appends to + // `sent_requests` and can reallocate the vector the callback itself lives in. + auto reply = mock_net->sent_requests[0].callback; + reply(true, false, 200, {}, resp.dump()); + + // The request is stored, but the batch was not final, so nothing has been reported yet. + CHECK(calls == 0); + + REQUIRE(mock_net->sent_requests.size() == 2); + auto second = *mock_net->sent_requests[1].request.body; + CHECK(namespaces_in(second) == std::vector{21}); + CHECK(params_for(second, 21)["last_hash"] == "hash1"); + + // Nothing left behind it: an empty answer is still an answer, and is what makes the batch + // final. + auto reply2 = mock_net->sent_requests[1].callback; + reply2(true, false, 200, {}, make_empty_response(second).dump()); + + CHECK(calls == 1); +} + +TEST_CASE("Poll: `more` with nothing returned does not continue", "[core][poll]") { + TempCore core; + auto* mock_net = attach_mock_network(*core); + + TestHelper::poll(*core); + REQUIRE(mock_net->sent_requests.size() == 1); + + auto first = *mock_net->sent_requests[0].request.body; + auto resp = make_empty_response(first); + set_more(resp, first, 21); + + auto reply = mock_net->sent_requests[0].callback; + reply(true, false, 200, {}, resp.dump()); + + // There is no new hash to move the cursor to, so another round would ask the same question. + CHECK(mock_net->sent_requests.size() == 1); +} diff --git a/tests/test_pro_backend.cpp b/tests/test_pro_backend.cpp index 3ae3861da..6eeebacfc 100644 --- a/tests/test_pro_backend.cpp +++ b/tests/test_pro_backend.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -19,12 +20,12 @@ static bool span_u8_equals(span_u8 s, std::string_view str) { } TEST_CASE("Pro Backend C API", "[pro_backend]") { // Setup: Generate keys and payment token hash - bytes32 master_pubkey = {}; - bytes64 master_privkey = {}; + cbytes32 master_pubkey = {}; + cbytes64 master_privkey = {}; crypto_sign_ed25519_keypair(master_pubkey.data, master_privkey.data); - bytes32 rotating_pubkey = {}; - bytes64 rotating_privkey = {}; + cbytes32 rotating_pubkey = {}; + cbytes64 rotating_privkey = {}; crypto_sign_ed25519_keypair(rotating_pubkey.data, rotating_privkey.data); { @@ -137,7 +138,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { } SECTION("session_pro_backend_pro_proof_response_parse") { - std::array fake_revocation_tag; + b32 fake_revocation_tag; randombytes_buf(fake_revocation_tag.data(), fake_revocation_tag.size()); nlohmann::json j; @@ -249,6 +250,40 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { j_bad["result"]["account_auto_renewing"] = 1; // int, not bool REQUIRE_THROWS_AS(parse_pro_proof(j_bad.dump()), session::parse_error_type); + // Binary fields are accepted hex- or base64-encoded, padded or not, decoding to + // the same bytes. 32 bytes is 64 hex chars, 44 padded base64 or 43 unpadded, so + // the three are distinguishable by length alone. + nlohmann::json j_b64 = j; + j_b64["result"]["revocation_tag"] = oxenc::to_base64(fake_revocation_tag); + auto b64_padded = oxenc::to_base64(rotating_pubkey.data); + REQUIRE(b64_padded.size() == 44); + j_b64["result"]["rotating_pkey"] = b64_padded; + auto b64_unpadded = b64_padded.substr(0, 43); + REQUIRE(b64_unpadded.back() != '='); + j_b64["result"]["sig"] = oxenc::to_base64(master_privkey.data); + auto enc_cpp = parse_pro_proof(j_b64.dump()); + REQUIRE(enc_cpp); + CHECK(enc_cpp.proof.revocation_tag == fake_revocation_tag); + CHECK(std::memcmp( + enc_cpp.proof.rotating_pubkey.data(), + rotating_pubkey.data, + sizeof(rotating_pubkey.data)) == 0); + + // An unpadded value of the same field decodes identically. + nlohmann::json j_b64u = j_b64; + j_b64u["result"]["rotating_pkey"] = b64_unpadded; + auto unpadded_cpp = parse_pro_proof(j_b64u.dump()); + REQUIRE(unpadded_cpp); + CHECK(std::memcmp( + unpadded_cpp.proof.rotating_pubkey.data(), + rotating_pubkey.data, + sizeof(rotating_pubkey.data)) == 0); + + // A length matching neither encoding is rejected, naming the field. + nlohmann::json j_short = j; + j_short["result"]["revocation_tag"] = oxenc::to_hex(fake_revocation_tag).substr(2); + REQUIRE_THROWS_AS(parse_pro_proof(j_short.dump()), session::parse_error_key); + // The non-auto-renewing account: a genuine zero grace, and `E + 0 == E`. nlohmann::json j_false = j; j_false["result"]["account_grace_period_duration"] = 0; @@ -339,7 +374,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { j["result"]["retain_for"] = 2592000; j["result"]["items"] = nlohmann::json::array(); - std::array fake_revocation_tag; + b32 fake_revocation_tag; randombytes_buf(fake_revocation_tag.data(), fake_revocation_tag.size()); auto obj = nlohmann::json::object(); @@ -604,7 +639,8 @@ TEST_CASE("Pro Backend X25519 pubkey matches the converted Ed25519 pubkey", "[pr // PUBKEY_X25519 is a hardcoded convenience constant; assert it equals the runtime conversion of // the Ed25519 PUBKEY so the two can never silently drift. unsigned char converted[32] = {}; - REQUIRE(crypto_sign_ed25519_pk_to_curve25519(converted, PUBKEY.data()) == 0); + REQUIRE(crypto_sign_ed25519_pk_to_curve25519( + converted, reinterpret_cast(PUBKEY.data())) == 0); REQUIRE(std::memcmp(converted, PUBKEY_X25519.data(), sizeof(converted)) == 0); // The C export points at the same bytes. REQUIRE(std::memcmp( @@ -772,7 +808,7 @@ TEST_CASE("Pro backend known-answer vectors", "[pro_backend][pro_kat]") { CHECK(sig_covers(req.data, "master_sig", details_cursor_msg_hex, master_pk)); } SECTION("pro proof") { - session::ProProof proof; + ProProof proof; std::memset(proof.revocation_tag.data(), 0x11, proof.revocation_tag.size()); std::memcpy(proof.rotating_pubkey.data(), rotating_pk.data(), 32); proof.expiry_at = expiry; @@ -784,9 +820,11 @@ TEST_CASE("Pro backend known-answer vectors", "[pro_backend][pro_kat]") { // Direct pin of the proof (spec section 2) message construction. CHECK(oxenc::to_hex(proof.signed_message()) == proof_msg_hex); // The frozen backend signature (key 0x03 over that message) verifies; a wrong key does not. - CHECK(proof.verify_signature(backend_pk)); - auto wrong = backend_pk; - wrong[0] ^= 0x01; + std::array bpk{}; + std::memcpy(bpk.data(), backend_pk.data(), 32); + CHECK(proof.verify_signature(bpk)); + auto wrong = bpk; + wrong[0] = static_cast(std::to_integer(wrong[0]) ^ 0x01); CHECK_FALSE(proof.verify_signature(wrong)); } } @@ -912,7 +950,7 @@ static PostFn make_direct_http_transport(CURL* curl, std::string base_url) { // Innermost v4 onion piece to `/oxen/v4/lsrpc` (onion-request mode). `backend_ed25519_pubkey` // is the backend's signing pubkey (fetched from /status); its x25519 form is the destination key. static PostFn make_onion_v4_transport( - CURL* curl, std::string base_url, std::span backend_ed25519_pubkey) { + CURL* curl, std::string base_url, std::span backend_ed25519_pubkey) { using namespace session::network; using namespace session::onionreq; @@ -940,10 +978,10 @@ static PostFn make_onion_v4_transport( /*nodes=*/{}, EncryptType::xchacha20}; - std::vector body_bytes{ - reinterpret_cast(body.data()), - reinterpret_cast(body.data()) + body.size()}; - std::vector blob = builder.generate_onion_blob(std::move(body_bytes)); + std::vector body_bytes{ + reinterpret_cast(body.data()), + reinterpret_cast(body.data()) + body.size()}; + std::vector blob = builder.generate_onion_blob(std::move(body_bytes)); // The lsrpc body is the raw encrypted onion blob; post it as opaque bytes. (An unheadered // curl POST defaults to application/x-www-form-urlencoded, which makes Werkzeug consume the @@ -1000,7 +1038,7 @@ TEST_CASE("Pro backend live /status round-trip", "[pro_backend][pro_live]") { // this exercises the full v4 onion encrypt/decrypt round-trip against the real backend. auto pubkey_bytes = oxenc::from_hex(direct_pubkey); REQUIRE(pubkey_bytes.size() == 32); - std::array ed_pubkey{}; + std::array ed_pubkey{}; std::memcpy(ed_pubkey.data(), pubkey_bytes.data(), ed_pubkey.size()); PostFn onion = make_onion_v4_transport(curl, base_url, ed_pubkey); @@ -1026,13 +1064,13 @@ static void run_seed_helper(const std::vector& args) { REQUIRE(std::system(cmd.c_str()) == 0); } -static std::array fetch_backend_pubkey(const PostFn& transport) { +static std::array fetch_backend_pubkey(const PostFn& transport) { auto j = nlohmann::json::parse(transport("status", "application/json", "")); REQUIRE(j.at("status").get() == "ok"); auto hex = j.at("result").at("signing_pubkey").get(); auto bytes = oxenc::from_hex(hex); REQUIRE(bytes.size() == 32); - std::array out{}; + std::array out{}; std::memcpy(out.data(), bytes.data(), out.size()); return out; } @@ -1054,7 +1092,7 @@ TEST_CASE("Pro backend live full flow", "[pro_backend][pro_live]") { const std::string& base_url = g_test_pro_backend_dev_server_url; PostFn direct = make_direct_http_transport(curl, base_url); - std::array backend_pubkey = fetch_backend_pubkey(direct); + std::array backend_pubkey = fetch_backend_pubkey(direct); PostFn onion = make_onion_v4_transport(curl, base_url, backend_pubkey); // Provider under test; the whole flow below re-runs once per SECTION. @@ -1093,7 +1131,7 @@ TEST_CASE("Pro backend live full flow", "[pro_backend][pro_live]") { "--plan", "1m"}); - auto now = session::sysclock_now_s(); + auto now = session::clock_now_s(); // 1) generate_pro_proof: redemption is implicit, so this master-signed request binds the seeded // payment before answering and returns a signed proof for the paired rotating key. @@ -1184,7 +1222,7 @@ TEST_CASE("Pro backend live get_pro_revocations", "[pro_backend][pro_live]") { const std::string& base_url = g_test_pro_backend_dev_server_url; PostFn direct = make_direct_http_transport(curl, base_url); - std::array backend_pubkey = fetch_backend_pubkey(direct); + std::array backend_pubkey = fetch_backend_pubkey(direct); PostFn onion = make_onion_v4_transport(curl, base_url, backend_pubkey); std::array master_pk{}, rotating_pk{}; @@ -1207,7 +1245,7 @@ TEST_CASE("Pro backend live get_pro_revocations", "[pro_backend][pro_live]") { "1m"}); // Bind + redeem it implicitly (any master-signed request binds unbound payments) and get a // proof to revoke. - ProRequest gen_req = pro_proof_request(master_sk, rotating_sk, session::sysclock_now_s()); + ProRequest gen_req = pro_proof_request(master_sk, rotating_sk, session::clock_now_s()); GenerateProProofResponse gen = parse_pro_proof(send(onion, gen_req)); INFO("generate_pro_proof " << gen.error.value_or("")); REQUIRE(gen.status == ResponseStatus::Ok); diff --git a/tests/test_proto.cpp b/tests/test_proto.cpp index f79706fb9..bb673c302 100644 --- a/tests/test_proto.cpp +++ b/tests/test_proto.cpp @@ -16,17 +16,11 @@ const std::vector groups{ Namespace::ConvoInfoVolatile, Namespace::UserGroups}; -const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; -std::array ed_pk_raw; -std::array ed_sk_raw; -std::span load_seed() { - crypto_sign_ed25519_seed_keypair(ed_pk_raw.data(), ed_sk_raw.data(), seed.data()); - return {ed_sk_raw.data(), ed_sk_raw.size()}; -} -auto ed_sk = load_seed(); +const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; +auto [ed_pk, ed_sk] = ed25519::keypair(seed); TEST_CASE("Protobuf Handling - Wrap, Unwrap", "[config][proto][wrap]") { - auto msg = "Hello from the other side"_bytes; + auto msg = to_vector("Hello from the other side"_bytes); SECTION("Wrap/unwrap message types") { for (auto& n : groups) { @@ -60,7 +54,7 @@ TEST_CASE("Protobuf Handling - Wrap, Unwrap", "[config][proto][wrap]") { } TEST_CASE("Protobuf Handling - Error Handling", "[config][proto][error]") { - auto msg = "Hello from the other side"_bytes; + auto msg = to_vector("Hello from the other side"_bytes); auto addendum = "jfeejj0ifdoesam"_bytes; const auto user_profile_msg = protos::wrap_config(ed_sk, msg, 1, Namespace::UserProfile); @@ -79,11 +73,8 @@ TEST_CASE("Protobuf Handling - Error Handling", "[config][proto][error]") { TEST_CASE("Protobuf old config loading test", "[config][proto][old]") { - const auto seed = "f887566576de6c16d9ec251d55e24c1400000000000000000000000000000000"_hexbytes; - std::array ed_pk_raw; - std::array ed_sk_raw; - crypto_sign_ed25519_seed_keypair(ed_pk_raw.data(), ed_sk_raw.data(), seed.data()); - std::span ed_sk{ed_sk_raw.data(), ed_sk_raw.size()}; + const auto seed = "f887566576de6c16d9ec251d55e24c1400000000000000000000000000000000"_hex_b; + auto [local_ed_pk, local_ed_sk] = ed25519::keypair(seed); auto old_conf = "080112c2060a03505554120f2f6170692f76312f6d6573736167651a9f060806120028e1c5a0beaf313801" @@ -105,7 +96,7 @@ TEST_CASE("Protobuf old config loading test", "[config][proto][old]") { "51bbd320ba901ff6110dad0c70442286cf6220a53c6f9693636a42d5523eeb1e5fb3453169581384fb8a8f" "3914fb6c01900a4f872f55742b117ddd7bd40c4c5911bb214e28eb9450dbdd0d831a93054c63f9a04bf50c" "db9aac0032c484062d7ba7bbe64e07bcd633eec8378d5d914732693c5e298f015ebde2ae45769ed319e267" - "f0528f5cc6da268343b6647b20bae6e9ee8d92cca702"_hexbytes; + "f0528f5cc6da268343b6647b20bae6e9ee8d92cca702"_hex_b; - CHECK_NOTHROW(protos::unwrap_config(ed_sk, old_conf, Namespace::UserProfile)); + CHECK_NOTHROW(protos::unwrap_config(local_ed_sk, old_conf, Namespace::UserProfile)); } diff --git a/tests/test_schema_registry.hpp b/tests/test_schema_registry.hpp new file mode 100644 index 000000000..458c7d09e --- /dev/null +++ b/tests/test_schema_registry.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include +#include + +/// Registry generated from tests/schema/, used to exercise Core's schema_extension option with a +/// second real consumer of session_schema_dir() rather than a hand-written migration array. +namespace session::test::schema { + +extern const std::span MIGRATIONS; + +/// See session::core::schema::FULL_SCHEMA. +extern const std::string_view FULL_SCHEMA; + +} // namespace session::test::schema diff --git a/tests/test_session_encrypt.cpp b/tests/test_session_encrypt.cpp index 1ff6236e0..0bc913b23 100644 --- a/tests/test_session_encrypt.cpp +++ b/tests/test_session_encrypt.cpp @@ -1,8 +1,10 @@ #include -#include #include #include +#include +#include +#include #include #include @@ -12,51 +14,47 @@ TEST_CASE("Session protocol encryption", "[session-protocol][encrypt]") { using namespace session; - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair(ed_pk.data(), ed_sk.data(), seed.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data())); - REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); + REQUIRE(oxenc::to_hex(ed_pk) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); - REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == + REQUIRE(oxenc::to_hex(curve_pk) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - auto sid = "05" + oxenc::to_hex(curve_pk.begin(), curve_pk.end()); - std::vector sid_raw; + auto sid = "05" + oxenc::to_hex(curve_pk); + std::vector sid_raw; oxenc::from_hex(sid.begin(), sid.end(), std::back_inserter(sid_raw)); REQUIRE(sid == "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - REQUIRE(sid_raw == - "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"_hexbytes); - - const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hexbytes; - std::array ed_pk2, curve_pk2; - std::array ed_sk2; - crypto_sign_ed25519_seed_keypair(ed_pk2.data(), ed_sk2.data(), seed2.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk2.data(), ed_pk2.data())); - REQUIRE(oxenc::to_hex(ed_pk2.begin(), ed_pk2.end()) == + REQUIRE(oxenc::to_hex(sid_raw) == + "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); + + const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hex_b; + auto [ed_pk2, ed_sk2] = ed25519::keypair(seed2); + auto curve_pk2 = ed25519::pk_to_x25519(ed_pk2); + REQUIRE(oxenc::to_hex(ed_pk2) == "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"); - REQUIRE(oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()) == + REQUIRE(oxenc::to_hex(curve_pk2) == "aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - auto sid2 = "05" + oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()); + auto sid2 = "05" + oxenc::to_hex(curve_pk2); REQUIRE(sid2 == "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - std::vector sid_raw2; + std::vector sid_raw2; oxenc::from_hex(sid2.begin(), sid2.end(), std::back_inserter(sid_raw2)); - REQUIRE(sid_raw2 == - "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"_hexbytes); + REQUIRE(oxenc::to_hex(sid_raw2) == + "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); SECTION("full secret, prefixed sid") { - auto enc = encrypt_for_recipient(to_span(ed_sk), sid_raw2, to_span("hello")); + auto enc = encrypt_for_recipient(ed_sk, sid_raw2, to_span("hello")); CHECK(to_string(enc) != "hello"); - CHECK_THROWS(decrypt_incoming(to_span(ed_sk), enc)); + CHECK_THROWS(decrypt_incoming(ed_sk, enc)); - auto [msg, sender] = decrypt_incoming(to_span(ed_sk2), enc); - CHECK(to_hex(sender) == oxenc::to_hex(ed_pk.begin(), ed_pk.end())); + auto [msg, sender] = decrypt_incoming(ed_sk2, enc); + CHECK(oxenc::to_hex(sender) == oxenc::to_hex(ed_pk)); CHECK(to_string(msg) == "hello"); auto broken = enc; - broken[2] ^= 0x02; - CHECK_THROWS(decrypt_incoming(to_span(ed_sk2), broken)); + broken[2] ^= std::byte{0x02}; + CHECK_THROWS(decrypt_incoming(ed_sk2, broken)); } SECTION("only seed, unprefixed sid") { constexpr auto lorem_ipsum = @@ -67,22 +65,18 @@ TEST_CASE("Session protocol encryption", "[session-protocol][encrypt]") { "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in " "culpa qui officia deserunt mollit anim id est laborum."sv; auto enc = - encrypt_for_recipient({to_span(ed_sk).data(), 32}, sid_raw2, to_span(lorem_ipsum)); - CHECK(std::search( - enc.begin(), - enc.end(), - to_unsigned("dolore magna"), - to_unsigned("dolore magna") + strlen("dolore magna")) == enc.end()); + encrypt_for_recipient(ed25519::extract_seed(ed_sk), sid_raw2, to_span(lorem_ipsum)); + CHECK_FALSE(std::ranges::search(enc, "dolore magna"_bytes)); - CHECK_THROWS(decrypt_incoming(to_span(ed_sk), enc)); + CHECK_THROWS(decrypt_incoming(ed_sk, enc)); - auto [msg, sender] = decrypt_incoming(to_span(ed_sk2), enc); - CHECK(to_hex(sender) == oxenc::to_hex(ed_pk.begin(), ed_pk.end())); + auto [msg, sender] = decrypt_incoming(ed_sk2, enc); + CHECK(oxenc::to_hex(sender) == oxenc::to_hex(ed_pk)); CHECK(to_string(msg) == lorem_ipsum); auto broken = enc; - broken[14] ^= 0x80; - CHECK_THROWS(decrypt_incoming(to_span(ed_sk2), broken)); + broken[14] ^= std::byte{0x80}; + CHECK_THROWS(decrypt_incoming(ed_sk2, broken)); } } @@ -90,43 +84,39 @@ TEST_CASE("Session protocol deterministic encryption", "[session-protocol][encry using namespace session; - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair(ed_pk.data(), ed_sk.data(), seed.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data())); - REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); + REQUIRE(oxenc::to_hex(ed_pk) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); - REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == + REQUIRE(oxenc::to_hex(curve_pk) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - auto sid = "05" + oxenc::to_hex(curve_pk.begin(), curve_pk.end()); - std::vector sid_raw; + auto sid = "05" + oxenc::to_hex(curve_pk); + std::vector sid_raw; oxenc::from_hex(sid.begin(), sid.end(), std::back_inserter(sid_raw)); REQUIRE(sid == "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - REQUIRE(sid_raw == - "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"_hexbytes); - - const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hexbytes; - std::array ed_pk2, curve_pk2; - std::array ed_sk2; - crypto_sign_ed25519_seed_keypair(ed_pk2.data(), ed_sk2.data(), seed2.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk2.data(), ed_pk2.data())); - REQUIRE(oxenc::to_hex(ed_pk2.begin(), ed_pk2.end()) == + REQUIRE(oxenc::to_hex(sid_raw) == + "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); + + const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hex_b; + auto [ed_pk2, ed_sk2] = ed25519::keypair(seed2); + auto curve_pk2 = ed25519::pk_to_x25519(ed_pk2); + REQUIRE(oxenc::to_hex(ed_pk2) == "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"); - REQUIRE(oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()) == + REQUIRE(oxenc::to_hex(curve_pk2) == "aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - auto sid2 = "05" + oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()); + auto sid2 = "05" + oxenc::to_hex(curve_pk2); REQUIRE(sid2 == "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - std::vector sid_raw2; + std::vector sid_raw2; oxenc::from_hex(sid2.begin(), sid2.end(), std::back_inserter(sid_raw2)); - REQUIRE(sid_raw2 == - "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"_hexbytes); + REQUIRE(oxenc::to_hex(sid_raw2) == + "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - auto enc1 = encrypt_for_recipient(to_span(ed_sk), sid_raw2, to_span("hello")); - auto enc2 = encrypt_for_recipient(to_span(ed_sk), sid_raw2, to_span("hello")); + auto enc1 = encrypt_for_recipient(ed_sk, sid_raw2, to_span("hello")); + auto enc2 = encrypt_for_recipient(ed_sk, sid_raw2, to_span("hello")); REQUIRE(enc1 != enc2); - auto enc_det = encrypt_for_recipient_deterministic(to_span(ed_sk), sid_raw2, to_span("hello")); + auto enc_det = encrypt_for_recipient_deterministic(ed_sk, sid_raw2, to_span("hello")); CHECK(enc_det != enc1); CHECK(enc_det != enc2); CHECK(enc_det.size() == enc1.size()); @@ -136,13 +126,13 @@ TEST_CASE("Session protocol deterministic encryption", "[session-protocol][encry "6aa3b7b218bdc6dd7c1adccda8ef4897f0f458492240b39079c27a6c791067ab26a03067a7602b50f0434639" "906f93e548f909d5286edde365ebddc146"); - auto [msg, sender] = decrypt_incoming(to_span(ed_sk2), enc_det); - CHECK(to_hex(sender) == oxenc::to_hex(ed_pk.begin(), ed_pk.end())); + auto [msg, sender] = decrypt_incoming(ed_sk2, enc_det); + CHECK(oxenc::to_hex(sender) == oxenc::to_hex(ed_pk)); CHECK(to_string(msg) == "hello"); } -static std::array prefixed(unsigned char prefix, const session::uc32& pubkey) { - std::array result; +static session::b33 prefixed(std::byte prefix, const session::b32& pubkey) { + session::b33 result; result[0] = prefix; std::memcpy(result.data() + 1, pubkey.data(), 32); return result; @@ -152,86 +142,58 @@ TEST_CASE("Session blinding protocol encryption", "[session-blinding-protocol][e using namespace session; - const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; - const auto server_pk = - "1d7e7f92b1ed3643855c98ecac02fc7274033a3467653f047d6e433540c03f17"_hexbytes; - std::array ed_pk, curve_pk; - std::array ed_sk; - crypto_sign_ed25519_seed_keypair(ed_pk.data(), ed_sk.data(), seed.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk.data(), ed_pk.data())); - REQUIRE(oxenc::to_hex(ed_pk.begin(), ed_pk.end()) == + const auto seed = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + constexpr auto server_pk = + "1d7e7f92b1ed3643855c98ecac02fc7274033a3467653f047d6e433540c03f17"_hex_b; + auto [ed_pk, ed_sk] = ed25519::keypair(seed); + auto curve_pk = ed25519::pk_to_x25519(ed_pk); + REQUIRE(oxenc::to_hex(ed_pk) == "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"); - REQUIRE(oxenc::to_hex(curve_pk.begin(), curve_pk.end()) == + REQUIRE(oxenc::to_hex(curve_pk) == "d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - auto sid = "05" + oxenc::to_hex(curve_pk.begin(), curve_pk.end()); - std::vector sid_raw; + auto sid = "05" + oxenc::to_hex(curve_pk); + std::vector sid_raw; oxenc::from_hex(sid.begin(), sid.end(), std::back_inserter(sid_raw)); REQUIRE(sid == "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); - REQUIRE(sid_raw == - "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"_hexbytes); - auto [blind15_pk, blind15_sk] = blind15_key_pair(to_span(ed_sk), to_span(server_pk)); - auto [blind25_pk, blind25_sk] = blind25_key_pair(to_span(ed_sk), to_span(server_pk)); - auto blind15_pk_prefixed = prefixed(0x15, blind15_pk); - auto blind25_pk_prefixed = prefixed(0x25, blind25_pk); - - const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hexbytes; - std::array ed_pk2, curve_pk2; - std::array ed_sk2; - crypto_sign_ed25519_seed_keypair(ed_pk2.data(), ed_sk2.data(), seed2.data()); - REQUIRE(0 == crypto_sign_ed25519_pk_to_curve25519(curve_pk2.data(), ed_pk2.data())); - REQUIRE(oxenc::to_hex(ed_pk2.begin(), ed_pk2.end()) == + REQUIRE(oxenc::to_hex(sid_raw) == + "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); + auto [blind15_pk, blind15_sk] = blind15_key_pair(ed_sk, server_pk); + auto [blind25_pk, blind25_sk] = blind25_key_pair(ed_sk, server_pk); + auto blind15_pk_prefixed = prefixed(std::byte{0x15}, blind15_pk); + auto blind25_pk_prefixed = prefixed(std::byte{0x25}, blind25_pk); + + const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hex_b; + auto [ed_pk2, ed_sk2] = ed25519::keypair(seed2); + auto curve_pk2 = ed25519::pk_to_x25519(ed_pk2); + REQUIRE(oxenc::to_hex(ed_pk2) == "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"); - REQUIRE(oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()) == + REQUIRE(oxenc::to_hex(curve_pk2) == "aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - auto sid2 = "05" + oxenc::to_hex(curve_pk2.begin(), curve_pk2.end()); + auto sid2 = "05" + oxenc::to_hex(curve_pk2); REQUIRE(sid2 == "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); - std::vector sid_raw2; + std::vector sid_raw2; oxenc::from_hex(sid2.begin(), sid2.end(), std::back_inserter(sid_raw2)); - REQUIRE(sid_raw2 == - "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"_hexbytes); - auto [blind15_pk2, blind15_sk2] = blind15_key_pair(to_span(ed_sk2), to_span(server_pk)); - auto [blind25_pk2, blind25_sk2] = blind25_key_pair(to_span(ed_sk2), to_span(server_pk)); - auto blind15_pk2_prefixed = prefixed(0x15, blind15_pk2); - auto blind25_pk2_prefixed = prefixed(0x25, blind25_pk2); + REQUIRE(oxenc::to_hex(sid_raw2) == + "05aa654f00fc39fc69fd0db829410ca38177d7732a8d2f0934ab3872ac56d5aa74"); + auto [blind15_pk2, blind15_sk2] = blind15_key_pair(ed_sk2, server_pk); + auto [blind25_pk2, blind25_sk2] = blind25_key_pair(ed_sk2, server_pk); + auto blind15_pk2_prefixed = prefixed(std::byte{0x15}, blind15_pk2); + auto blind25_pk2_prefixed = prefixed(std::byte{0x25}, blind25_pk2); SECTION("blind15, full secret, recipient decrypt") { auto enc = encrypt_for_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind15_pk2_prefixed.data(), 33}, - to_span("hello")); + ed_sk, server_pk, blind15_pk2_prefixed, to_span("hello")); CHECK(to_string(enc) != "hello"); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - to_span(blind15_pk), - {blind15_pk2_prefixed.data(), 33}, - enc)); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - to_span(blind15_pk2), - enc)); - auto [msg, sender] = decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, - enc); + ed_sk2, server_pk, blind15_pk_prefixed, blind15_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == "hello"); auto broken = enc; - broken[23] ^= 0x80; // 1 + 5 + 16 = 22 is the start of the nonce + broken[23] ^= std::byte{0x80}; // 1 + 5 + 16 = 22 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, - broken)); + ed_sk2, server_pk, blind15_pk_prefixed, blind15_pk2_prefixed, broken)); } SECTION("blind15, only seed, sender decrypt") { constexpr auto lorem_ipsum = @@ -242,32 +204,28 @@ TEST_CASE("Session blinding protocol encryption", "[session-blinding-protocol][e "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in " "culpa qui officia deserunt mollit anim id est laborum."sv; auto enc = encrypt_for_blinded_recipient( - {to_span(ed_sk).data(), 32}, - to_span(server_pk), - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk), + server_pk, + blind15_pk2_prefixed, to_span(lorem_ipsum)); - CHECK(std::search( - enc.begin(), - enc.end(), - to_unsigned("dolore magna"), - to_unsigned("dolore magna") + strlen("dolore magna")) == enc.end()); + CHECK_FALSE(std::ranges::search(enc, "dolore magna"_bytes)); auto [msg, sender] = decrypt_from_blinded_recipient( - {to_span(ed_sk).data(), 32}, - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk), + server_pk, + blind15_pk_prefixed, + blind15_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == lorem_ipsum); auto broken = enc; - broken[463] ^= 0x80; // 1 + 445 + 16 = 462 is the start of the nonce + broken[463] ^= std::byte{0x80}; // 1 + 445 + 16 = 462 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - {to_span(ed_sk).data(), 32}, - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk), + server_pk, + blind15_pk_prefixed, + blind15_pk2_prefixed, broken)); } SECTION("blind15, only seed, recipient decrypt") { @@ -279,111 +237,59 @@ TEST_CASE("Session blinding protocol encryption", "[session-blinding-protocol][e "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in " "culpa qui officia deserunt mollit anim id est laborum."sv; auto enc = encrypt_for_blinded_recipient( - {to_span(ed_sk).data(), 32}, - to_span(server_pk), - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk), + server_pk, + blind15_pk2_prefixed, to_span(lorem_ipsum)); - CHECK(std::search( - enc.begin(), - enc.end(), - to_unsigned("dolore magna"), - to_unsigned("dolore magna") + strlen("dolore magna")) == enc.end()); + CHECK_FALSE(std::ranges::search(enc, "dolore magna"_bytes)); auto [msg, sender] = decrypt_from_blinded_recipient( - {to_span(ed_sk2).data(), 32}, - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk2), + server_pk, + blind15_pk_prefixed, + blind15_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == lorem_ipsum); auto broken = enc; - broken[463] ^= 0x80; // 1 + 445 + 16 = 462 is the start of the nonce + broken[463] ^= std::byte{0x80}; // 1 + 445 + 16 = 462 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - {to_span(ed_sk2).data(), 32}, - to_span(server_pk), - {blind15_pk_prefixed.data(), 33}, - {blind15_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk2), + server_pk, + blind15_pk_prefixed, + blind15_pk2_prefixed, broken)); } SECTION("blind25, full secret, sender decrypt") { auto enc = encrypt_for_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind25_pk2_prefixed.data(), 33}, - to_span("hello")); + ed_sk, server_pk, blind25_pk2_prefixed, to_span("hello")); CHECK(to_string(enc) != "hello"); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - to_span(blind25_pk), - {blind25_pk2_prefixed.data(), 33}, - enc)); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - to_span(blind25_pk2), - enc)); - auto [msg, sender] = decrypt_from_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, - enc); + ed_sk, server_pk, blind25_pk_prefixed, blind25_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == "hello"); auto broken = enc; - broken[23] ^= 0x80; // 1 + 5 + 16 = 22 is the start of the nonce + broken[23] ^= std::byte{0x80}; // 1 + 5 + 16 = 22 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, - broken)); + ed_sk, server_pk, blind25_pk_prefixed, blind25_pk2_prefixed, broken)); } SECTION("blind25, full secret, recipient decrypt") { auto enc = encrypt_for_blinded_recipient( - to_span(ed_sk), - to_span(server_pk), - {blind25_pk2_prefixed.data(), 33}, - to_span("hello")); + ed_sk, server_pk, blind25_pk2_prefixed, to_span("hello")); CHECK(to_string(enc) != "hello"); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - to_span(blind25_pk), - {blind25_pk2_prefixed.data(), 33}, - enc)); - CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - to_span(blind25_pk2), - enc)); - auto [msg, sender] = decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, - enc); + ed_sk2, server_pk, blind25_pk_prefixed, blind25_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == "hello"); auto broken = enc; - broken[23] ^= 0x80; // 1 + 5 + 16 = 22 is the start of the nonce + broken[23] ^= std::byte{0x80}; // 1 + 5 + 16 = 22 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - to_span(ed_sk2), - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, - broken)); + ed_sk2, server_pk, blind25_pk_prefixed, blind25_pk2_prefixed, broken)); } SECTION("blind25, only seed, recipient decrypt") { constexpr auto lorem_ipsum = @@ -394,32 +300,28 @@ TEST_CASE("Session blinding protocol encryption", "[session-blinding-protocol][e "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in " "culpa qui officia deserunt mollit anim id est laborum."sv; auto enc = encrypt_for_blinded_recipient( - {to_span(ed_sk).data(), 32}, - to_span(server_pk), - {blind25_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk), + server_pk, + blind25_pk2_prefixed, to_span(lorem_ipsum)); - CHECK(std::search( - enc.begin(), - enc.end(), - to_unsigned("dolore magna"), - to_unsigned("dolore magna") + strlen("dolore magna")) == enc.end()); + CHECK_FALSE(std::ranges::search(enc, "dolore magna"_bytes)); auto [msg, sender] = decrypt_from_blinded_recipient( - {to_span(ed_sk2).data(), 32}, - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk2), + server_pk, + blind25_pk_prefixed, + blind25_pk2_prefixed, enc); CHECK(sender == sid); CHECK(to_string(msg) == lorem_ipsum); auto broken = enc; - broken[463] ^= 0x80; // 1 + 445 + 16 = 462 is the start of the nonce + broken[463] ^= std::byte{0x80}; // 1 + 445 + 16 = 462 is the start of the nonce CHECK_THROWS(decrypt_from_blinded_recipient( - {to_span(ed_sk2).data(), 32}, - to_span(server_pk), - {blind25_pk_prefixed.data(), 33}, - {blind25_pk2_prefixed.data(), 33}, + ed25519::extract_seed(ed_sk2), + server_pk, + blind25_pk_prefixed, + blind25_pk2_prefixed, broken)); } } @@ -430,17 +332,16 @@ TEST_CASE("Session ONS response decryption", "[session-ons][decrypt]") { std::string_view name = "test"; auto ciphertext = "3575802dd9bfea72672a208840f37ca289ceade5d3ffacabe2d231f109d204329fc33e28c33" - "1580d9a8c9b8a64cacfec97"_hexbytes; + "1580d9a8c9b8a64cacfec97"_hex_b; auto ciphertext_legacy = - "dbd4bc89bd2c9e5322fd9f4cadcaa66a0c38f15d0c927a86cc36e895fe1f3c532a3958d972563f52ca858e94eec22dc360"_hexbytes; - auto nonce = "00112233445566778899aabbccddeeff00ffeeddccbbaa99"_hexbytes; + "dbd4bc89bd2c9e5322fd9f4cadcaa66a0c38f15d0c927a86cc36e895fe1f3c532a3958d972563f52ca858e94eec22dc360"_hex_b; + constexpr auto nonce = "00112233445566778899aabbccddeeff00ffeeddccbbaa99"_hex_b; CHECK(decrypt_ons_response(name, ciphertext, nonce) == "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); CHECK(decrypt_ons_response(name, ciphertext_legacy, std::nullopt) == "05d2ad010eeb72d72e561d9de7bd7b6989af77dcabffa03a5111a6c859ae5c3a72"); CHECK_THROWS(decrypt_ons_response(name, to_span("invalid"), nonce)); - CHECK_THROWS(decrypt_ons_response(name, ciphertext, to_span("invalid"))); } TEST_CASE("Session ONS response decryption C API", "[session-ons][session_decrypt_ons_response]") { @@ -449,10 +350,10 @@ TEST_CASE("Session ONS response decryption C API", "[session-ons][session_decryp auto name = "test\0"; auto ciphertext = "3575802dd9bfea72672a208840f37ca289ceade5d3ffacabe2d231f109d204329fc33e28c33" - "1580d9a8c9b8a64cacfec97"_hexbytes; + "1580d9a8c9b8a64cacfec97"_hex_u; auto ciphertext_legacy = - "dbd4bc89bd2c9e5322fd9f4cadcaa66a0c38f15d0c927a86cc36e895fe1f3c532a3958d972563f52ca858e94eec22dc360"_hexbytes; - auto nonce = "00112233445566778899aabbccddeeff00ffeeddccbbaa99"_hexbytes; + "dbd4bc89bd2c9e5322fd9f4cadcaa66a0c38f15d0c927a86cc36e895fe1f3c532a3958d972563f52ca858e94eec22dc360"_hex_u; + auto nonce = "00112233445566778899aabbccddeeff00ffeeddccbbaa99"_hex_u; char ons1[67]; CHECK(session_decrypt_ons_response( @@ -470,30 +371,123 @@ TEST_CASE("Session push notification decryption", "[session-notification][decryp auto payload = "00112233445566778899aabbccddeeff00ffeeddccbbaa991bcba42892762dbeecbfb1a375f" - "ab4aca5f0991e99eb0344ceeafa"_hexbytes; + "ab4aca5f0991e99eb0344ceeafa"_hex_b; auto payload_padded = "00112233445566778899aabbccddeeff00ffeeddccbbaa991bcba42892762dbeecbfb1a375f" - "ab4aca5f0991e99eb0344ceeafa"_hexbytes; - auto enc_key = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; + "ab4aca5f0991e99eb0344ceeafa"_hex_b; + constexpr auto enc_key = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; CHECK(decrypt_push_notification(payload, enc_key) == to_vector("TestMessage")); CHECK(decrypt_push_notification(payload_padded, enc_key) == to_vector("TestMessage")); CHECK_THROWS(decrypt_push_notification(to_span("invalid"), enc_key)); - CHECK_THROWS(decrypt_push_notification(payload, to_span("invalid"))); } TEST_CASE("xchacha20", "[session][xchacha20]") { using namespace session; auto payload = - "da74ac6e96afda1c5a07d5bde1b8b1e1c05be73cb3c84112f31f00369d67154d00ff029090b069b48c3cf603d838d4ef623d54"_hexbytes; - auto enc_key = "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hexbytes; + "da74ac6e96afda1c5a07d5bde1b8b1e1c05be73cb3c84112f31f00369d67154d00ff029090b069b48c3cf603d838d4ef623d54"_hex_b; + constexpr auto enc_key = + "0123456789abcdef0123456789abcdeffedcba9876543210fedcba9876543210"_hex_b; CHECK(decrypt_xchacha20(payload, enc_key) == to_vector("TestMessage")); CHECK_THROWS(decrypt_xchacha20(to_span("invalid"), enc_key)); - CHECK_THROWS(decrypt_xchacha20(payload, to_span("invalid"))); auto ciphertext = encrypt_xchacha20(to_span("TestMessage"), enc_key); CHECK(decrypt_xchacha20(ciphertext, enc_key) == to_vector("TestMessage")); - CHECK_THROWS(encrypt_xchacha20(payload, to_span("invalid"))); +} + +TEST_CASE("v2 PFS+PQ message encryption", "[session-protocol][encrypt][v2]") { + using namespace session; + + // Sender: existing well-known test keypair 1 + const auto seed1 = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; + auto [sender_ed_pk, sender_ed_sk] = ed25519::keypair(seed1); + + // Recipient: long-term session identity from test keypair 2 + const auto seed2 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hex_b; + auto [recip_ed_pk, recip_ed_sk] = ed25519::keypair(seed2); + auto recip_curve_pk = ed25519::pk_to_x25519(recip_ed_pk); + auto recip_x25519_sec = ed25519::sk_to_x25519(recip_ed_sk); + + b33 recip_session_id; + recip_session_id[0] = std::byte{0x05}; + std::copy(recip_curve_pk.begin(), recip_curve_pk.end(), recip_session_id.begin() + 1); + + // Recipient PFS X25519 account key (deterministic) + const auto pfs_x25519_sec = + "aabbccddeeff0011223344556677889900112233445566778899aabbccddeeff"_hex_b; + auto pfs_x25519_pub = x25519::scalarmult_base(pfs_x25519_sec); + + // Recipient PFS ML-KEM-768 account key (deterministic, needs 64-byte seed) + const auto pfs_mlkem_seed = + "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef" + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"_hex_b; + std::array pfs_mlkem_pub; + std::array pfs_mlkem_sec; + mlkem768::keygen(pfs_mlkem_pub, pfs_mlkem_sec, pfs_mlkem_seed); + + // Encrypt a message from sender to recipient + auto ct = encrypt_for_recipient_v2( + sender_ed_sk, + recip_session_id, + pfs_x25519_pub, + pfs_mlkem_pub, + to_span("hello world"), + std::nullopt); + + // Ciphertext is padded to a multiple of 256 bytes + CHECK(ct.size() % 256 == 0); + + // decrypt_incoming_v2_prefix recovers the 2-byte ML-KEM pubkey prefix using the + // recipient's long-term X25519 keys (cheap; no PFS keys needed at this stage) + auto prefix = decrypt_incoming_v2_prefix(recip_x25519_sec, recip_curve_pk, ct); + CHECK(prefix[0] == pfs_mlkem_pub[0]); + CHECK(prefix[1] == pfs_mlkem_pub[1]); + + // Decrypt with the correct keys succeeds + auto result = decrypt_incoming_v2( + recip_session_id, pfs_x25519_sec, pfs_x25519_pub, pfs_mlkem_sec, ct); + CHECK(result.content == to_vector("hello world")); + CHECK(result.sender_session_id[0] == std::byte{0x05}); + CHECK(!result.pro_signature); + + // The recovered sender session ID matches the sender's X25519 pubkey + auto sender_curve_pk = ed25519::pk_to_x25519(sender_ed_pk); + CHECK(std::equal( + result.sender_session_id.begin() + 1, + result.sender_session_id.end(), + sender_curve_pk.begin())); + + // Wrong X25519 key throws DecryptV2Error (wrong-key failure, not a format error) + b32 wrong_x25519_sec; + std::ranges::copy(pfs_x25519_sec, wrong_x25519_sec.begin()); + wrong_x25519_sec[0] ^= std::byte{0xff}; + auto wrong_x25519_pub = x25519::scalarmult_base(wrong_x25519_sec); + CHECK_THROWS_AS( + decrypt_incoming_v2( + recip_session_id, wrong_x25519_sec, wrong_x25519_pub, pfs_mlkem_sec, ct), + DecryptV2Error); + + // Truncated ciphertext throws before key matching (unrecoverable format error) + auto truncated = std::vector(ct.begin(), ct.begin() + 100); + CHECK_THROWS_AS( + decrypt_incoming_v2_prefix(recip_x25519_sec, recip_curve_pk, truncated), + std::runtime_error); + + // Encrypting and decrypting with a pro private key + auto [pro_pk, pro_sk] = ed25519::keypair(); + auto ct_pro = encrypt_for_recipient_v2( + sender_ed_sk, + recip_session_id, + pfs_x25519_pub, + pfs_mlkem_pub, + to_span("hello world"), + pro_sk); + auto result_pro = decrypt_incoming_v2( + recip_session_id, pfs_x25519_sec, pfs_x25519_pub, pfs_mlkem_sec, ct_pro); + REQUIRE(result_pro.pro_signature.has_value()); + // The signature should be 64 bytes and verifiable with the pro public key. + CHECK(result_pro.pro_signature->size() == 64); } diff --git a/tests/test_session_protocol.cpp b/tests/test_session_protocol.cpp index efd24a7a5..2e822c1ba 100644 --- a/tests/test_session_protocol.cpp +++ b/tests/test_session_protocol.cpp @@ -17,20 +17,20 @@ using namespace session; struct SerialisedProtobufContentWithProForTesting { ProProof proof; std::string plaintext; - std::vector plaintext_padded; - array_uc64 sig_over_plaintext_with_user_pro_key; - array_uc64 sig_over_plaintext_padded_with_user_pro_key; - bytes64 sig_over_plaintext_with_user_pro_key_c; + std::vector plaintext_padded; + b64 sig_over_plaintext_with_user_pro_key; + b64 sig_over_plaintext_padded_with_user_pro_key; + cbytes64 sig_over_plaintext_with_user_pro_key_c; }; static SerialisedProtobufContentWithProForTesting build_protobuf_content_with_session_pro( std::string_view data_body, - const array_uc64& user_rotating_privkey, - const array_uc64& pro_backend_privkey, + const ed25519::PrivKeySpan& user_rotating_privkey, + const ed25519::PrivKeySpan& pro_backend_privkey, std::chrono::sys_seconds content_at, std::chrono::sys_seconds pro_expiry_at, - session_protocol_pro_message_bitset msg_bitset, - session_protocol_pro_profile_bitset profile_bitset, + uint64_t msg_bitset, + uint64_t profile_bitset, bool omit_proof = false) { SerialisedProtobufContentWithProForTesting result = {}; @@ -44,24 +44,18 @@ static SerialisedProtobufContentWithProForTesting build_protobuf_content_with_se data->set_body(std::string(data_body)); // Generate a dummy proof - crypto_sign_ed25519_sk_to_pk(result.proof.rotating_pubkey.data(), user_rotating_privkey.data()); + std::ranges::copy(user_rotating_privkey.pubkey(), result.proof.rotating_pubkey.begin()); result.proof.expiry_at = pro_expiry_at; // Sign the proof by the dummy "Session Pro Backend" key (Ed25519 over the message directly) - auto proof_msg = result.proof.signed_message(); - crypto_sign_ed25519_detached( - result.proof.sig.data(), - nullptr, - proof_msg.data(), - proof_msg.size(), - pro_backend_privkey.data()); + result.proof.sig = ed25519::sign(pro_backend_privkey, result.proof.signed_message()); // Create protobuf `Content.proMessage` SessionProtos::ProMessage* pro = content.mutable_promessage(); - pro->set_profilebitset(profile_bitset.data); - pro->set_msgbitset(msg_bitset.data); + pro->set_profilebitset(profile_bitset); + pro->set_msgbitset(msg_bitset); - // Create protobuf `Content.proMessage.proof` + // Create protobuf `Content.proMessage.proof`. // // `omit_proof` leaves it out entirely, which is what a proof this client cannot read looks like // from here: a new proof format arrives as its own field rather than as a version bump on this @@ -84,19 +78,10 @@ static SerialisedProtobufContentWithProForTesting build_protobuf_content_with_se REQUIRE(result.plaintext_padded.size() % SESSION_PROTOCOL_COMMUNITY_OR_1O1_MSG_PADDING == 0); // Sign the plaintext with the user's pro key - crypto_sign_ed25519_detached( - result.sig_over_plaintext_with_user_pro_key.data(), - nullptr, - reinterpret_cast(result.plaintext.data()), - result.plaintext.size(), - user_rotating_privkey.data()); - - crypto_sign_ed25519_detached( - result.sig_over_plaintext_padded_with_user_pro_key.data(), - nullptr, - reinterpret_cast(result.plaintext_padded.data()), - result.plaintext_padded.size(), - user_rotating_privkey.data()); + result.sig_over_plaintext_with_user_pro_key = + ed25519::sign(user_rotating_privkey, to_span(result.plaintext)); + result.sig_over_plaintext_padded_with_user_pro_key = + ed25519::sign(user_rotating_privkey, result.plaintext_padded); // Setup the C versions for convenience std::memcpy( @@ -116,7 +101,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { session_protocol_pro_features_for_message( SESSION_PROTOCOL_STANDARD_CHARACTER_LIMIT); REQUIRE(pro_msg.status == SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_SUCCESS); - REQUIRE(pro_msg.bitset.data == 0); + REQUIRE(pro_msg.bitset == 0); } // Exceeding the standard size threshold @@ -125,8 +110,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { session_protocol_pro_features_for_message( SESSION_PROTOCOL_STANDARD_CHARACTER_LIMIT + 1); REQUIRE(pro_msg.status == SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_SUCCESS); - REQUIRE(session_protocol_pro_message_bitset_is_set( - pro_msg.bitset, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT)); + REQUIRE((pro_msg.bitset & SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT)); } // At the max size threshold @@ -135,8 +119,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { session_protocol_pro_features_for_message( SESSION_PROTOCOL_PRO_HIGHER_CHARACTER_LIMIT); REQUIRE(pro_msg.status == SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_SUCCESS); - REQUIRE(session_protocol_pro_message_bitset_is_set( - pro_msg.bitset, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT)); + REQUIRE((pro_msg.bitset & SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT)); } // Over the max size threshold @@ -146,7 +129,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { SESSION_PROTOCOL_PRO_HIGHER_CHARACTER_LIMIT + 1); REQUIRE(pro_msg.status == SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS_EXCEEDS_CHARACTER_LIMIT); - REQUIRE(pro_msg.bitset.data == 0); + REQUIRE(pro_msg.bitset == 0); } } @@ -163,22 +146,19 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Generate the user's Session Pro rotating key for testing encrypted payloads with Session // Pro metadata const auto user_pro_seed = - "0123456789abcdef0123456789abcdeff00baa00000000000000000000000000"_hexbytes; - array_uc32 user_pro_ed_pk; - array_uc64 user_pro_ed_sk; - crypto_sign_ed25519_seed_keypair( - user_pro_ed_pk.data(), user_pro_ed_sk.data(), user_pro_seed.data()); + "0123456789abcdef0123456789abcdeff00baa00000000000000000000000000"_hex_b; + auto [user_pro_ed_pk, user_pro_ed_sk] = ed25519::keypair(user_pro_seed); SECTION("Encrypt with and w/o pro sig produce same payload size") { // Same payload size because the encrypt function should put in a dummy signature if one // wasn't specific to make pro and non-pro envelopes indistinguishable. - bytes33 recipient_pubkey = {}; + cbytes33 recipient_pubkey = {}; std::memcpy(recipient_pubkey.data, keys.session_pk1.data(), sizeof(recipient_pubkey.data)); // Withhold the pro signature char error[256]; session_protocol_encoded_for_destination encrypt_without_pro_sig = - session_protocol_encode_for_1o1( + session_protocol_encode_dm_v1( data_body.data(), data_body.size(), keys.ed_sk0.data(), @@ -194,7 +174,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Set the pro signature session_protocol_encoded_for_destination encrypt_with_pro_sig = - session_protocol_encode_for_1o1( + session_protocol_encode_dm_v1( data_body.data(), data_body.size(), keys.ed_sk0.data(), @@ -216,8 +196,6 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Setup a dummy "Session Pro Backend" key // We reuse test key 1 as the "Session Pro" backend key that signs the proofs as it // doesn't matter what key really, just that we have one available for signing. - const array_uc64& pro_backend_ed_sk = keys.ed_sk1; - const array_uc32& pro_backend_ed_pk = keys.ed_pk1; char error[256]; SECTION("Encrypt/decrypt for contact in default namespace w/o pro attached") { @@ -236,9 +214,9 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Encrypt session_protocol_encoded_for_destination encrypt_result = {}; { - bytes33 recipient_pubkey = {}; + cbytes33 recipient_pubkey = {}; std::memcpy(recipient_pubkey.data, keys.session_pk1.data(), keys.session_pk1.size()); - encrypt_result = session_protocol_encode_for_1o1( + encrypt_result = session_protocol_encode_dm_v1( plaintext.data(), plaintext.size(), keys.ed_sk0.data(), @@ -261,8 +239,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); INFO("ERROR: " << error); @@ -274,8 +252,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { ProProof nil_proof = {}; REQUIRE(decrypt_result.pro.status == SESSION_PROTOCOL_PRO_STATUS_NIL); // Pro was not attached - REQUIRE(decrypt_result.pro.msg_bitset.data == 0); - REQUIRE(decrypt_result.pro.profile_bitset.data == 0); + REQUIRE(decrypt_result.pro.msg_bitset == 0); + REQUIRE(decrypt_result.pro.profile_bitset == 0); // No proof was attached, so the decoded proof is empty (matches a default-constructed one). REQUIRE(std::memcmp( decrypt_result.pro.proof.sig.data, @@ -298,54 +276,43 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { build_protobuf_content_with_session_pro( /*data_body*/ data_body, /*user_rotating_privkey*/ user_pro_ed_sk, - /*pro_backend_privkey*/ pro_backend_ed_sk, + /*pro_backend_privkey*/ keys.ed_sk1, /*content_at=*/timestamp_s, /*pro_expiry_at*/ timestamp_s, /*msg_bitset*/ {}, /*profile_bitset*/ {}); // Setup base destination object with the pro signature w/ Session pubkey 1 as the recipient - bytes64 base_pro_sig = {}; + cbytes64 base_pro_sig = {}; std::memcpy( base_pro_sig.data, protobuf_content.sig_over_plaintext_with_user_pro_key.data(), sizeof(base_pro_sig.data)); - session_protocol_destination base_dest = {}; - base_dest.sent_timestamp_ms = timestamp_ms.time_since_epoch().count(); - base_dest.pro_rotating_ed25519_privkey = user_pro_ed_sk.data(); - base_dest.pro_rotating_ed25519_privkey_len = user_pro_ed_sk.size(); - - REQUIRE(sizeof(base_dest.recipient_pubkey.data) == keys.session_pk1.size()); - std::memcpy(base_dest.recipient_pubkey.data, keys.session_pk1.data(), keys.session_pk1.size()); + uint64_t base_sent_timestamp_ms = timestamp_ms.time_since_epoch().count(); + cbytes33 base_recipient_pubkey = {}; + REQUIRE(sizeof(base_recipient_pubkey.data) == keys.session_pk1.size()); + std::memcpy(base_recipient_pubkey.data, keys.session_pk1.data(), keys.session_pk1.size()); SECTION("Check non-encryptable messages produce only plaintext") { - auto dest_list = { - SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX, - SESSION_PROTOCOL_DESTINATION_TYPE_SYNC_OR_1O1}; - - for (auto dest_type : dest_list) { - if (dest_type == SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX) - INFO("Trying community inbox"); - else - INFO("Trying contacts to non-default namespace"); - - session_protocol_destination dest = base_dest; - dest.type = dest_type; - if (dest_type == SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX) { - auto [blind15_pk, blind15_sk] = session::blind15_key_pair( - keys.ed_sk1, keys.ed_pk1, /*blind factor*/ nullptr); - dest.recipient_pubkey.data[0] = 0x15; - std::memcpy(dest.recipient_pubkey.data + 1, blind15_pk.data(), blind15_pk.size()); - } + SECTION("Community inbox") { + auto [blind15_pk, blind15_sk] = session::blind15_key_pair( + keys.ed_sk1, to_byte_span<32>(keys.ed_pk1.data()), /*blind factor*/ nullptr); + cbytes33 blind15_recipient = {}; + blind15_recipient.data[0] = 0x15; + std::memcpy(blind15_recipient.data + 1, blind15_pk.data(), blind15_pk.size()); + cbytes32 community_pubkey = {}; session_protocol_encoded_for_destination encrypt_result = - session_protocol_encode_for_destination( + session_protocol_encode_for_community_inbox( protobuf_content.plaintext.data(), protobuf_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - &dest, + &blind15_recipient, + &community_pubkey, + user_pro_ed_sk.data(), + user_pro_ed_sk.size(), error, sizeof(error)); INFO("ERROR: " << error); @@ -353,17 +320,35 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { REQUIRE(encrypt_result.error_len_incl_null_terminator == 0); session_protocol_encode_for_destination_free(&encrypt_result); } + + SECTION("Contact in non-default namespace") { + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_dm_v1( + protobuf_content.plaintext.data(), + protobuf_content.plaintext.size(), + keys.ed_sk0.data(), + keys.ed_sk0.size(), + base_sent_timestamp_ms, + &base_recipient_pubkey, + user_pro_ed_sk.data(), + user_pro_ed_sk.size(), + error, + sizeof(error)); + INFO("ERROR: " << error); + REQUIRE(encrypt_result.ciphertext.size > 0); + REQUIRE(encrypt_result.error_len_incl_null_terminator == 0); + session_protocol_encode_for_destination_free(&encrypt_result); + } } SECTION("Encrypt/decrypt for contact in default namespace with Pro") { // Encrypt content - session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_for_1o1( + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_dm_v1( protobuf_content.plaintext.data(), protobuf_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - base_dest.sent_timestamp_ms, - &base_dest.recipient_pubkey, + base_sent_timestamp_ms, + &base_recipient_pubkey, user_pro_ed_sk.data(), user_pro_ed_sk.size(), error, @@ -379,8 +364,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); REQUIRE(decrypt_result.success); @@ -396,8 +381,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { decrypt_result.pro.proof.sig.data, protobuf_content.proof.sig.data(), sizeof(decrypt_result.pro.proof.sig.data)) == 0); - REQUIRE(decrypt_result.pro.msg_bitset.data == 0); // No features requested - REQUIRE(decrypt_result.pro.profile_bitset.data == 0); // No features requested + REQUIRE(decrypt_result.pro.msg_bitset == 0); // No features requested + REQUIRE(decrypt_result.pro.profile_bitset == 0); // No features requested // Verify the content can be parsed w/ protobufs SessionProtos::Content decrypt_content = {}; @@ -416,20 +401,20 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { build_protobuf_content_with_session_pro( /*data_body*/ data_body, /*user_rotating_privkey*/ user_pro_ed_sk, - /*pro_backend_privkey*/ pro_backend_ed_sk, + /*pro_backend_privkey*/ keys.ed_sk1, /*content_at=*/timestamp_s, /*pro_expiry_at*/ timestamp_s, /*msg_bitset*/ {}, /*profile_bitset*/ {}, /*omit_proof*/ true); - session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_for_1o1( + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_dm_v1( future_content.plaintext.data(), future_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - base_dest.sent_timestamp_ms, - &base_dest.recipient_pubkey, + base_sent_timestamp_ms, + &base_recipient_pubkey, user_pro_ed_sk.data(), user_pro_ed_sk.size(), error, @@ -444,8 +429,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); // The message is delivered rather than silently swallowed by the unreadable proof... @@ -471,31 +456,29 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { session_protocol_pro_features_for_msg pro_msg = session_protocol_pro_features_for_message(large_message.size()); - REQUIRE(session_protocol_pro_message_bitset_is_set( - pro_msg.bitset, SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT)); + REQUIRE((pro_msg.bitset & SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT)); - session_protocol_pro_profile_bitset profile_bitset = {}; - session_protocol_pro_profile_bitset_set( - &profile_bitset, SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE); + uint64_t profile_bitset = 0; + profile_bitset |= SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE; SerialisedProtobufContentWithProForTesting protobuf_content_with_pro_and_features = build_protobuf_content_with_session_pro( /*data_body*/ large_message, /*user_rotating_privkey*/ user_pro_ed_sk, - /*pro_backend_privkey*/ pro_backend_ed_sk, + /*pro_backend_privkey*/ keys.ed_sk1, /*content_at*/ timestamp_s, /*pro_expiry_at*/ timestamp_s, /*msg_bitset*/ pro_msg.bitset, /*proilfe_bitset*/ profile_bitset); // Encrypt content - session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_for_1o1( + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_dm_v1( protobuf_content_with_pro_and_features.plaintext.data(), protobuf_content_with_pro_and_features.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - base_dest.sent_timestamp_ms, - &base_dest.recipient_pubkey, + base_sent_timestamp_ms, + &base_recipient_pubkey, user_pro_ed_sk.data(), user_pro_ed_sk.size(), error, @@ -512,14 +495,14 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); INFO("ERROR: " << error); REQUIRE(decrypt_result.success); REQUIRE(decrypt_result.error_len_incl_null_terminator == 0); - REQUIRE(decrypt_result.envelope.timestamp_ms == base_dest.sent_timestamp_ms); + REQUIRE(decrypt_result.envelope.timestamp_ms == base_sent_timestamp_ms); session_protocol_encode_for_destination_free(&encrypt_result); // Verify pro @@ -531,12 +514,12 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { decrypt_result.pro.proof.sig.data, protobuf_content.proof.sig.data(), sizeof(decrypt_result.pro.proof.sig.data)) == 0); - REQUIRE(session_protocol_pro_profile_bitset_is_set( - decrypt_result.pro.profile_bitset, - SESSION_PROTOCOL_PRO_PROFILE_FEATURES_PRO_BADGE)); - REQUIRE(session_protocol_pro_message_bitset_is_set( - decrypt_result.pro.msg_bitset, - SESSION_PROTOCOL_PRO_MESSAGE_FEATURES_10K_CHARACTER_LIMIT)); + REQUIRE( + (decrypt_result.pro.profile_bitset & + SESSION_PROTOCOL_PRO_PROFILE_FEATURE_PRO_BADGE)); + REQUIRE( + (decrypt_result.pro.msg_bitset & + SESSION_PROTOCOL_PRO_MESSAGE_FEATURE_10K_CHARACTER_LIMIT)); // Verify the content can be parsed w/ protobufs SessionProtos::Content decrypt_content = {}; @@ -549,19 +532,21 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { } SECTION("Encrypt/decrypt for legacy groups is rejected") { - session_protocol_destination dest = base_dest; - dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_GROUP; - assert(dest.recipient_pubkey.data[0] == 0x05); + CHECK(base_recipient_pubkey.data[0] == 0x05); + cbytes32 group_enc_key = {}; - session_protocol_encoded_for_destination encrypt_result = - session_protocol_encode_for_destination( - protobuf_content.plaintext.data(), - protobuf_content.plaintext.size(), - keys.ed_sk0.data(), - keys.ed_sk0.size(), - &dest, - error, - sizeof(error)); + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_for_group( + protobuf_content.plaintext.data(), + protobuf_content.plaintext.size(), + keys.ed_sk0.data(), + keys.ed_sk0.size(), + base_sent_timestamp_ms, + &base_recipient_pubkey, + &group_enc_key, + nullptr, + 0, + error, + sizeof(error)); REQUIRE(encrypt_result.error_len_incl_null_terminator > 0); REQUIRE(encrypt_result.error_len_incl_null_terminator <= sizeof(error)); REQUIRE(!encrypt_result.success); @@ -571,28 +556,30 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { SECTION("Encrypt/decrypt for groups v2 (w/ encrypted envelope, plaintext content) with Pro") { // TODO: Finish setting up a fake group const auto group_v2_seed = - "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hexbytes; - array_uc64 group_v2_sk = {}; - array_uc32 group_v2_pk = {}; - crypto_sign_ed25519_seed_keypair( - group_v2_pk.data(), group_v2_sk.data(), group_v2_seed.data()); + "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hex_b; + auto [group_v2_pk, group_v2_sk] = ed25519::keypair(group_v2_seed); // Encrypt session_protocol_encoded_for_destination encrypt_result = {}; { - bytes33 group_v2_session_pk = {}; - bytes32 group_v2_session_sk = {}; + cbytes33 group_v2_session_pk = {}; + cbytes32 group_v2_session_sk = {}; group_v2_session_pk.data[0] = 0x03; - std::memcpy(group_v2_session_pk.data + 1, group_v2_pk.data(), group_v2_pk.size()); std::memcpy( - group_v2_session_sk.data, group_v2_sk.data(), sizeof(group_v2_session_sk.data)); + group_v2_session_pk.data + 1, + to_unsigned(group_v2_pk.data()), + group_v2_pk.size()); + std::memcpy( + group_v2_session_sk.data, + to_unsigned(group_v2_sk.data()), + sizeof(group_v2_session_sk.data)); encrypt_result = session_protocol_encode_for_group( protobuf_content.plaintext.data(), protobuf_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - base_dest.sent_timestamp_ms, + base_sent_timestamp_ms, &group_v2_session_pk, &group_v2_session_sk, user_pro_ed_sk.data(), @@ -605,9 +592,9 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { } // Decrypt envelope - span_u8 key = {group_v2_sk.data(), group_v2_sk.size()}; + span_u8 key = {to_unsigned(group_v2_sk.data()), 32}; session_protocol_decode_envelope_keys decrypt_keys = {}; - decrypt_keys.group_ed25519_pubkey = {group_v2_pk.data(), group_v2_pk.size()}; + decrypt_keys.group_ed25519_pubkey = {to_unsigned(group_v2_pk.data()), group_v2_pk.size()}; decrypt_keys.decrypt_keys = &key; decrypt_keys.decrypt_keys_len = 1; @@ -617,8 +604,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); INFO("Decrypt for group error: " << error); @@ -637,13 +624,13 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { SECTION("Encrypt/decrypt for sync messages with Pro") { // Encrypt - session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_for_1o1( + session_protocol_encoded_for_destination encrypt_result = session_protocol_encode_dm_v1( protobuf_content.plaintext.data(), protobuf_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), - base_dest.sent_timestamp_ms, - &base_dest.recipient_pubkey, + base_sent_timestamp_ms, + &base_recipient_pubkey, user_pro_ed_sk.data(), user_pro_ed_sk.size(), error, @@ -660,8 +647,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); REQUIRE(decrypt_result.error_len_incl_null_terminator == 0); @@ -676,8 +663,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { decrypt_result.pro.proof.sig.data, protobuf_content.proof.sig.data(), sizeof(decrypt_result.pro.proof.sig.data)) == 0); - REQUIRE(decrypt_result.pro.msg_bitset.data == 0); // No features requested - REQUIRE(decrypt_result.pro.profile_bitset.data == 0); // No features requested + REQUIRE(decrypt_result.pro.msg_bitset == 0); // No features requested + REQUIRE(decrypt_result.pro.profile_bitset == 0); // No features requested // Verify the content can be parsed w/ protobufs SessionProtos::Content decrypt_content = {}; @@ -702,7 +689,7 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { build_protobuf_content_with_session_pro( /*data_body*/ data_body, /*user_rotating_privkey*/ user_pro_ed_sk, - /*pro_backend_privkey*/ pro_backend_ed_sk, + /*pro_backend_privkey*/ keys.ed_sk1, /*content_at=*/ std::chrono::sys_seconds( std::chrono::duration_cast( @@ -712,13 +699,13 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { /*profile_bitset*/ {}); session_protocol_encoded_for_destination encrypt_bad_result = - session_protocol_encode_for_1o1( + session_protocol_encode_dm_v1( bad_protobuf_content.plaintext.data(), bad_protobuf_content.plaintext.size(), keys.ed_sk0.data(), keys.ed_sk0.size(), bad_timestamp_ms.count(), - &base_dest.recipient_pubkey, + &base_recipient_pubkey, user_pro_ed_sk.data(), user_pro_ed_sk.size(), error, @@ -729,8 +716,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &decrypt_keys, encrypt_bad_result.ciphertext.data, encrypt_bad_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); REQUIRE(decrypt_result.success); @@ -741,14 +728,14 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Try decrypt with a bad backend key { - array_uc32 bad_pro_backend_ed_pk = pro_backend_ed_pk; - bad_pro_backend_ed_pk[0] ^= 1; + uc32 bad_pro_ed_pk = keys.ed_pk1; + bad_pro_ed_pk[0] ^= 1; session_protocol_decoded_envelope decrypt_result = session_protocol_decode_envelope( &decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - bad_pro_backend_ed_pk.data(), - bad_pro_backend_ed_pk.size(), + bad_pro_ed_pk.data(), + bad_pro_ed_pk.size(), error, sizeof(error)); REQUIRE(decrypt_result.success); @@ -768,8 +755,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &bad_decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); INFO("Checking error from bad envelope decryption: " << std::string_view( @@ -790,8 +777,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { &multi_decrypt_keys, encrypt_result.ciphertext.data, encrypt_result.ciphertext.size, - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); REQUIRE(decrypt_result.success); @@ -817,8 +804,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { encoded.ciphertext.data, encoded.ciphertext.size, timestamp_s.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; @@ -840,8 +827,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { encoded.ciphertext.data, encoded.ciphertext.size, timestamp_s.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; @@ -861,8 +848,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { envelope_plaintext.data(), envelope_plaintext.size(), timestamp_s.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; @@ -885,8 +872,8 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { envelope_plaintext.data(), envelope_plaintext.size(), timestamp_s.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; @@ -897,32 +884,29 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { SECTION("Encode/decode for community inbox (content message)") { const auto community_seed = - "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hexbytes; - array_uc64 community_sk = {}; - array_uc32 community_pk = {}; - crypto_sign_ed25519_seed_keypair( - community_pk.data(), community_sk.data(), community_seed.data()); - - bytes32 session_blind15_sk0 = {}; - bytes33 session_blind15_pk0 = {}; + "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hex_b; + auto [community_pk, community_sk] = ed25519::keypair(community_seed); + + cbytes32 session_blind15_sk0 = {}; + cbytes33 session_blind15_pk0 = {}; session_blind15_pk0.data[0] = 0x15; session_blind15_key_pair( keys.ed_sk0.data(), - community_pk.data(), + to_unsigned(community_pk.data()), session_blind15_pk0.data + 1, session_blind15_sk0.data); - bytes32 session_blind15_sk1 = {}; - bytes33 session_blind15_pk1 = {}; + cbytes32 session_blind15_sk1 = {}; + cbytes33 session_blind15_pk1 = {}; session_blind15_pk1.data[0] = 0x15; session_blind15_key_pair( keys.ed_sk1.data(), - community_pk.data(), + to_unsigned(community_pk.data()), session_blind15_pk1.data + 1, session_blind15_sk1.data); - bytes33 recipient_pubkey = session_blind15_pk1; - bytes32 community_pubkey = {}; + cbytes33 recipient_pubkey = session_blind15_pk1; + cbytes32 community_pubkey = {}; std::memcpy(community_pubkey.data, community_pk.data(), community_pk.size()); session_protocol_encoded_for_destination encoded = @@ -942,16 +926,16 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { auto [decrypted_cipher, sender_id] = session::decrypt_from_blinded_recipient( keys.ed_sk1, community_pk, - {session_blind15_pk0.data, sizeof(session_blind15_pk0.data)}, - {session_blind15_pk1.data, sizeof(session_blind15_pk1.data)}, - {encoded.ciphertext.data, encoded.ciphertext.size}); + to_byte_span(session_blind15_pk0.data), + to_byte_span(session_blind15_pk1.data), + to_byte_span(encoded.ciphertext.data, encoded.ciphertext.size)); session_protocol_decoded_community_message decoded = session_protocol_decode_for_community( decrypted_cipher.data(), decrypted_cipher.size(), timestamp_s.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + keys.ed_pk1.data(), + keys.ed_pk1.size(), error, sizeof(error)); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; @@ -963,10 +947,12 @@ TEST_CASE("Pro rotating-seed derivation", "[session-protocol][pro][pro_kat]") { // Deterministic BLAKE2b of the Pro master seed and the floored rotation period, so every device // derives the same seed for the same period. Vectors computed independently (Python // hashlib.blake2b, person="ProRotatingSeed_", input = seed || decimal-ASCII(period_start)). - auto master = "0101010101010101010101010101010101010101010101010101010101010101"_hexbytes; + auto master = + oxenc::from_hex("0101010101010101010101010101010101010101010101010101010101010101"); auto seed_hex = [&](int64_t unix_ts) { auto s = ProProof::rotating_seed( - master, std::chrono::sys_seconds{std::chrono::seconds{unix_ts}}); + to_byte_span(master.data(), master.size()), + std::chrono::sys_seconds{std::chrono::seconds{unix_ts}}); return oxenc::to_hex(s.begin(), s.end()); }; diff --git a/tests/test_snode_pool.cpp b/tests/test_snode_pool.cpp index d470ed193..fccc0628b 100644 --- a/tests/test_snode_pool.cpp +++ b/tests/test_snode_pool.cpp @@ -14,17 +14,13 @@ class TestSnodePool : public SnodePool { TestSnodePool( config::SnodePool config, - std::shared_ptr loop, - std::shared_ptr disk_loop, + oxen::quic::Loop& loop, + oxen::quic::Loop& disk_loop, network_fetcher_t direct_fetcher = [](Request, network_response_callback_t) {}) : - SnodePool( - std::move(config), - std::move(loop), - std::move(disk_loop), - std::move(direct_fetcher)) {} + SnodePool(std::move(config), loop, disk_loop, std::move(direct_fetcher)) {} void reset_state_with_cache(std::vector cache) { - _loop->call_get([this, cache] { + _jq.call_get([this, cache] { _snode_cache = cache; _snode_strikes.clear(); }); @@ -37,7 +33,7 @@ class TestSnodePool : public SnodePool { } void debug_queue_post_refresh_callback(std::function cb) { - _loop->call_get([this, cb = std::move(cb)]() mutable { + _jq.call_get([this, cb = std::move(cb)]() mutable { _after_snode_cache_refresh.push_back(std::move(cb)); }); } @@ -54,7 +50,7 @@ class TestSnodePool : public SnodePool { // best-effort. The return value is what actually makes it safe: it confirms the capacity // really is tight rather than letting the test quietly stop exercising the bug. bool debug_remove_post_refresh_callback_spare_capacity() { - return _loop->call_get([this] { + return _jq.call_get([this] { auto exact_sized_copy = _after_snode_cache_refresh; _after_snode_cache_refresh = std::move(exact_sized_copy); return _after_snode_cache_refresh.capacity() == _after_snode_cache_refresh.size(); @@ -62,7 +58,7 @@ class TestSnodePool : public SnodePool { } size_t pending_post_refresh_callbacks() { - return _loop->call_get([this] { return _after_snode_cache_refresh.size(); }); + return _jq.call_get([this] { return _after_snode_cache_refresh.size(); }); } // Called from the test thread, so this also covers `_update_cache` being entered from off the @@ -71,7 +67,7 @@ class TestSnodePool : public SnodePool { void debug_on_refresh_complete(std::vector> raw_results) { auto total_requests = static_cast(raw_results.size()); - _loop->call_get([&] { + _jq.call_get([&] { _on_refresh_complete("test", std::move(raw_results), false, true, total_requests); }); } @@ -79,7 +75,7 @@ class TestSnodePool : public SnodePool { // Encodes nodes the way the storage server returns them, so they can be fed to // `_on_refresh_complete`: 51 bytes per node, all multi-byte fields big-endian -std::vector to_snode_cache_bin(const std::vector& nodes) { +static std::vector to_snode_cache_bin(const std::vector& nodes) { std::vector result; result.reserve(nodes.size() * 51); @@ -90,7 +86,7 @@ std::vector to_snode_cache_bin(const std::vector& nodes for (const auto& node : nodes) { for (auto byte : node.view_remote_key()) - result.push_back(static_cast(byte)); + result.push_back(byte); append(node.swarm_id, 8); append(node.ip.addr, 4); @@ -119,10 +115,10 @@ TEST_CASE("Network", "[network][get_unused_nodes]") { 0, 3, // cache_node_strike_threshold false}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; - auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hexbytes; - auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hexbytes; - auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; + auto ed_pk2 = "5ea34e72bb044654a6a23675690ef5ffaaf1656b02f93fb76655f9cbdbe89876"_hex_b; + auto ed_pk3 = "e17a692033200ae41350df9709754edde7343e2cf2f23e88f993319e0720e5e5"_hex_b; + auto ed_pk4 = "7b633fa6fb462b90db6f0f50384190ce7715e31b7aa93d87dbd7e94e33d4251f"_hex_b; std::vector snode_cache; std::vector unused_nodes; @@ -160,7 +156,7 @@ TEST_CASE("Network", "[network][get_unused_nodes]") { auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); snode_pool->reset_state_with_cache(snode_cache); // Should return a result in a different order (since this is random, it's possible that it @@ -217,7 +213,7 @@ TEST_CASE("Network", "[network][get_unused_nodes]") { 0, 3, // cache_node_strike_threshold false}; - snode_pool = std::make_shared(pool_config, loop, disk_loop); + snode_pool = std::make_shared(pool_config, *loop, *disk_loop); snode_pool->reset_state_with_cache(snode_cache); unused_nodes = snode_pool->get_unused_nodes(20); std::sort(unused_nodes.begin(), unused_nodes.end()); @@ -243,7 +239,7 @@ TEST_CASE("Network", "[network][update_cache]") { 0, 3, // cache_node_strike_threshold false}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; std::vector snode_cache; for (uint16_t i = 0; i < 5; ++i) @@ -257,7 +253,7 @@ TEST_CASE("Network", "[network][update_cache]") { auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); // Should tolerate a post-refresh callback registering another post-refresh callback (which is // what a deferred `get_swarm` does when the refresh left the cache empty) rather than @@ -299,7 +295,7 @@ TEST_CASE("Network", "[network][refresh_min_cache_size]") { 0, 3, // cache_node_strike_threshold false}; - auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hexbytes; + auto ed_pk = "4cb76fdc6d32278e3f83dbf608360ecc6b65727934b85d2fb86862ff98c46ab7"_hex_b; std::vector snode_cache; for (uint16_t i = 0; i < 20; ++i) @@ -313,7 +309,7 @@ TEST_CASE("Network", "[network][refresh_min_cache_size]") { auto loop = std::make_shared(); auto disk_loop = std::make_shared(); - auto snode_pool = std::make_shared(pool_config, loop, disk_loop); + auto snode_pool = std::make_shared(pool_config, *loop, *disk_loop); snode_pool->reset_state_with_cache(snode_cache); REQUIRE(snode_pool->size() == 20); diff --git a/tests/test_sqlite_bind.cpp b/tests/test_sqlite_bind.cpp new file mode 100644 index 000000000..51ab92b32 --- /dev/null +++ b/tests/test_sqlite_bind.cpp @@ -0,0 +1,110 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace session; +using namespace session::literals; + +namespace { + +/// A throwaway unencrypted database with one table, for exercising the bind helpers against real +/// SQLite rather than a mock: what is being tested is which parameter a value lands on, and only +/// SQLite can answer that. +struct TempDb { + std::filesystem::path path; + sqlite::Database db; + + TempDb() : + path{std::filesystem::temp_directory_path() / + "{}.db"_format(random::unique_id("test_sqlite", 7))}, + db{path} { + auto c = db.conn(); + c.sql.exec("CREATE TABLE t (id INTEGER PRIMARY KEY NOT NULL, name TEXT) STRICT"); + } + + ~TempDb() { + std::error_code ec; + std::filesystem::remove(path, ec); + std::filesystem::remove(path.string() + "-wal", ec); + std::filesystem::remove(path.string() + "-shm", ec); + } +}; + +} // namespace + +TEST_CASE("sqlite - bind_each", "[sqlite][bind]") { + TempDb t; + auto c = t.db.conn(); + for (int i = 1; i <= 6; i++) + c.prepared_exec("INSERT INTO t (id, name) VALUES (?, ?)", i, "row{}"_format(i)); + + auto ids_in = [&](const auto& query, const auto&... bind) { + std::vector got; + for (auto&& id : c.prepared_results(query, bind...)) + got.push_back(id); + return got; + }; + + SECTION("binds a container across a variable-length IN list") { + std::vector want{2, 4, 5}; + CHECK(ids_in("SELECT id FROM t WHERE id IN ({}) ORDER BY id"_format( + sqlite::placeholders(want.size())), + sqlite::bind_each{want}) == want); + } + + SECTION("a single element is not a special case") { + std::vector want{3}; + CHECK(ids_in("SELECT id FROM t WHERE id IN ({}) ORDER BY id"_format( + sqlite::placeholders(want.size())), + sqlite::bind_each{want}) == want); + } + + SECTION("the parameters around it number from where it leaves off") { + // The point of the running counter: `hi` is the 5th parameter because the sequence consumed + // three, not the 3rd because it is the third argument. + std::vector some{1, 2, 6}; + auto got = + ids_in("SELECT id FROM t WHERE id > ? AND id IN ({}) AND id < ? ORDER BY id"_format( + sqlite::placeholders(some.size())), + 1, + sqlite::bind_each{some}, + 6); + CHECK(got == std::vector{2}); + } + + SECTION("more than one sequence in the same call") { + std::vector lo{1, 2, 3}, hi{3, 4, 5}; + auto got = + ids_in("SELECT id FROM t WHERE id IN ({}) AND id IN ({}) ORDER BY id"_format( + sqlite::placeholders(lo.size()), sqlite::placeholders(hi.size())), + sqlite::bind_each{lo}, + sqlite::bind_each{hi}); + CHECK(got == std::vector{3}); + } + + SECTION("an iterator pair binds part of a container") { + std::vector all{2, 4, 5, 6}; + auto got = ids_in( + "SELECT id FROM t WHERE id IN ({}) ORDER BY id"_format(sqlite::placeholders(2)), + sqlite::bind_each{all.begin(), all.begin() + 2}); + CHECK(got == std::vector{2, 4}); + } + + SECTION("elements bind by type, not as blobs") { + // Strings go through the same bind_oneshot_single as anywhere else, so a sequence of them + // matches TEXT rather than arriving as something SQLite compares unequal to everything. + std::vector names{"row2", "row5"}; + std::vector got; + for (auto&& id : c.prepared_results( + "SELECT id FROM t WHERE name IN ({}) ORDER BY id"_format( + sqlite::placeholders(names.size())), + sqlite::bind_each{names})) + got.push_back(id); + CHECK(got == std::vector{2, 5}); + } +} diff --git a/tests/test_swarm_retry.cpp b/tests/test_swarm_retry.cpp new file mode 100644 index 000000000..1a07419d8 --- /dev/null +++ b/tests/test_swarm_retry.cpp @@ -0,0 +1,219 @@ +#include +#include +#include +#include + +#include "test_helper.hpp" + +using namespace session; +using namespace session::network; +using namespace std::literals; + +namespace { + +/// A swarm member. Only the pubkey distinguishes them here; the addresses are never dialled, +/// because FakeRouter answers without going anywhere. +std::string key_hex(uint8_t n) { + return fmt::format("{:02x}{}", n, std::string(62, '0')); +} + +service_node node_at(uint8_t n) { + return service_node{ + ed25519_pubkey::from_hex(key_hex(n)), + oxen::quic::ipv4{127, 0, 0, 1}, + static_cast(1000 + n), + static_cast(2000 + n), + {2, 8, 0}, + 0, + 0}; +} + +/// A Network with its router replaced and one swarm primed, which is the least a test needs to +/// exercise anything Network does above routing. +struct ScriptedNetwork { + std::shared_ptr net; + std::shared_ptr router = std::make_shared(); + x25519_pubkey swarm_pubkey; + std::vector swarm; + + explicit ScriptedNetwork(size_t members) { + net = std::make_shared(network::config::Config{}); + swarm_pubkey = x25519_pubkey::from_hex(key_hex(0xAA)); + + for (size_t i = 0; i < members; i++) + swarm.push_back(node_at(static_cast(i + 1))); + + TestHelper::set_router(*net, router); + TestHelper::seed_swarm(TestHelper::snode_pool(*net), swarm_pubkey, swarm); + } + + /// A request addressed to the swarm, starting at whichever member the caller would have picked. + Request to(const service_node& first, std::optional overall = 60s) { + Request req{first, "store", std::vector{}, RequestCategory::standard_small, 10s}; + req.swarm_pubkey = swarm_pubkey; + req.overall_timeout = overall; + return req; + } + + /// The answer is delivered from the loop, not from send_request, so this waits for it. The + /// promise is shared rather than captured by reference: if the callback never comes, a + /// reference to a local here would dangle rather than merely time out. + std::pair send(Request req) { + auto done = std::make_shared>>(); + auto waiter = done->get_future(); + net->send_request(std::move(req), [done](bool ok, bool, int16_t status, auto, auto) { + done->set_value({ok, status}); + }); + REQUIRE(waiter.wait_for(5s) == std::future_status::ready); + return waiter.get(); + } +}; + +} // namespace + +/// Whether every entry is distinct -- what "once per node" means, given get_swarm hands members +/// back in a shuffled order rather than a fixed one. +bool all_distinct(const std::vector& tried) { + auto sorted = tried; + std::ranges::sort(sorted, [](const auto& a, const auto& b) { return a.hex() < b.hex(); }); + return std::ranges::adjacent_find(sorted) == sorted.end(); +} + +TEST_CASE("Network: an unreachable node moves the request to the next swarm member", "[network]") { + ScriptedNetwork n{4}; + + // Only one member participates in session routing; the rest have no relay contact. + n.router->replies[n.swarm[2].remote_pubkey] = {}; + + auto [ok, status] = n.send(n.to(n.swarm[0])); + CHECK(ok); + CHECK(status == 200); + + // It reached the one that works, having spent no member twice on the way. Which members it + // tried first is not asserted: get_swarm shuffles, so the order is deliberately not fixed. + REQUIRE(n.router->tried.size() >= 2); + CHECK(n.router->tried.size() <= n.swarm.size()); + CHECK(n.router->tried.back() == n.swarm[2].remote_pubkey); + CHECK(all_distinct(n.router->tried)); +} + +TEST_CASE("Network: running out of swarm members reports the original failure", "[network]") { + ScriptedNetwork n{3}; + // Nobody answers. + + auto [ok, status] = n.send(n.to(n.swarm[0])); + CHECK_FALSE(ok); + // The reason each member was unusable, not "no members left" -- which would tell the caller + // less than what it already had. + CHECK(status == ERROR_INVALID_DESTINATION); + + // Every member tried, once each: it ends when selection has nothing left rather than at a + // fixed count, and never revisits one already spent. + REQUIRE(n.router->tried.size() == 3); + CHECK(all_distinct(n.router->tried)); +} + +TEST_CASE("Network: a failure that is not the node's fault is not retried elsewhere", "[network]") { + ScriptedNetwork n{3}; + + // A 500 says the request was carried and the server disliked it. Asking a different member of + // the same swarm the same question gets the same answer, so this is not what the walk is for. + n.router->replies[n.swarm[0].remote_pubkey] = {false, false, 500, "nope"}; + + auto [ok, status] = n.send(n.to(n.swarm[0])); + CHECK_FALSE(ok); + CHECK(status == 500); + CHECK(n.router->tried.size() == 1); +} + +TEST_CASE("Network: a request with no swarm has nowhere else to go", "[network]") { + ScriptedNetwork n{3}; + + // Something aimed at a node rather than at an account -- a cache refresh, a clock resync -- + // has no swarm to walk, so the failure is simply reported. + auto req = n.to(n.swarm[0]); + req.swarm_pubkey.reset(); + + auto [ok, status] = n.send(std::move(req)); + CHECK_FALSE(ok); + CHECK(status == ERROR_INVALID_DESTINATION); + CHECK(n.router->tried.size() == 1); +} + +TEST_CASE("Network: attempts are bounded by the overall budget", "[network]") { + SECTION("each attempt gets the per-request timeout while there is budget for it") { + ScriptedNetwork n{3}; + n.send(n.to(n.swarm[0], 60s)); + + REQUIRE(n.router->timeouts.size() == 3); + for (auto t : n.router->timeouts) + CHECK(t == 10s); + } + + SECTION("a shrinking budget shortens the retry rather than overrunning it") { + ScriptedNetwork n{3}; + // Less than one full attempt's worth of budget, but more than the minimum worth starting. + n.send(n.to(n.swarm[0], 6s)); + + REQUIRE(n.router->timeouts.size() >= 2); + // The first attempt is the caller's own request, untouched -- the budget only governs what + // this layer *adds*. + CHECK(n.router->timeouts[0] == 10s); + // Every retry after it is capped by what remains of the operation. + for (size_t i = 1; i < n.router->timeouts.size(); i++) + CHECK(n.router->timeouts[i] <= 6s); + } + + SECTION("too little left to be worth starting stops the walk early") { + ScriptedNetwork n{4}; + // Below MIN_RETRY_BUDGET, so the first failure ends it rather than starting an attempt + // that cannot finish. + n.send(n.to(n.swarm[0], 1s)); + + CHECK(n.router->tried.size() == 1); + } +} + +TEST_CASE( + "Network: an owner reference dropped mid-callback does not tear the Network down from its " + "own loop", + "[network]") { + // Two members so that the first, unreachable one sends the request through + // _retry_next_swarm_node: that goes via SnodePool::get_swarm, which answers from the loop, so + // the second attempt -- and the callback below -- run on the loop thread rather than on this + // one. + ScriptedNetwork n{2}; + n.router->replies[n.swarm[1].remote_pubkey] = {}; + + auto reached_callback = std::promise{}; + auto in_callback = reached_callback.get_future(); + std::atomic answered = false; + + n.net->send_request( + n.to(n.swarm[0]), [&reached_callback, &answered](bool ok, bool, int16_t, auto, auto) { + answered = ok; + reached_callback.set_value(); + + // Stay on the loop thread while the reference below goes, which is the interleaving + // that used to abort: the callback held a shared_ptr of its own, so + // dropping the owner's left the loop thread as the last owner, and ~Network joins + // that thread. + std::this_thread::sleep_for(50ms); + }); + + REQUIRE(in_callback.wait_for(5s) == std::future_status::ready); + + auto observer = std::weak_ptr{n.net}; + n.net.reset(); + + // Waits for the Network to actually be gone rather than merely unreferenced from here: the + // teardown is what fails, so it has to happen while this test is still running. Nothing else + // holds a reference, so this returns as soon as the callback has finished. + for (int i = 0; i < 500 && !observer.expired(); i++) + std::this_thread::sleep_for(10ms); + + // Surviving to here is the assertion: the failure was an abort out of a destructor rather than + // a wrong answer. + CHECK(observer.expired()); + CHECK(answered); +} diff --git a/tests/test_utils.cpp b/tests/test_utils.cpp index 5db8927e9..572229e11 100644 --- a/tests/test_utils.cpp +++ b/tests/test_utils.cpp @@ -1,4 +1,5 @@ #include +#include #include "utils.hpp" @@ -24,4 +25,36 @@ TEST_CASE("Network", "[network][parse_url]") { CHECK(path2.value_or("NULL") == "/test/123456"); CHECK(path3.value_or("NULL") == "NULL"); CHECK(path4.value_or("NULL") == "/test?value=test"); +} + +TEST_CASE("from_epoch helpers are the inverse of epoch_seconds/epoch_ms", "[clock]") { + using namespace std::chrono; + using namespace session; + + // Round-trip through epoch_seconds / from_epoch_s + auto t_s = clock_now_s(); + int64_t count_s = epoch_seconds(t_s); + auto t_s2 = from_epoch_s(count_s); + CHECK(t_s == t_s2); + + // Round-trip through epoch_ms / from_epoch_ms + auto t_ms = clock_now_ms(); + int64_t count_ms = epoch_ms(t_ms); + auto t_ms2 = from_epoch_ms(count_ms); + CHECK(t_ms == t_ms2); + + // Generic from_epoch with seconds precision + int64_t unix_s = 1'700'000'000; + auto tp_s = from_epoch_s(unix_s); + CHECK(epoch_seconds(tp_s) == unix_s); + + // Generic from_epoch with milliseconds precision + int64_t unix_ms = 1'700'000'000'000LL; + auto tp_ms = from_epoch_ms(unix_ms); + CHECK(epoch_ms(tp_ms) == unix_ms); + + // from_epoch template returns sys_time + auto tp_generic = from_epoch(unix_s); + static_assert(std::same_as); + CHECK(epoch_seconds(tp_generic) == unix_s); } \ No newline at end of file diff --git a/tests/test_xed25519.cpp b/tests/test_xed25519.cpp index b847cb303..365d6b846 100644 --- a/tests/test_xed25519.cpp +++ b/tests/test_xed25519.cpp @@ -1,128 +1,89 @@ #include -#include -#include #include #include +#include "session/crypto/ed25519.hpp" #include "session/util.hpp" #include "session/xed25519.h" #include "session/xed25519.hpp" -constexpr std::array seed1{ - 0xfe, 0xcd, 0x9a, 0x60, 0x34, 0xbc, 0x9a, 0xba, 0x27, 0x39, 0x25, 0xde, 0xe7, - 0x06, 0x2b, 0x12, 0x33, 0x34, 0x58, 0x7c, 0x3c, 0x62, 0x57, 0x34, 0x1a, 0xfa, - 0xe2, 0xd7, 0xfe, 0x85, 0xe1, 0x22, 0xf4, 0xef, 0x87, 0x39, 0x08, 0xf6, 0xa5, - 0x37, 0x7b, 0xa3, 0x85, 0x3f, 0x0e, 0x2f, 0xa3, 0x26, 0xee, 0xd9, 0xe7, 0x41, - 0xed, 0xf9, 0xf7, 0xd0, 0x31, 0x1a, 0x3e, 0xcc, 0x66, 0xa5, 0x7b, 0x32}; -constexpr std::array seed2{ - 0x86, 0x59, 0xef, 0xdc, 0xbe, 0x09, 0x49, 0xe0, 0xf8, 0x11, 0x41, 0xe6, 0xd3, - 0x97, 0xe8, 0xbe, 0x75, 0xf4, 0x5d, 0x09, 0x26, 0x2f, 0x20, 0x9d, 0x59, 0x50, - 0xe9, 0x79, 0x89, 0xeb, 0x43, 0xc7, 0x35, 0x70, 0xb6, 0x9a, 0x47, 0xdc, 0x09, - 0x45, 0x44, 0xc1, 0xc5, 0x08, 0x9c, 0x40, 0x41, 0x4b, 0xbd, 0xa1, 0xff, 0xdd, - 0xe8, 0xaa, 0xb2, 0x61, 0x7f, 0xe9, 0x37, 0xee, 0x74, 0xa5, 0xee, 0x81}; - -constexpr std::span pub1{seed1.data() + 32, 32}; -constexpr std::span pub2{seed2.data() + 32, 32}; - -constexpr std::array xpub1{ - 0xfe, 0x94, 0xb7, 0xad, 0x4b, 0x7f, 0x1c, 0xc1, 0xbb, 0x92, 0x67, - 0x1f, 0x1f, 0x0d, 0x24, 0x3f, 0x22, 0x6e, 0x11, 0x5b, 0x33, 0x77, - 0x04, 0x65, 0xe8, 0x2b, 0x50, 0x3f, 0xc3, 0xe9, 0x6e, 0x1f, -}; -constexpr std::array xpub2{ - 0x05, 0xc9, 0xa9, 0xbf, 0x17, 0x8f, 0xa6, 0x44, 0xd4, 0x4b, 0xeb, - 0xf6, 0x28, 0x71, 0x6d, 0xc7, 0xf2, 0xdf, 0x3d, 0x08, 0x42, 0xe9, - 0x78, 0x81, 0x96, 0x2c, 0x72, 0x36, 0x99, 0x15, 0x20, 0x73, -}; - -constexpr std::array pub2_abs{ - 0x35, 0x70, 0xb6, 0x9a, 0x47, 0xdc, 0x09, 0x45, 0x44, 0xc1, 0xc5, - 0x08, 0x9c, 0x40, 0x41, 0x4b, 0xbd, 0xa1, 0xff, 0xdd, 0xe8, 0xaa, - 0xb2, 0x61, 0x7f, 0xe9, 0x37, 0xee, 0x74, 0xa5, 0xee, 0x01, -}; - -template -static std::string view_hex(const std::array& x) { - return oxenc::to_hex(session::to_span(x)); -} +using namespace session; +using namespace session::literals; + +// Full 64-byte libsodium-style Ed25519 keys (32-byte seed || 32-byte pubkey) +constexpr auto seed1 = + "fecd9a6034bc9aba273925dee7062b123334587c3c6257341afae2d7fe85e122" + "f4ef873908f6a5377ba3853f0e2fa326eed9e741edf9f7d0311a3ecc66a57b32"_hex_b; +constexpr auto seed2 = + "8659efdcbe0949e0f81141e6d397e8be75f45d09262f209d5950e97989eb43c7" + "3570b69a47dc094544c1c5089c40414bbda1ffdde8aab2617fe937ee74a5ee81"_hex_b; + +// Ed25519 pubkeys (second half of the seed arrays) +constexpr auto pub1 = seed1.last<32>(); +constexpr auto pub2 = seed2.last<32>(); + +// Expected X25519 pubkeys derived from the Ed25519 pubkeys +constexpr auto xpub1 = "fe94b7ad4b7f1cc1bb92671f1f0d243f226e115b33770465e82b503fc3e96e1f"_hex_b; +constexpr auto xpub2 = "05c9a9bf178fa644d44bebf628716dc7f2df3d0842e97881962c723699152073"_hex_b; + +// The "absolute" (positive) version of pub2's Ed25519 pubkey +constexpr auto pub2_abs = "3570b69a47dc094544c1c5089c40414bbda1ffdde8aab2617fe937ee74a5ee01"_hex_b; TEST_CASE("XEd25519 pubkey conversion", "[xed25519][pubkey]") { - std::array xpk1; - int rc = crypto_sign_ed25519_pk_to_curve25519(xpk1.data(), pub1.data()); - REQUIRE(rc == 0); - REQUIRE(view_hex(xpk1) == view_hex(xpub1)); + auto xpk1 = ed25519::pk_to_x25519(pub1); + REQUIRE(oxenc::to_hex(xpk1) == oxenc::to_hex(xpub1)); - std::array xpk2; - rc = crypto_sign_ed25519_pk_to_curve25519(xpk2.data(), pub2.data()); - REQUIRE(rc == 0); - REQUIRE(view_hex(xpk2) == view_hex(xpub2)); + auto xpk2 = ed25519::pk_to_x25519(pub2); + REQUIRE(oxenc::to_hex(xpk2) == oxenc::to_hex(xpub2)); - auto xed1 = session::xed25519::pubkey(xpub1); - REQUIRE(view_hex(xed1) == oxenc::to_hex(pub1)); + auto xed1 = xed25519::pubkey(xpub1); + REQUIRE(oxenc::to_hex(xed1) == oxenc::to_hex(pub1)); // This one fails because the original Ed pubkey is negative - auto xed2 = session::xed25519::pubkey(xpub2); - REQUIRE(view_hex(xed2) != oxenc::to_hex(pub2)); + auto xed2 = xed25519::pubkey(xpub2); + REQUIRE(oxenc::to_hex(xed2) != oxenc::to_hex(pub2)); // After making the xed negative we should be okay: - xed2[31] |= 0x80; - REQUIRE(view_hex(xed2) == oxenc::to_hex(pub2)); + xed2[31] |= std::byte{0x80}; + REQUIRE(oxenc::to_hex(xed2) == oxenc::to_hex(pub2)); } TEST_CASE("XEd25519 signing", "[xed25519][sign]") { - std::array xsk1; - int rc = crypto_sign_ed25519_sk_to_curve25519(xsk1.data(), seed1.data()); - REQUIRE(rc == 0); - std::array xpk1; - rc = crypto_sign_ed25519_pk_to_curve25519(xpk1.data(), pub1.data()); - - std::array xsk2; - rc = crypto_sign_ed25519_sk_to_curve25519(xsk2.data(), seed2.data()); - REQUIRE(rc == 0); - std::array xpk2; - rc = crypto_sign_ed25519_pk_to_curve25519(xpk2.data(), pub2.data()); + auto xsk1 = ed25519::sk_to_x25519(ed25519::PrivKeySpan{seed1}); + auto xsk2 = ed25519::sk_to_x25519(seed2.first<32>()); - const auto msg = session::to_span("hello world"); + const auto msg = "hello world"_bytes; - auto xed_sig1 = session::xed25519::sign(xsk1, msg); + auto xed_sig1 = xed25519::sign(xsk1, msg); - rc = crypto_sign_ed25519_verify_detached(xed_sig1.data(), msg.data(), msg.size(), pub1.data()); - REQUIRE(rc == 0); + REQUIRE(ed25519::verify(xed_sig1, pub1, msg)); - auto xed_sig2 = session::xed25519::sign(xsk2, msg); + auto xed_sig2 = xed25519::sign(xsk2, msg); // This one will fail, because Xed signing always uses the positive but our actual pub2 is the // negative: - rc = crypto_sign_ed25519_verify_detached(xed_sig2.data(), msg.data(), msg.size(), pub2.data()); - REQUIRE(rc != 0); + REQUIRE_FALSE(ed25519::verify(xed_sig2, pub2, msg)); // Flip it, though, and it should work: - rc = crypto_sign_ed25519_verify_detached( - xed_sig2.data(), msg.data(), msg.size(), pub2_abs.data()); - REQUIRE(rc == 0); + REQUIRE(ed25519::verify(xed_sig2, pub2_abs, msg)); } TEST_CASE("XEd25519 verification", "[xed25519][verify]") { - std::array xsk1; - int rc = crypto_sign_ed25519_sk_to_curve25519(xsk1.data(), seed1.data()); - REQUIRE(rc == 0); - - std::array xsk2; - rc = crypto_sign_ed25519_sk_to_curve25519(xsk2.data(), seed2.data()); - REQUIRE(rc == 0); + auto xsk1 = ed25519::sk_to_x25519(ed25519::PrivKeySpan{seed1}); + auto xsk2 = ed25519::sk_to_x25519(seed2.first<32>()); - const auto msg = session::to_span("hello world"); + const auto msg = "hello world"_bytes; - auto xed_sig1 = session::xed25519::sign(xsk1, msg); - auto xed_sig2 = session::xed25519::sign(xsk2, msg); + auto xed_sig1 = xed25519::sign(xsk1, msg); + auto xed_sig2 = xed25519::sign(xsk2, msg); - REQUIRE(session::xed25519::verify(xed_sig1, xpub1, msg)); - REQUIRE(session::xed25519::verify(xed_sig2, xpub2, msg)); + REQUIRE(xed25519::verify(xed_sig1, xpub1, msg)); + REQUIRE(xed25519::verify(xed_sig2, xpub2, msg)); // Unlike regular Ed25519, XEd25519 uses randomness in the signature, so signing the same value // a second should give us a different signature: - auto xed_sig1b = session::xed25519::sign(xsk1, msg); - REQUIRE(view_hex(xed_sig1b) != view_hex(xed_sig1)); + auto xed_sig1b = xed25519::sign(xsk1, msg); + REQUIRE(oxenc::to_hex(xed_sig1b) != oxenc::to_hex(xed_sig1)); } TEST_CASE("XEd25519 string overloads reject invalid input sizes", "[xed25519]") { @@ -133,77 +94,100 @@ TEST_CASE("XEd25519 string overloads reject invalid input sizes", "[xed25519]") std::string signature(64, '\0'); std::string long_signature(65, '\0'); - CHECK_THROWS_AS(session::xed25519::sign(short_key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS(session::xed25519::sign(long_key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::sign(short_key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::sign(long_key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS( - session::xed25519::verify(short_signature, key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS( - session::xed25519::verify(long_signature, key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS( - session::xed25519::verify(signature, short_key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS( - session::xed25519::verify(signature, long_key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::verify(short_signature, key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::verify(long_signature, key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::verify(signature, short_key, "hello world"), std::invalid_argument); + CHECK_THROWS_AS(xed25519::verify(signature, long_key, "hello world"), std::invalid_argument); - CHECK_THROWS_AS(session::xed25519::pubkey(short_key), std::invalid_argument); - CHECK_THROWS_AS(session::xed25519::pubkey(long_key), std::invalid_argument); + CHECK_THROWS_AS(xed25519::pubkey(short_key), std::invalid_argument); + CHECK_THROWS_AS(xed25519::pubkey(long_key), std::invalid_argument); } TEST_CASE("XEd25519 pubkey conversion (C wrapper)", "[xed25519][pubkey][c]") { - auto xed1 = session::xed25519::pubkey(xpub1); - REQUIRE(view_hex(xed1) == oxenc::to_hex(pub1)); + auto xed1 = xed25519::pubkey(xpub1); + REQUIRE(oxenc::to_hex(xed1) == oxenc::to_hex(pub1)); // This one fails because the original Ed pubkey is negative - auto xed2 = session::xed25519::pubkey(xpub2); - REQUIRE(view_hex(xed2) != oxenc::to_hex(pub2)); + auto xed2 = xed25519::pubkey(xpub2); + REQUIRE(oxenc::to_hex(xed2) != oxenc::to_hex(pub2)); // After making the xed negative we should be okay: - xed2[31] |= 0x80; - REQUIRE(view_hex(xed2) == oxenc::to_hex(pub2)); + xed2[31] |= std::byte{0x80}; + REQUIRE(oxenc::to_hex(xed2) == oxenc::to_hex(pub2)); } + TEST_CASE("XEd25519 signing (C wrapper)", "[xed25519][sign][c]") { - std::array xsk1; - int rc = crypto_sign_ed25519_sk_to_curve25519(xsk1.data(), seed1.data()); - REQUIRE(rc == 0); - std::array xpk1; - rc = crypto_sign_ed25519_pk_to_curve25519(xpk1.data(), pub1.data()); + auto xsk1 = ed25519::sk_to_x25519(ed25519::PrivKeySpan{seed1}); + auto xsk2 = ed25519::sk_to_x25519(seed2.first<32>()); + + const auto msg = "hello world"_bytes; + + b64 xed_sig1, xed_sig2; + REQUIRE(session_xed25519_sign( + to_unsigned(xed_sig1.data()), + to_unsigned(xsk1.data()), + to_unsigned(msg.data()), + msg.size())); + REQUIRE(session_xed25519_sign( + to_unsigned(xed_sig2.data()), + to_unsigned(xsk2.data()), + to_unsigned(msg.data()), + msg.size())); + + REQUIRE(ed25519::verify(xed_sig1, pub1, msg)); + REQUIRE_FALSE(ed25519::verify(xed_sig2, pub2, msg)); // Failure expected (pub2 is negative) + REQUIRE(ed25519::verify(xed_sig2, pub2_abs, msg)); // Flipped sign should work +} - std::array xsk2; - rc = crypto_sign_ed25519_sk_to_curve25519(xsk2.data(), seed2.data()); - REQUIRE(rc == 0); - std::array xpk2; - rc = crypto_sign_ed25519_pk_to_curve25519(xpk2.data(), pub2.data()); +TEST_CASE("XEd25519 std::byte overloads", "[xed25519][byte]") { + auto xsk1 = ed25519::sk_to_x25519(ed25519::PrivKeySpan{seed1}); - const auto msg = session::to_span("hello world"); + const auto msg = "hello world"_bytes; - std::array xed_sig1, xed_sig2; - REQUIRE(session_xed25519_sign(xed_sig1.data(), xsk1.data(), msg.data(), msg.size())); - REQUIRE(session_xed25519_sign(xed_sig2.data(), xsk2.data(), msg.data(), msg.size())); + // sign() byte overload should return a std::byte array. + auto sig_b = xed25519::sign(std::span{xsk1}, msg); + static_assert(std::same_as>); - rc = crypto_sign_ed25519_verify_detached(xed_sig1.data(), msg.data(), msg.size(), pub1.data()); - REQUIRE(rc == 0); + // The signature must verify via the ed25519 helper. + REQUIRE(ed25519::verify(sig_b, pub1, msg)); - rc = crypto_sign_ed25519_verify_detached(xed_sig2.data(), msg.data(), msg.size(), pub2.data()); - REQUIRE(rc != 0); // Failure expected (pub2 is negative) + // verify() byte overload. + REQUIRE(xed25519::verify(sig_b, xpub1, msg)); - rc = crypto_sign_ed25519_verify_detached( - xed_sig2.data(), msg.data(), msg.size(), pub2_abs.data()); - REQUIRE(rc == 0); // Flipped sign should work + // pubkey() byte overload should return a std::byte array. + auto ed_pk_b = xed25519::pubkey(xpub1); + static_assert(std::same_as>); + REQUIRE(oxenc::to_hex(ed_pk_b) == oxenc::to_hex(pub1)); } -TEST_CASE("XEd25519 verification (C wrapper)", "[xed25519][verify][c]") { - std::array xsk1; - int rc = crypto_sign_ed25519_sk_to_curve25519(xsk1.data(), seed1.data()); - REQUIRE(rc == 0); - - std::array xsk2; - rc = crypto_sign_ed25519_sk_to_curve25519(xsk2.data(), seed2.data()); - REQUIRE(rc == 0); - const auto msg = session::to_span("hello world"); - - std::array xed_sig1, xed_sig2; - REQUIRE(session_xed25519_sign(xed_sig1.data(), xsk1.data(), msg.data(), msg.size())); - REQUIRE(session_xed25519_sign(xed_sig2.data(), xsk2.data(), msg.data(), msg.size())); - - REQUIRE(session_xed25519_verify(xed_sig1.data(), xpub1.data(), msg.data(), msg.size())); - REQUIRE(session_xed25519_verify(xed_sig2.data(), xpub2.data(), msg.data(), msg.size())); +TEST_CASE("XEd25519 verification (C wrapper)", "[xed25519][verify][c]") { + auto xsk1 = ed25519::sk_to_x25519(ed25519::PrivKeySpan{seed1}); + auto xsk2 = ed25519::sk_to_x25519(seed2.first<32>()); + + const auto msg = "hello world"_bytes; + + b64 xed_sig1, xed_sig2; + REQUIRE(session_xed25519_sign( + to_unsigned(xed_sig1.data()), + to_unsigned(xsk1.data()), + to_unsigned(msg.data()), + msg.size())); + REQUIRE(session_xed25519_sign( + to_unsigned(xed_sig2.data()), + to_unsigned(xsk2.data()), + to_unsigned(msg.data()), + msg.size())); + + REQUIRE(session_xed25519_verify( + to_unsigned(xed_sig1.data()), + to_unsigned(xpub1.data()), + to_unsigned(msg.data()), + msg.size())); + REQUIRE(session_xed25519_verify( + to_unsigned(xed_sig2.data()), + to_unsigned(xpub2.data()), + to_unsigned(msg.data()), + msg.size())); } diff --git a/tests/utils.hpp b/tests/utils.hpp index 047a3ba77..c2c109a19 100644 --- a/tests/utils.hpp +++ b/tests/utils.hpp @@ -5,20 +5,40 @@ #include #include +#include #include +#include #include #include #include #include #include +#include #include +#include "session/clock.hpp" #include "session/types.hpp" #include "session/util.hpp" +// RAII helper that saves the current AdjustedClock offset, installs a new one on construction, +// and restores the prior offset on destruction. +struct ScopedClockOffset { + explicit ScopedClockOffset(session::AdjustedClock::duration new_offset) : + _saved{session::AdjustedClock::get_offset()} { + session::AdjustedClock::set_offset(new_offset); + } + ~ScopedClockOffset() { session::AdjustedClock::set_offset(_saved); } + ScopedClockOffset(const ScopedClockOffset&) = delete; + ScopedClockOffset& operator=(const ScopedClockOffset&) = delete; + + private: + session::AdjustedClock::duration _saved; +}; + using namespace std::literals; using namespace oxenc::literals; using namespace oxen::log::literals; +using namespace session; namespace session { @@ -129,17 +149,8 @@ class CallTracker { } // namespace session -inline std::vector operator""_bytes(const char* x, size_t n) { - auto begin = reinterpret_cast(x); - return {begin, begin + n}; -} -inline std::vector operator""_hexbytes(const char* x, size_t n) { - std::vector bytes; - oxenc::from_hex(x, x + n, std::back_inserter(bytes)); - return bytes; -} - -inline std::string to_hex(std::vector bytes) { +template +inline std::string to_hex(const Container& bytes) { std::string hex; oxenc::to_hex(bytes.begin(), bytes.end(), std::back_inserter(hex)); return hex; @@ -171,16 +182,20 @@ inline int64_t get_timestamp_us() { .count(); } -inline std::string printable(std::span x) { +inline std::string printable(std::span x) { std::string p; - for (auto c : x) { + for (auto b : x) { + auto c = static_cast(b); if (c >= 0x20 && c <= 0x7e) - p += c; + p += static_cast(c); else p += "\\x" + oxenc::to_hex(&c, &c + 1); } return p; } +inline std::string printable(std::span x) { + return printable(session::as_span(x)); +} inline std::string printable(std::string_view x) { return printable(session::to_span(x)); } @@ -191,7 +206,7 @@ inline std::string printable(fmt::format_string format, T&&... args) { } std::string printable(const unsigned char* x) = delete; inline std::string printable(const unsigned char* x, size_t n) { - return printable({x, n}); + return printable(std::span{x, n}); } template @@ -205,17 +220,17 @@ std::set> make_set(T&&... args) { } struct TestKeys { - session::array_uc32 seed0; - session::array_uc64 ed_sk0; - session::array_uc32 ed_pk0; - session::array_uc32 curve_pk0; - session::array_uc33 session_pk0; - - session::array_uc32 seed1; - session::array_uc64 ed_sk1; - session::array_uc32 ed_pk1; - session::array_uc32 curve_pk1; - session::array_uc33 session_pk1; + session::uc32 seed0; + session::uc64 ed_sk0; + session::uc32 ed_pk0; + session::uc32 curve_pk0; + session::uc33 session_pk0; + + session::uc32 seed1; + session::uc64 ed_sk1; + session::uc32 ed_pk1; + session::uc32 curve_pk1; + session::uc33 session_pk1; }; static inline TestKeys get_deterministic_test_keys() { @@ -225,7 +240,7 @@ static inline TestKeys get_deterministic_test_keys() { // Key 0 { // Seed - auto seed0 = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hexbytes; + auto seed0 = "0123456789abcdef0123456789abcdef00000000000000000000000000000000"_hex_b; std::memcpy(result.seed0.data(), seed0.data(), seed0.size()); // Ed25519 @@ -243,7 +258,7 @@ static inline TestKeys get_deterministic_test_keys() { // Key 1 { // Seed - auto seed1 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hexbytes; + auto seed1 = "00112233445566778899aabbccddeeff00000000000000000000000000000000"_hex_b; std::memcpy(result.seed1.data(), seed1.data(), seed1.size()); // Ed25519 @@ -279,3 +294,84 @@ struct scope_exit { cleanup(); } }; + +// ── Async/callback helpers (adapted from oxen-libquic/tests/utils.hpp) ──────────────────────── + +template +struct functional_helper : public functional_helper {}; +template +struct functional_helper { + using return_type = Ret; + static constexpr bool is_void = std::is_void_v; + using type = std::function; +}; +template +using functional_helper_t = typename functional_helper::type; + +struct set_on_exit { + std::promise& p; + explicit set_on_exit(std::promise& p) : p{p} {} + ~set_on_exit() { p.set_value(); } +}; + +/// Wraps a callable in a promise/future pair. When passed as a std::function argument (via +/// implicit conversion), it calls the inner callable and then signals the promise, allowing tests +/// to block until an asynchronous callback fires. +/// +/// Usage: +/// bool got_it = false; +/// callback_waiter waiter{[&got_it](bool x) { got_it = x; }}; +/// async_operation(waiter); // waiter implicitly converts to std::function +/// REQUIRE(waiter.wait()); // blocks up to 5s +/// CHECK(got_it); +template +struct callback_waiter { + using Func_t = functional_helper_t; + + Func_t func; + std::shared_ptr> p{std::make_shared>()}; + std::future f{p->get_future()}; + + explicit callback_waiter(T f) : func{std::move(f)} {} + + [[nodiscard]] bool wait(std::chrono::milliseconds timeout = 5s) { + return f.wait_for(timeout) == std::future_status::ready; + } + + [[nodiscard]] bool is_ready() { return wait(0ms); } + + // Deliberate implicit conversion to std::function<...>: calls the inner callable then signals + // the promise. + operator Func_t() { + return [p = p, func = func](auto&&... args) { + set_on_exit prom_setter{*p}; + return func(std::forward(args)...); + }; + } + + void call() { this->operator Func_t()(); } +}; + +/// Polls a condition, sleeping between checks. Returns the last result of f() as soon as it is +/// truthy, or when the timeout expires (returning the last falsy result). +template Callback> +auto wait_for( + Callback f, + std::chrono::milliseconds timeout = 5s, + std::chrono::milliseconds check_interval = 25ms) { + auto end = std::chrono::steady_clock::now() + timeout; + for (;;) { + auto val = f(); + if (val || std::chrono::steady_clock::now() >= end) + return val; + std::this_thread::sleep_for(check_interval); + } +} + +// require_future(f) — asserts that std::future f becomes ready within 5s. +// require_future(f, timeout) — asserts that f becomes ready within the given timeout. +#define _require_future2(f, timeout) REQUIRE((f).wait_for(timeout) == std::future_status::ready) +#define _require_future1(f) _require_future2((f), 5s) +#define GET_REQUIRE_FUTURE_MACRO(_1, _2, NAME, ...) NAME +#define require_future(...) \ + GET_REQUIRE_FUTURE_MACRO(__VA_ARGS__, _require_future2, _require_future1)(__VA_ARGS__) diff --git a/utils/format.sh b/utils/format.sh index f565ad5f1..bc461afa0 100755 --- a/utils/format.sh +++ b/utils/format.sh @@ -20,7 +20,7 @@ if [ $? -ne 0 ]; then fi cd "$(dirname $0)/../" -readarray -t sources < <(find include proto src tests | grep -E '\.([hc](pp)?)$' | grep -v '\#' | grep -v Catch2 | grep -v -E '\.pb\.(h|cc)$') +readarray -t sources < <(find include proto src tests | grep -E '\.([hc](pp)?)$' | grep -v '\#' | grep -v Catch2 | grep -v -E '\.pb\.(h|cc)$' | grep -v -E 'ip_country/data\.cpp$') if [ "$1" = "verify" ] ; then if [ $($binary --output-replacements-xml "${sources[@]}" | grep '' | wc -l) -ne 0 ] ; then exit 2 diff --git a/utils/test-bigendian.sh b/utils/test-bigendian.sh index 5b0c0ac3e..c7b2463c5 100755 --- a/utils/test-bigendian.sh +++ b/utils/test-bigendian.sh @@ -4,14 +4,16 @@ # then (especially after touching hashing or any wire/config serialization). # # Most of libsession runs only on little-endian hosts, so endian-sensitive byte serialization — the -# Pro signed-digest encoding, config data, protobuf packing — normally never exercises its big-endian -# path. This script builds and runs the test suite inside an emulated big-endian target (s390x) so -# those paths run for real. +# Pro signed-digest encoding (hash::detail::make_hashable's byte-swap branch), config data, protobuf +# packing — normally never exercises its big-endian path. This script builds and runs the test suite +# inside an emulated big-endian target (s390x) so those paths run for real. In particular it exercises +# the [endian] known-answer test (hash::blake2b_pers integer args must be little-endian) on an actual +# big-endian machine, where make_hashable takes its otherwise-never-compiled swap branch. # # session-router (the onion-routing layer) is endian-irrelevant to what we test and drags in the -# heaviest deps, so we build with -DENABLE_NETWORKING_SROUTER=OFF. oxen::quic itself can't be turned -# off (a couple of ungated backend-session test sources include its headers), so it still builds — we -# satisfy it with libngtcp2 + gnutls from apt (nghttp3 is not needed; libquic uses raw QUIC). +# heaviest deps, so we build with -DENABLE_NETWORKING_SROUTER=OFF. This branch has no ENABLE_NETWORKING +# toggle (networking is always required), so oxen::quic still builds regardless — we satisfy it with +# libngtcp2 + gnutls from apt (nghttp3 is not needed; libquic uses raw QUIC). # # Cost: everything runs under qemu-user emulation, so expect a slow build (oxen-libquic is the bulk of # it). Fine for an occasional manual audit; it is deliberately not in CI. @@ -43,10 +45,11 @@ docker run --platform=linux/s390x --rm -v "$PWD:/src" -w /src debian:sid bash -e rm -rf build-bigendian # fresh configure each run; the dir lives in the mounted tree and would otherwise reuse a stale CMake cache cmake -B build-bigendian -G Ninja -DBUILD_STATIC_DEPS=OFF -DENABLE_NETWORKING_SROUTER=OFF -DCMAKE_BUILD_TYPE=Release cmake --build build-bigendian --target testAll - # Run the endian-sensitive tags on this big-endian host. Catch2 exits 0 even when a filter clause - # matches no tests, so capture the output and fail loudly if any tag went unmatched — that guards - # against a tag rename silently shrinking the audit. ([endian] is a pfs-only test, not present here.) - out=$(./build-bigendian/tests/testAll "[hash],[session-protocol],[pro_backend],[config]") + # Run the endian-sensitive tags on this big-endian host. [endian] is the make_hashable little-endian + # known-answer test; the hash / protocol / Pro / config tags also exercise serialization here. Catch2 + # exits 0 even when a filter clause matches no tests, so capture the output and fail loudly if any tag + # went unmatched — that guards against a tag rename silently shrinking the audit. + out=$(./build-bigendian/tests/testAll "[endian],[hash],[session-protocol],[pro_backend],[config]") echo "$out" if echo "$out" | grep -q "No test cases matched"; then echo "ERROR: a filter tag matched no tests — the audit under-ran (tag renamed?)" >&2 diff --git a/utils/update-ip-country-db.py b/utils/update-ip-country-db.py new file mode 100755 index 000000000..6a0214b06 --- /dev/null +++ b/utils/update-ip-country-db.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 + +"""Refreshes the bundled IP-to-country database. + +Downloads a DB-IP Country Lite CSV release and generates src/network/ip_country/data.cpp from it, +which is the translation unit compiled in when libsession-util is built with +-DWITH_IP_GEOLOCATION=ON. The generated file is neither committed (it is several megabytes of +tables) nor fetched during a build, so run this before configuring with that option, and again +whenever the snapshot is due a refresh; cmake fails with these instructions if it is missing. + + utils/update-ip-country-db.py # current release + utils/update-ip-country-db.py --month 2026-08 # a specific one + utils/update-ip-country-db.py --csv dbip.csv.gz # one already downloaded + +DB-IP Lite is CC BY 4.0: redistribution is permitted with attribution, and unlike MaxMind's GeoLite2 +there is no clause requiring the copy to stay current, so a stale bundled snapshot is a quality +question rather than a licensing one. +""" + +import argparse +import collections +import datetime +import gzip +import io +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +DOWNLOAD_URL = "https://download.db-ip.com/free/{release}.csv.gz" +LANDING_PAGE = "https://db-ip.com/db/download/ip-to-country-lite" +ATTRIBUTION = "IP Geolocation by DB-IP (https://db-ip.com)" +USER_AGENT = "libsession-util update-ip-country-db" + +# range_codes[] in data.hpp is uint8_t, so index 0 (unknown) plus the codes must fit in 256. +MAX_CODES = 256 + +CC_RE = re.compile(r"^[A-Z]{2}$") + +REPO = Path(__file__).resolve().parent.parent +DEFAULT_OUTPUT = REPO / "src" / "network" / "ip_country" / "data.cpp" + + +def release_name(month): + return f"dbip-country-lite-{month}" + + +def download(month): + """Fetches a release, falling back to the previous month when the current one isn't out yet.""" + months = [month] if month else [] + if not months: + today = datetime.date.today() + months = [ + today.strftime("%Y-%m"), + (today.replace(day=1) - datetime.timedelta(days=1)).strftime("%Y-%m"), + ] + + for m in months: + url = DOWNLOAD_URL.format(release=release_name(m)) + print(f"Fetching {url}", file=sys.stderr) + # Cloudflare fronts the download and 403s urllib's default User-Agent, so send our own. + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(request) as resp: + return m, resp.read() + except urllib.error.HTTPError as e: + if e.code != 404 or m == months[-1]: + raise + print(f" {m} not published yet, trying the previous month", file=sys.stderr) + + raise RuntimeError("no release found") + + +def parse(csv_bytes): + """Reads the CSV into (start, end, cc) IPv4 rows, sorted, and the count of IPv6 rows skipped.""" + if csv_bytes[:2] == b"\x1f\x8b": + csv_bytes = gzip.decompress(csv_bytes) + + rows = [] + skipped_v6 = 0 + for lineno, line in enumerate(io.StringIO(csv_bytes.decode()), 1): + line = line.strip() + if not line: + continue + try: + start, end, cc = line.split(",") + except ValueError: + raise ValueError(f"line {lineno}: expected 'start,end,cc', got {line!r}") + if ":" in start: + skipped_v6 += 1 + continue + rows.append((ip_to_int(start, lineno), ip_to_int(end, lineno), cc)) + + rows.sort() + return rows, skipped_v6 + + +def ip_to_int(addr, lineno): + octets = addr.split(".") + if len(octets) != 4: + raise ValueError(f"line {lineno}: {addr!r} is not an IPv4 address") + value = 0 + for octet in octets: + n = int(octet) + if not 0 <= n <= 255: + raise ValueError(f"line {lineno}: {addr!r} is not an IPv4 address") + value = value << 8 | n + return value + + +def tile(rows): + """Turns the rows into the start-only tiling that data.cpp stores. + + The table covers the whole address space so that a lookup needs no end column: a range runs + until the next start. DB-IP's rows already tile it, but gaps are filled with the unknown code + and adjacent same-country ranges merged in case a future release stops doing so. + """ + tiles = [] # (start, cc), with "" for unknown + + def add(start, cc): + if tiles and tiles[-1][1] == cc: + return + tiles.append((start, cc)) + + position = 0 + for start, end, cc in rows: + if start < position: + raise ValueError(f"overlapping range at {start:#010x}") + if start > position: + add(position, "") + # ZZ is DB-IP's marker for space it has no country for, which is our unknown slot. + add(start, "" if cc == "ZZ" else cc) + position = end + 1 + + if position <= 0xFFFFFFFF: + add(position, "") + + counts = collections.Counter(cc for _, cc in tiles if cc) + for cc in counts: + if not CC_RE.match(cc): + raise ValueError(f"{cc!r} is not an ISO 3166-1 alpha-2 country code") + if len(counts) + 1 > MAX_CODES: + raise ValueError( + f"{len(counts) + 1} country codes exceeds the {MAX_CODES} that a uint8_t index holds; " + "widen range_codes() in src/network/ip_country/data.hpp, the array in data.cpp, and " + "MAX_CODES here" + ) + + # Numbering by descending range count, ties alphabetical: the countries holding the most ranges + # get the shortest indices, which takes ~0.4MB off the generated source. It also keeps one + # release's table close to the last one's, since the countries that come and go between releases + # are the rare ones, numbered at the end where nothing follows them to shift. + code_list = [""] + sorted(counts, key=lambda cc: (-counts[cc], cc)) + codes = {cc: i for i, cc in enumerate(code_list)} + + return [(start, codes[cc]) for start, cc in tiles], code_list + + +def group_by_16(tiles): + """Groups the tiles by the /16 their start falls in, keeping them in order. + + Lines in the generated arrays never span a /16, so a refresh that adds or drops a range rewrites + only that /16's few lines instead of reflowing every line below it, which keeps one release's + file comparable to the last's. + """ + groups = [] + for tile in tiles: + key = tile[0] >> 16 + if not groups or groups[-1][0] != key: + groups.append((key, [])) + groups[-1][1].append(tile) + return groups + + +def columns(groups, per_line, index, formatter=str, label=False): + """Formats one field of the grouped tiles as indented rows of comma-separated items.""" + out = io.StringIO() + for key, tiles in groups: + for i in range(0, len(tiles), per_line): + row = ", ".join(formatter(t[index]) for t in tiles[i : i + per_line]) + # The addresses say where they are; the bare code indices need telling. + prefix = "/*{}.{}*/ ".format(key >> 8, key & 0xFF) if label and i == 0 else "" + out.write(f" {prefix}{row},\n") + return out.getvalue() + + +def octets(value): + """An address as an `oxen::quic::ipv4` initializer, e.g. {95,216,0,0}.""" + return "{{{},{},{},{}}}".format( + value >> 24, value >> 16 & 0xFF, value >> 8 & 0xFF, value & 0xFF + ) + + +def generate(path, release, tiles, code_list, rows, skipped_v6): + groups = group_by_16(tiles) + table_bytes = len(tiles) * 5 + len(code_list) * 2 + + cc_lines = "".join( + " {},\n".format(", ".join('"{}"sv'.format(cc) for cc in code_list[i : i + 12])) + for i in range(1, len(code_list), 12) + ) + + with open(path, "w") as out: + out.write( + f"""// Generated by utils/update-ip-country-db.py from {release}.csv.gz -- do not edit. +// +// {ATTRIBUTION}, licensed under CC BY 4.0. +// Source: {LANDING_PAGE} +// +// {len(rows)} IPv4 rows in, {len(tiles)} ranges and {len(code_list) - 1} country codes out, +// {table_bytes / 1e6:.2f} MB of .rodata. {skipped_v6} IPv6 rows were skipped: nothing reads them +// yet, and adding them means a second table rather than a change to this one. +// +// The two range arrays hold plain integers rather than pointers on purpose: a table of pointers +// needs a relocation per entry, which under PIE would turn megabytes of shared, file-backed .rodata +// into dirty private memory at every process start. The country table is 246 entries, so its +// relocations cost nothing worth avoiding. +// +// A line never spans a /16, and the code indices carry their /16 as a comment, so that a refresh +// that adds or drops a range rewrites those few lines rather than reflowing the whole file, and one +// release's table can be compared against the last's. + +// clang-format off +// (utils/format.sh skips this file as well; reflowing a third of a million initializers is neither +// quick nor an improvement.) + +#include "data.hpp" + +#include + +namespace session::ip_country::detail {{ + +using namespace std::literals; + +namespace {{ + + constexpr ipv4 starts[] = {{ +{columns(groups, 6, 0, octets)} }}; + + constexpr uint8_t codes[] = {{ +{columns(groups, 20, 1, label=True)} }}; + + // Ordered by how many ranges each country holds, so that the common ones get the shortest + // indices above and a country appearing or vanishing renumbers as little as possible. + constexpr std::string_view countries[] = {{ + ""sv, // index 0: unassigned or reserved +{cc_lines} }}; + + static_assert(std::size(starts) == std::size(codes), "every range needs exactly one country"); + static_assert( + std::size(countries) <= 256, "codes[] is uint8_t and cannot index more countries"); + +}} // namespace + +std::span range_starts() {{ + return starts; +}} + +std::span range_codes() {{ + return codes; +}} + +std::span country_codes() {{ + return countries; +}} + +std::string_view attribution() {{ + return "{ATTRIBUTION}"; +}} + +std::string_view database_version() {{ + return "{release}"; +}} + +}} // namespace session::ip_country::detail +""" + ) + + print( + f"Wrote {path} ({path.stat().st_size / 1e6:.1f} MB of source):\n" + f" release {release}\n" + f" ranges {len(tiles)} (from {len(rows)} IPv4 rows, {skipped_v6} IPv6 skipped)\n" + f" codes {len(code_list) - 1}\n" + f" compiled {table_bytes / 1e6:.2f} MB", + file=sys.stderr, + ) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--csv", type=Path, help="use this CSV (.csv or .csv.gz) instead of downloading") + parser.add_argument( + "--month", + help="release as YYYY-MM: which one to download, or which one --csv holds when its " + "filename doesn't say (default: the current one)", + ) + parser.add_argument( + "-o", "--output", type=Path, default=DEFAULT_OUTPUT, help=f"where to write (default: {DEFAULT_OUTPUT})" + ) + args = parser.parse_args() + + if args.csv: + csv_bytes = args.csv.read_bytes() + # The release is what the API reports as its version, so it comes from the filename the + # download hands out rather than being invented here. + month = args.month or (m.group(1) if (m := re.search(r"\d{4}-\d{2}", args.csv.name)) else None) + if not month: + parser.error( + f"can't tell the release month from {args.csv.name!r}; pass --month YYYY-MM" + ) + release = release_name(month) + else: + month, csv_bytes = download(args.month) + release = release_name(month) + + rows, skipped_v6 = parse(csv_bytes) + if not rows: + raise RuntimeError("no IPv4 rows in the CSV") + tiles, code_list = tile(rows) + generate(args.output, release, tiles, code_list, rows, skipped_v6) + + +if __name__ == "__main__": + main() diff --git a/utils/verify_mnemonics.py b/utils/verify_mnemonics.py new file mode 100644 index 000000000..bee98d06b --- /dev/null +++ b/utils/verify_mnemonics.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import os +import sys + +def verify_language(filepath): + with open(filepath, 'r', encoding='utf-8') as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + + if len(lines) != 1629: + print(f"[-] {filepath}: Invalid line count ({len(lines)}), expected 1629") + return False + + english_name = lines[0] + prefix_len = int(lines[2]) + words = lines[3:] + + prefixes = {} + collisions = [] + + for word in words: + # Take the prefix in codepoints, case-folded for case-insensitive comparison + prefix_cf = word[:prefix_len].casefold() + if prefix_cf in prefixes: + collisions.append((prefix_cf, prefixes[prefix_cf], word)) + else: + prefixes[prefix_cf] = word + + if collisions: + print(f"[-] {english_name} ({filepath}): Found {len(collisions)} CASE-INSENSITIVE collisions at prefix length {prefix_len}:") + for pref, word1, word2 in collisions[:10]: + print(f" Prefix '{pref}' matches both '{word1}' and '{word2}'") + if len(collisions) > 10: + print(f" ... and {len(collisions) - 10} more.") + return False + + # Check if prefix_len is larger than necessary (case-insensitive) + min_needed = 1 + while True: + test_prefixes = set() + collision_found = False + for word in words: + p = word[:min_needed].casefold() + if p in test_prefixes: + collision_found = True + break + test_prefixes.add(p) + if not collision_found: + break + min_needed += 1 + + if min_needed < prefix_len: + print(f"[!] {english_name}: prefix_len is {prefix_len}, but {min_needed} would suffice (case-insensitive).") + elif min_needed > prefix_len: + print(f"[-] {english_name}: prefix_len {prefix_len} is INSUFFICIENT for case-insensitive uniqueness (needs {min_needed})") + return False + + print(f"[+] {english_name}: Verified case-insensitive (prefix_len={prefix_len})") + return True + +def main(): + lang_dir = "src/mnemonics/languages" + if not os.path.exists(lang_dir): + print(f"Error: Directory {lang_dir} not found.") + sys.exit(1) + + files = [f for f in os.listdir(lang_dir) if f.endswith('.txt')] + files.sort() + + success = True + for filename in files: + if not verify_language(os.path.join(lang_dir, filename)): + success = False + + if not success: + sys.exit(1) + +if __name__ == "__main__": + main()