diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e75931eb..4000e7296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,6 @@ on: push: branches: - master - - staging - - trying env: RUSTFLAGS: -Dwarnings @@ -21,17 +19,18 @@ jobs: - run: cargo fmt --all -- --check run_clippy: + name: Clippy runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: clippy - override: true + - name: Install clippy + run: rustup toolchain install 1.67.1 --component clippy - name: Cache rust cargo artifacts uses: swatinem/rust-cache@v2 - - run: cargo clippy --workspace --tests --examples --benches --features repl -- -D warnings + - name: Run clippy + env: + RUSTUP_TOOLCHAIN: 1.67.1 + run: scripts/clippy.sh docs: name: Rust doc comments @@ -41,13 +40,6 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v3 - - name: Install rust stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - components: rust-docs - override: true - name: Cache rust cargo artifacts uses: swatinem/rust-cache@v2 - name: Rustdoc @@ -79,39 +71,37 @@ jobs: steps: - uses: actions/checkout@master - - name: Install ${{ matrix.rust }} - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust }} - override: true + - name: Install Rust ${{ matrix.rust }} + run: rustup toolchain install ${{ matrix.rust }} + - run: rustup override set ${{ matrix.rust }} - name: Cache rust cargo artifacts uses: swatinem/rust-cache@v2 - - name: check - run: cargo check --all --bins --examples --tests --features repl --benches + - name: Check + run: cargo check --workspace --bins --examples --tests --benches - - name: tests - run: cargo test --all + - name: Tests + run: cargo test --workspace - - name: test cargo vendor + - name: Test cargo vendor run: cargo vendor - - name: install python + - name: Install python if: ${{ matrix.python }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - name: install tox + - name: Install tox if: ${{ matrix.python }} run: pip install tox - - name: build C library + - name: Build C library if: ${{ matrix.python }} run: cargo build -p deltachat_ffi --features jsonrpc - - name: run python tests + - name: Run python tests if: ${{ matrix.python }} env: DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} @@ -120,28 +110,28 @@ jobs: working-directory: python run: tox -e lint,mypy,doc,py3 - - name: build deltachat-rpc-server + - name: Build deltachat-rpc-server if: ${{ matrix.python }} run: cargo build -p deltachat-rpc-server - - name: add deltachat-rpc-server to path + - name: Add deltachat-rpc-server to path if: ${{ matrix.python }} run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH - - name: run deltachat-rpc-client tests + - name: Run deltachat-rpc-client tests if: ${{ matrix.python }} env: DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} working-directory: deltachat-rpc-client run: tox -e py3,lint - - name: install pypy + - name: Install pypy if: ${{ matrix.python }} uses: actions/setup-python@v4 with: python-version: 'pypy${{ matrix.python }}' - - name: run pypy tests + - name: Run pypy tests if: ${{ matrix.python }} env: DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 709ac43eb..a1c3b4b82 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -12,14 +12,14 @@ jobs: name: 'Package @deltachat/jsonrpc-client and upload to download.delta.chat' runs-on: ubuntu-18.04 steps: - - name: install tree + - name: Install tree run: sudo apt install tree - name: Checkout uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: '16' - - name: get tag + - name: Get tag id: tag uses: dawidd6/action-get-tag@v1 continue-on-error: true @@ -38,11 +38,11 @@ jobs: npm --version node --version echo $DELTACHAT_JSONRPC_TAR_GZ - - name: install dependencies without running scripts + - name: Install dependencies without running scripts run: | cd deltachat-jsonrpc/typescript npm install --ignore-scripts - - name: package + - name: Package shell: bash run: | cd deltachat-jsonrpc/typescript @@ -65,7 +65,7 @@ jobs: chmod 600 __TEMP_INPUT_KEY_FILE scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/" continue-on-error: true - - name: "Post links to details" + - name: Post links to details if: steps.upload-preview.outcome == 'success' run: node ./node/scripts/postLinksToDetails.js env: diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index bacdbcb4c..f0cb1cc09 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -9,7 +9,7 @@ on: jobs: prebuild: - name: 'prebuild' + name: Prebuild runs-on: ${{ matrix.os }} strategy: matrix: @@ -65,17 +65,17 @@ jobs: pack-module: needs: prebuild - name: 'Package deltachat-node and upload to download.delta.chat' + name: Package deltachat-node and upload to download.delta.chat runs-on: ubuntu-18.04 steps: - - name: install tree + - name: Install tree run: sudo apt install tree - name: Checkout uses: actions/checkout@v3 - uses: actions/setup-node@v2 with: node-version: '16' - - name: get tag + - name: Get tag id: tag uses: dawidd6/action-get-tag@v1 continue-on-error: true @@ -97,15 +97,15 @@ jobs: npm --version node --version echo $DELTACHAT_NODE_TAR_GZ - - name: Download ubuntu prebuild + - name: Download Ubuntu prebuild uses: actions/download-artifact@v1 with: name: ubuntu-18.04 - - name: Download macos prebuild + - name: Download macOS prebuild uses: actions/download-artifact@v1 with: name: macos-latest - - name: Download windows prebuild + - name: Download Windows prebuild uses: actions/download-artifact@v1 with: name: windows-latest @@ -117,23 +117,23 @@ jobs: tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds tree node/prebuilds rm -rf ubuntu-18.04 macos-latest windows-latest - - name: install dependencies without running scripts + - name: Install dependencies without running scripts run: | npm install --ignore-scripts - - name: build constants + - name: Build constants run: | npm run build:core:constants - - name: build typescript part + - name: Build TypeScript part run: | npm run build:bindings:ts - - name: package + - name: Package shell: bash run: | mv node/README.md README.md npm pack . ls -lah mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ - - name: Upload Prebuild + - name: Upload prebuild uses: actions/upload-artifact@v3 with: name: deltachat-node.tgz @@ -148,7 +148,7 @@ jobs: chmod 600 __TEMP_INPUT_KEY_FILE scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/" continue-on-error: true - - name: "Post links to details" + - name: Post links to details if: steps.upload-preview.outcome == 'success' run: node ./node/scripts/postLinksToDetails.js env: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index d2f210f39..e0d557d42 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -9,7 +9,7 @@ on: jobs: tests: - name: 'tests' + name: Tests runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/.github/workflows/repl.yml b/.github/workflows/repl.yml index c0a76fc37..efad7558e 100644 --- a/.github/workflows/repl.yml +++ b/.github/workflows/repl.yml @@ -13,17 +13,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.66.0 - override: true - - - name: build - run: cargo build --example repl --features repl,vendored + - name: Build + run: cargo build -p deltachat-repl --features vendored - name: Upload binary uses: actions/upload-artifact@v3 with: name: repl.exe - path: 'target/debug/examples/repl.exe' + path: 'target/debug/deltachat-repl.exe' diff --git a/CHANGELOG.md b/CHANGELOG.md index 782e46b8d..29bd7b02a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,18 +3,38 @@ ## Unreleased ## Changes +- Use read/write timeouts instead of per-command timeouts for SMTP #3985 +- Cache DNS results for SMTP connections #3985 +- Prefer TLS over STARTTLS during autoconfiguration #4021 +- Use SOCKS5 configuration for HTTP requests #4017 ## Fixes - Fix Securejoin for multiple devices on a joining side #3982 +- python: handle NULL value returned from `dc_get_msg()` #4020 + Account.`get_message_by_id` may return `None` in this case. ## API-Changes +- Remove bitflags from `get_chat_msgs()` interface #4022 + C interface is not changed. + Rust and JSON-RPC API have `flags` integer argument + replaced with two boolean flags `info_only` and `add_daymarker`. +- jsonrpc: add API to check if the message is sent by a bot #3877 + + +## 1.107.1 + +### Changes +- Log server security (TLS/STARTTLS/plain) type #4005 + +### Fixes +- Disable SMTP pipelining #4006 ## 1.107.0 ### Changes - Pipeline SMTP commands #3924 -- Cache DNS results #3970 +- Cache DNS results for IMAP connections #3970 ### Fixes - Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 diff --git a/Cargo.lock b/Cargo.lock index 95dfe2b8a..304027515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,21 +219,18 @@ dependencies = [ [[package]] name = "async-smtp" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ade89127f9e0d44f9e83cf574d499060005cd45b7dc76be89c0167487fe8edd" +checksum = "7384febcabdd07a498c9f4fbaa7e488ff4eb60d0ade14b47b09ec44b8f645301" dependencies = [ - "async-native-tls", - "async-trait", + "anyhow", "base64 0.13.1", "bufstream", - "fast-socks5", "futures 0.3.26", "hostname", "log", "nom 7.1.1", "pin-project", - "pin-utils", "thiserror", "tokio 1.25.0", ] @@ -402,7 +399,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide 0.6.2", + "miniz_oxide", "object", "rustc-demangle", ] @@ -771,6 +768,23 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "concolor" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318d6c16e73b3a900eb212ad6a82fc7d298c5ab8184c7a9998646455bc474a16" +dependencies = [ + "bitflags", + "concolor-query", + "is-terminal", +] + +[[package]] +name = "concolor-query" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82a90734b3d5dcf656e7624cca6bce9c3a90ee11f900e80141a7427ccfb3d317" + [[package]] name = "concurrent-queue" version = "2.0.0" @@ -1127,7 +1141,6 @@ dependencies = [ "chrono", "criterion", "deltachat_derive", - "dirs", "email", "encoded-words", "escaper", @@ -1162,7 +1175,6 @@ dependencies = [ "reqwest", "rusqlite", "rust-hsluv", - "rustyline", "sanitize-filename", "sendme", "serde", @@ -1181,10 +1193,10 @@ dependencies = [ "tokio-io-timeout", "tokio-stream", "tokio-tar", - "toml 0.7.0", + "toml 0.7.1", "trust-dns-resolver", "url", - "uuid 1.2.2", + "uuid 1.3.0", ] [[package]] @@ -1209,6 +1221,21 @@ dependencies = [ "yerpc", ] +[[package]] +name = "deltachat-repl" +version = "1.107.0" +dependencies = [ + "ansi_term", + "anyhow", + "deltachat", + "dirs", + "log", + "pretty_env_logger", + "rusqlite", + "rustyline", + "tokio 1.25.0", +] + [[package]] name = "deltachat-rpc-server" version = "1.107.0" @@ -1738,12 +1765,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", - "miniz_oxide 0.5.3", + "miniz_oxide", ] [[package]] @@ -2109,12 +2136,13 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "human-panic" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36" +checksum = "87eb03e654582b31967d414b86711a7bbd7c6b28a6b7d32857b7d1d45c0926f9" dependencies = [ "backtrace", - "os_type", + "concolor", + "os_info", "serde", "serde_derive", "termcolor", @@ -2593,15 +2621,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.6.2" @@ -2711,10 +2730,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ + "autocfg", "bitflags", "cfg-if 1.0.0", "libc", @@ -2920,9 +2940,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.22.0+1.1.1q" +version = "111.25.0+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853" +checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" dependencies = [ "cc", ] @@ -2941,21 +2961,23 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_info" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc1b4330bb29087e791ae2a5cf56be64fb8946a4ff5aec2ba11c6ca51f5d60" +dependencies = [ + "log", + "serde", + "winapi 0.3.9", +] + [[package]] name = "os_str_bytes" version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "os_type" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3df761f6470298359f84fcfb60d86db02acc22c251c37265c07a3d1057d2389" -dependencies = [ - "regex", -] - [[package]] name = "ouroboros" version = "0.15.2" @@ -3222,7 +3244,7 @@ dependencies = [ "bitflags", "crc32fast", "flate2", - "miniz_oxide 0.6.2", + "miniz_oxide", ] [[package]] @@ -3840,9 +3862,9 @@ checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" [[package]] name = "rustyline" -version = "10.0.0" +version = "10.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e" +checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -4031,9 +4053,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" dependencies = [ "serde", ] @@ -4071,9 +4093,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" dependencies = [ "serde", ] @@ -4433,10 +4455,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if 1.0.0", "once_cell", ] @@ -4811,9 +4834,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f560bc7fb3eb31f5eee1340c68a2160cad39605b7b9c9ec32045ddbdee13b85" +checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" dependencies = [ "serde", "serde_spanned", @@ -4823,18 +4846,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "886f31a9b85b6182cabd4d8b07df3b451afcc216563748201490940d2a28ed36" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d8716cdc5d20ec88a18a839edaf545edc71efa4a5ff700ef4a102c26cd8fa" +checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" dependencies = [ "indexmap", "nom8", @@ -5181,9 +5204,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" dependencies = [ "getrandom 0.2.7", "serde", diff --git a/Cargo.toml b/Cargo.toml index 2b5c53991..dbf3503e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,70 +28,66 @@ deltachat_derive = { path = "./deltachat_derive" } format-flowed = { path = "./format-flowed" } ratelimit = { path = "./deltachat-ratelimit" } -ansi_term = { version = "0.12.1", optional = true } anyhow = "1" -async-channel = "1.8.0" async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] } -async-smtp = { version = "0.6", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } -async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] } +async-smtp = { version = "0.8", default-features = false, features = ["runtime-tokio"] } +trust-dns-resolver = "0.22" +tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } +tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar backtrace = "0.3" base64 = "0.21" bitflags = "1.3" chrono = { version = "0.4", default-features=false, features = ["clock", "std"] } -dirs = { version = "4", optional=true } email = { git = "https://github.com/deltachat/rust-email", branch = "master" } encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" } escaper = "0.1" -fast-socks5 = "0.8" futures = "0.3" -futures-lite = "1.12.0" hex = "0.4.0" -humansize = "2" image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] } kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" -log = {version = "0.4.16", optional = true } mailparse = "0.14" native-tls = "0.2" +num_cpus = "1.15" num-derive = "0.3" num-traits = "0.2" -num_cpus = "1.15" once_cell = "1.17.0" percent-encoding = "2.2" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } -qrcodegen = "1.7.0" quick-xml = "0.27" r2d2 = "0.8" r2d2_sqlite = "0.20" rand = "0.8" regex = "1.7" -reqwest = { version = "0.11.14", features = ["json"] } rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" -rustyline = { version = "10", optional = true } sanitize-filename = "0.4" sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } -serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" sha2 = "0.10" smallvec = "1" strum = "0.24" strum_macros = "0.24" -tagger = "4.3.4" -textwrap = "0.16.0" thiserror = "1" -tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } -tokio-io-timeout = "1.2.0" -tokio-stream = { version = "0.1.11", features = ["fs"] } -tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar toml = "0.7" -trust-dns-resolver = "0.22" url = "2" uuid = { version = "1", features = ["serde", "v4"] } +fast-socks5 = "0.8" +humansize = "2" +qrcodegen = "1.7.0" +tagger = "4.3.4" +textwrap = "0.16.0" +async-channel = "1.8.0" +futures-lite = "1.12.0" +tokio-stream = { version = "0.1.11", features = ["fs"] } +tokio-io-timeout = "1.2.0" +reqwest = { version = "0.11.14", features = ["json"] } +async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] } [dev-dependencies] ansi_term = "0.12.0" @@ -111,18 +107,13 @@ members = [ "deltachat-jsonrpc", "deltachat-rpc-server", "deltachat-ratelimit", + "deltachat-repl", "format-flowed", ] [[example]] name = "simple" path = "examples/simple.rs" -required-features = ["repl"] - -[[example]] -name = "repl" -path = "examples/repl/main.rs" -required-features = ["repl"] [[bench]] @@ -156,10 +147,8 @@ harness = false [features] default = ["vendored"] internals = [] -repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"] vendored = [ "async-native-tls/vendored", - "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored" ] diff --git a/README.md b/README.md index 3ea334637..7e9f6b1eb 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,19 @@ $ curl https://sh.rustup.rs -sSf | sh Compile and run Delta Chat Core command line utility, using `cargo`: ``` -$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db +$ RUST_LOG=repl=info cargo run -p deltachat-repl -- ~/deltachat-db ``` where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist. +Optionally, install `deltachat-repl` binary with +``` +$ cargo install --path deltachat-repl/ +``` +and run as +``` +$ deltachat-repl ~/deltachat-db +``` + Configure your account (if not already configured): ``` diff --git a/benches/get_chat_msgs.rs b/benches/get_chat_msgs.rs index fac37dd87..3da10e659 100644 --- a/benches/get_chat_msgs.rs +++ b/benches/get_chat_msgs.rs @@ -14,7 +14,7 @@ async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) { .unwrap(); for c in chats.iter().take(10) { - black_box(chat::get_chat_msgs(&context, *c, 0).await.ok()); + black_box(chat::get_chat_msgs(&context, *c).await.ok()); } } diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index ebb236043..c7cd23bd2 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use anyhow::Context as _; -use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus}; use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; use deltachat::contact::{Contact, ContactId, Origin}; use deltachat::context::Context; @@ -60,7 +60,8 @@ use self::string::*; // this avoids panics if the ui just forgets to handle a case // - finally, this behaviour matches the old core-c API and UIs already depend on it -// TODO: constants +const DC_GCM_ADDDAYMARKER: u32 = 0x01; +const DC_GCM_INFO_ONLY: u32 = 0x02; // dc_context_t @@ -1156,12 +1157,21 @@ pub unsafe extern "C" fn dc_get_chat_msgs( } let ctx = &*context; + let info_only = (flags & DC_GCM_INFO_ONLY) != 0; + let add_daymarker = (flags & DC_GCM_ADDDAYMARKER) != 0; block_on(async move { Box::into_raw(Box::new( - chat::get_chat_msgs(ctx, ChatId::new(chat_id), flags) - .await - .unwrap_or_log_default(ctx, "failed to get chat msgs") - .into(), + chat::get_chat_msgs_ex( + ctx, + ChatId::new(chat_id), + MessageListOptions { + info_only, + add_daymarker, + }, + ) + .await + .unwrap_or_log_default(ctx, "failed to get chat msgs") + .into(), )) }) } diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 1ded9728c..9df681cb3 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -6,8 +6,9 @@ use anyhow::{anyhow, bail, ensure, Context, Result}; pub use deltachat::accounts::Accounts; use deltachat::{ chat::{ - self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, marknoticed_chat, - remove_contact_from_chat, Chat, ChatId, ChatItem, ProtectionStatus, + self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, get_chat_msgs_ex, + marknoticed_chat, remove_contact_from_chat, Chat, ChatId, ChatItem, MessageListOptions, + ProtectionStatus, }, chatlist::Chatlist, config::Config, @@ -803,7 +804,7 @@ impl CommandApi { let ctx = self.get_context(account_id).await?; // TODO: implement this in core with an SQL query, that will be way faster - let messages = get_chat_msgs(&ctx, ChatId::new(chat_id), 0).await?; + let messages = get_chat_msgs(&ctx, ChatId::new(chat_id)).await?; let mut first_unread_message_id = None; for item in messages.into_iter().rev() { if let ChatItem::Message { msg_id } = item { @@ -878,9 +879,23 @@ impl CommandApi { markseen_msgs(&ctx, msg_ids.into_iter().map(MsgId::new).collect()).await } - async fn get_message_ids(&self, account_id: u32, chat_id: u32, flags: u32) -> Result> { + async fn get_message_ids( + &self, + account_id: u32, + chat_id: u32, + info_only: bool, + add_daymarker: bool, + ) -> Result> { let ctx = self.get_context(account_id).await?; - let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?; + let msg = get_chat_msgs_ex( + &ctx, + ChatId::new(chat_id), + MessageListOptions { + info_only, + add_daymarker, + }, + ) + .await?; Ok(msg .iter() .map(|chat_item| -> u32 { @@ -896,10 +911,19 @@ impl CommandApi { &self, account_id: u32, chat_id: u32, - flags: u32, + info_only: bool, + add_daymarker: bool, ) -> Result> { let ctx = self.get_context(account_id).await?; - let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?; + let msg = get_chat_msgs_ex( + &ctx, + ChatId::new(chat_id), + MessageListOptions { + info_only, + add_daymarker, + }, + ) + .await?; Ok(msg .iter() .map(|chat_item| (*chat_item).into()) diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 0f35f0c6f..9030467f6 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -48,6 +48,10 @@ pub struct MessageObject { is_setupmessage: bool, is_info: bool, is_forwarded: bool, + + /// True if the message was sent by a bot. + is_bot: bool, + /// when is_info is true this describes what type of system message it is system_message_type: SystemMessageType, @@ -182,6 +186,7 @@ impl MessageObject { is_setupmessage: message.is_setupmessage(), is_info: message.is_info(), is_forwarded: message.is_forwarded(), + is_bot: message.is_bot(), system_message_type: message.get_info_type().into(), duration: message.get_duration(), diff --git a/deltachat-jsonrpc/typescript/example/example.ts b/deltachat-jsonrpc/typescript/example/example.ts index b7e37f9cc..15baf68b5 100644 --- a/deltachat-jsonrpc/typescript/example/example.ts +++ b/deltachat-jsonrpc/typescript/example/example.ts @@ -76,7 +76,8 @@ async function run() { const messageIds = await client.rpc.getMessageIds( selectedAccount, chatId, - 0 + false, + false ); const messages = await client.rpc.getMessages( selectedAccount, diff --git a/deltachat-jsonrpc/typescript/test/online.ts b/deltachat-jsonrpc/typescript/test/online.ts index 06c4c8954..b8fd84d08 100644 --- a/deltachat-jsonrpc/typescript/test/online.ts +++ b/deltachat-jsonrpc/typescript/test/online.ts @@ -97,7 +97,8 @@ describe("online tests", function () { const messageList = await dc.rpc.getMessageIds( accountId2, chatIdOnAccountB, - 0 + false, + false ); expect(messageList).have.length(1); @@ -133,7 +134,8 @@ describe("online tests", function () { const messageList = await dc.rpc.getMessageIds( accountId2, chatIdOnAccountB, - 0 + false, + false ); const message = await dc.rpc.getMessage( accountId2, @@ -150,7 +152,7 @@ describe("online tests", function () { await eventPromise2; const messageId = ( - await dc.rpc.getMessageIds(accountId1, chatId, 0) + await dc.rpc.getMessageIds(accountId1, chatId, false, false) ).reverse()[0]; const message2 = await dc.rpc.getMessage(accountId1, messageId); expect(message2.text).equal("super secret message"); diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml new file mode 100644 index 000000000..b6df1f474 --- /dev/null +++ b/deltachat-repl/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "deltachat-repl" +version = "1.107.0" +edition = "2021" + +[dependencies] +ansi_term = "0.12.1" +anyhow = "1" +deltachat = { path = "..", features = ["internals"]} +dirs = "4" +log = "0.4.16" +pretty_env_logger = "0.4" +rusqlite = "0.27" +rustyline = "10" +tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } + +[features] +default = ["vendored"] +vendored = ["deltachat/vendored"] diff --git a/examples/repl/cmdline.rs b/deltachat-repl/src/cmdline.rs similarity index 99% rename from examples/repl/cmdline.rs rename to deltachat-repl/src/cmdline.rs index 1d3d1edbb..d94c04a47 100644 --- a/examples/repl/cmdline.rs +++ b/deltachat-repl/src/cmdline.rs @@ -666,8 +666,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu let sel_chat = sel_chat.as_ref().unwrap(); let time_start = std::time::SystemTime::now(); - let msglist = - chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER).await?; + let msglist = chat::get_chat_msgs_ex( + &context, + sel_chat.get_id(), + chat::MessageListOptions { + info_only: false, + add_daymarker: true, + }, + ) + .await?; let time_needed = time_start.elapsed().unwrap_or_default(); let msglist: Vec = msglist diff --git a/examples/repl/main.rs b/deltachat-repl/src/main.rs similarity index 100% rename from examples/repl/main.rs rename to deltachat-repl/src/main.rs diff --git a/deltachat-rpc-client/examples/echobot_advanced.py b/deltachat-rpc-client/examples/echobot_advanced.py index 303f3ee66..3030941c8 100644 --- a/deltachat-rpc-client/examples/echobot_advanced.py +++ b/deltachat-rpc-client/examples/echobot_advanced.py @@ -64,7 +64,8 @@ async def main(): bot = Bot(account, hooks) if not await bot.is_configured(): - asyncio.create_task(bot.configure(email=sys.argv[1], password=sys.argv[2])) + # Save a reference to avoid garbage collection of the task. + _configure_task = asyncio.create_task(bot.configure(email=sys.argv[1], password=sys.argv[2])) await bot.run_forever() diff --git a/deltachat-rpc-client/examples/echobot_no_hooks.py b/deltachat-rpc-client/examples/echobot_no_hooks.py index bf4f6ce9e..fadf2e560 100644 --- a/deltachat-rpc-client/examples/echobot_no_hooks.py +++ b/deltachat-rpc-client/examples/echobot_no_hooks.py @@ -32,7 +32,7 @@ async def main(): async def process_messages(): for message in await account.get_fresh_messages_in_arrival_order(): snapshot = await message.get_snapshot() - if not snapshot.is_info: + if not snapshot.is_bot and not snapshot.is_info: await snapshot.chat.send_text(snapshot.text) await snapshot.message.mark_seen() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py index f53d19c92..dfdcd78c3 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py @@ -104,7 +104,8 @@ async def _run_cli( if not await client.is_configured(): assert args.email, "Account is not configured and email must be provided" assert args.password, "Account is not configured and password must be provided" - asyncio.create_task(client.configure(email=args.email, password=args.password)) + # Save a reference to avoid garbage collection of the task. + _configure_task = asyncio.create_task(client.configure(email=args.email, password=args.password)) await client.run_forever() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index cdcefcf95..73adbfb1c 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -174,9 +174,9 @@ class Chat: snapshot["message"] = Message(self.account, snapshot.id) return snapshot - async def get_messages(self, flags: int = 0) -> List[Message]: + async def get_messages(self, info_only: bool = False, add_daymarker: bool = False) -> List[Message]: """get the list of messages in this chat.""" - msgs = await self._rpc.get_message_ids(self.account.id, self.id, flags) + msgs = await self._rpc.get_message_ids(self.account.id, self.id, info_only, add_daymarker) return [Message(self.account, msg_id) for msg_id in msgs] async def get_fresh_message_count(self) -> int: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 3801dcc72..0e160445c 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -96,6 +96,9 @@ class NewMessage(EventFilter): content. :param command: If set, only match messages with the given command (ex. /help). Setting this property implies `is_info==False`. + :param is_bot: If set to True only match messages sent by bots, if set to None + match messages from bots and users. If omitted or set to False + only messages from users will be matched. :param is_info: If set to True only match info/system messages, if set to False only match messages that are not info/system messages. If omitted info/system messages as well as normal messages will be matched. @@ -113,10 +116,12 @@ class NewMessage(EventFilter): re.Pattern, ] = None, command: Optional[str] = None, + is_bot: Optional[bool] = False, is_info: Optional[bool] = None, func: Optional[Callable[[AttrDict], bool]] = None, ) -> None: super().__init__(func=func) + self.is_bot = is_bot self.is_info = is_info if command is not None and not isinstance(command, str): raise TypeError("Invalid command") @@ -133,19 +138,28 @@ class NewMessage(EventFilter): raise TypeError("Invalid pattern type") def __hash__(self) -> int: - return hash((self.pattern, self.func)) + return hash((self.pattern, self.command, self.is_bot, self.is_info, self.func)) def __eq__(self, other) -> bool: if isinstance(other, NewMessage): - return (self.pattern, self.command, self.is_info, self.func) == ( + return ( + self.pattern, + self.command, + self.is_bot, + self.is_info, + self.func, + ) == ( other.pattern, other.command, + other.is_bot, other.is_info, other.func, ) return False async def filter(self, event: AttrDict) -> bool: + if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot: + return False if self.is_info is not None and self.is_info != event.message_snapshot.is_info: return False if self.command and self.command != event.command: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 36969c9c1..87e97f0e0 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -2,6 +2,7 @@ import json import os from typing import AsyncGenerator, List, Optional +import asyncio import aiohttp import pytest_asyncio @@ -51,11 +52,13 @@ class ACFactory: await bot.configure(credentials["email"], credentials["password"]) return bot + async def get_online_account(self) -> Account: + account = await self.new_configured_account() + await account.start_io() + return account + async def get_online_accounts(self, num: int) -> List[Account]: - accounts = [await self.new_configured_account() for _ in range(num)] - for account in accounts: - await account.start_io() - return accounts + return await asyncio.gather(*[self.get_online_account() for _ in range(num)]) async def send_message( self, diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 1d02e1928..83a783f67 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -47,6 +47,7 @@ async def test_configure_starttls(acfactory) -> None: # Use STARTTLS await account.set_config("mail_security", "2") + await account.set_config("send_security", "2") await account.configure() assert await account.is_configured() @@ -215,6 +216,7 @@ async def test_message(acfactory) -> None: snapshot = await message.get_snapshot() assert snapshot.chat_id == chat_id assert snapshot.text == "Hello!" + assert not snapshot.is_bot assert repr(message) with pytest.raises(JsonRpcError): # chat is not accepted @@ -226,18 +228,46 @@ async def test_message(acfactory) -> None: await message.send_reaction("😎") +@pytest.mark.asyncio() +async def test_is_bot(acfactory) -> None: + """Test that we can recognize messages submitted by bots.""" + alice, bob = await acfactory.get_online_accounts(2) + + bob_addr = await bob.get_config("addr") + alice_contact_bob = await alice.create_contact(bob_addr, "Bob") + alice_chat_bob = await alice_contact_bob.create_chat() + + # Alice becomes a bot. + await alice.set_config("bot", "1") + await alice_chat_bob.send_text("Hello!") + + while True: + event = await bob.wait_for_event() + if event.type == EventType.INCOMING_MSG: + msg_id = event.msg_id + message = bob.get_message_by_id(msg_id) + snapshot = await message.get_snapshot() + assert snapshot.chat_id == event.chat_id + assert snapshot.text == "Hello!" + assert snapshot.is_bot + break + + @pytest.mark.asyncio() async def test_bot(acfactory) -> None: mock = MagicMock() user = (await acfactory.get_online_accounts(1))[0] bot = await acfactory.new_configured_bot() + bot2 = await acfactory.new_configured_bot() assert await bot.is_configured() assert await bot.account.get_config("bot") == "1" - hook = lambda e: mock.hook(e.msg_id), events.RawEvent(EventType.INCOMING_MSG) + hook = lambda e: mock.hook(e.msg_id) and None, events.RawEvent(EventType.INCOMING_MSG) bot.add_hook(*hook) event = await acfactory.process_message(from_account=user, to_client=bot, text="Hello!") + snapshot = await bot.account.get_message_by_id(event.msg_id).get_snapshot() + assert not snapshot.is_bot mock.hook.assert_called_once_with(event.msg_id) bot.remove_hook(*hook) @@ -252,6 +282,8 @@ async def test_bot(acfactory) -> None: mock.hook.assert_called_with(event.msg_id) event = await acfactory.process_message(from_account=user, to_client=bot, text="hello!") mock.hook.assert_called_with(event.msg_id) + await acfactory.process_message(from_account=bot2.account, to_client=bot, text="hello") + assert len(mock.hook.mock_calls) == 2 # bot messages are ignored between bots await acfactory.process_message(from_account=user, to_client=bot, text="hey!") assert len(mock.hook.mock_calls) == 2 bot.remove_hook(*hook) diff --git a/deltachat-rpc-client/tox.ini b/deltachat-rpc-client/tox.ini index 1c222e9a9..7f74daa37 100644 --- a/deltachat-rpc-client/tox.ini +++ b/deltachat-rpc-client/tox.ini @@ -25,5 +25,5 @@ deps = ruff black commands = - black --check src/ examples/ tests/ + black --check --diff src/ examples/ tests/ ruff src/ examples/ tests/ diff --git a/examples/simple.rs b/examples/simple.rs index c75af319a..cca50bb27 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -28,7 +28,7 @@ fn cb(event: EventType) { } } -/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`. +/// Run with `RUST_LOG=simple=info cargo run --release --example simple -- email pw`. #[tokio::main] async fn main() { pretty_env_logger::try_init_timed().ok(); diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 46118154d..b4510b7d0 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -141,21 +141,18 @@ dependencies = [ [[package]] name = "async-smtp" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da21e1dd19fbad3e095ad519fb1558ab77fd82e5c4778dca8f9be0464589e1e" +checksum = "7384febcabdd07a498c9f4fbaa7e488ff4eb60d0ade14b47b09ec44b8f645301" dependencies = [ - "async-native-tls", - "async-trait", + "anyhow", "base64 0.13.1", "bufstream", - "fast-socks5", "futures", "hostname", "log", "nom 7.1.1", "pin-project", - "pin-utils", "thiserror", "tokio", ] @@ -235,9 +232,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" @@ -699,7 +696,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.104.0" +version = "1.107.0" dependencies = [ "anyhow", "async-channel", @@ -708,7 +705,7 @@ dependencies = [ "async-smtp", "async_zip", "backtrace", - "base64 0.20.0", + "base64 0.21.0", "bitflags", "chrono", "deltachat_derive", @@ -738,6 +735,7 @@ dependencies = [ "r2d2", "r2d2_sqlite", "rand 0.8.5", + "ratelimit", "regex", "reqwest", "rusqlite", @@ -1812,6 +1810,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "num-bigint-dig" version = "0.8.2" @@ -1948,9 +1955,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.24.0+1.1.1s" +version = "111.25.0+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" dependencies = [ "cc", ] @@ -2326,6 +2333,10 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "ratelimit" +version = "1.0.0" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2363,11 +2374,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -2578,6 +2589,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2863,9 +2883,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.1" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", @@ -2952,11 +2972,36 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" dependencies = [ "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5" +dependencies = [ + "indexmap", + "nom8", + "serde", + "serde_spanned", + "toml_datetime", ] [[package]] diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index 7bb511ae8..569feb043 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -14,10 +14,10 @@ class EchoPlugin: message.create_chat() addr = message.get_sender_contact().addr if message.is_system_message(): - message.chat.send_text("echoing system message from {}:\n{}".format(addr, message)) + message.chat.send_text(f"echoing system message from {addr}:\n{message}") else: text = message.text - message.chat.send_text("echoing from {}:\n{}".format(addr, text)) + message.chat.send_text(f"echoing from {addr}:\n{text}") @account_hookimpl def ac_message_delivered(self, message): diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index 59705c379..7ca003c0d 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -14,7 +14,7 @@ class GroupTrackingPlugin: message.create_chat() addr = message.get_sender_contact().addr text = message.text - message.chat.send_text("echoing from {}:\n{}".format(addr, text)) + message.chat.send_text(f"echoing from {addr}:\n{text}") @account_hookimpl def ac_outgoing_message(self, message): @@ -28,7 +28,7 @@ class GroupTrackingPlugin: def ac_chat_modified(self, chat): print("ac_chat_modified:", chat.id, chat.get_name()) for member in chat.get_contacts(): - print("chat member: {}".format(member.addr)) + print(f"chat member: {member.addr}") @account_hookimpl def ac_member_added(self, chat, contact, actor, message): @@ -40,7 +40,7 @@ class GroupTrackingPlugin: ), ) for member in chat.get_contacts(): - print("chat member: {}".format(member.addr)) + print(f"chat member: {member.addr}") @account_hookimpl def ac_member_removed(self, chat, contact, actor, message): diff --git a/python/pyproject.toml b/python/pyproject.toml index 18dfe3e9b..569e849db 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -45,7 +45,7 @@ git_describe_command = "git describe --dirty --tags --long --match py-*.*" line-length = 120 [tool.ruff] -select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM"] +select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP032"] line-length = 120 [tool.isort] diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 227f88977..048d8e3a7 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -55,6 +55,7 @@ def run_cmdline(argv=None, account_plugins=None): ac.run_account(addr=args.email, password=args.password, account_plugins=account_plugins, show_ffi=args.show_ffi) - print("{}: waiting for message".format(ac.get_config("addr"))) + addr = ac.get_config("addr") + print(f"{addr}: waiting for message") ac.wait_shutdown() diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 9ace6fc0e..fffabe0d2 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -171,7 +171,7 @@ def extract_defines(flags): match = defines_re.match(line) if match: defines.append(match.group(1)) - return "\n".join("#define {} ...".format(d) for d in defines) + return "\n".join(f"#define {d} ..." for d in defines) def ffibuilder(): diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index df023288d..e9ec5e311 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -20,7 +20,6 @@ from .cutil import ( from_optional_dc_charpointer, iter_array, ) -from .events import EventThread, FFIEventLogger from .message import Message from .tracker import ConfigureTracker, ImexTracker @@ -63,6 +62,8 @@ class Account(object): MissingCredentials = MissingCredentials def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None: + from .events import EventThread + """initialize account object. :param db_path: a path to the account database. The database @@ -84,7 +85,7 @@ class Account(object): ptr = lib.dc_context_new_closed(db_path) if closed else lib.dc_context_new(ffi.NULL, db_path, ffi.NULL) if ptr == ffi.NULL: - raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path)) + raise ValueError(f"Could not dc_context_new: {os_name} {db_path}") self._dc_context = ffi.gc( ptr, lib.dc_context_unref, @@ -116,7 +117,7 @@ class Account(object): self._logging = True def __repr__(self): - return "".format(self.db_path) + return f"" # def __del__(self): # self.shutdown() @@ -127,7 +128,7 @@ class Account(object): def _check_config_key(self, name: str) -> None: if name not in self._configkeys: - raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(name, self._configkeys)) + raise KeyError(f"{name!r} not a valid config key, existing keys: {self._configkeys!r}") def get_info(self) -> Dict[str, str]: """return dictionary of built config parameters.""" @@ -141,7 +142,7 @@ class Account(object): log("=============== " + self.get_config("displayname") + " ===============") cursor = 0 for name, val in self.get_info().items(): - entry = "{}={}".format(name.upper(), val) + entry = f"{name.upper()}={val}" if cursor + len(entry) > 80: log("") cursor = 0 @@ -186,7 +187,7 @@ class Account(object): self._check_config_key(name) namebytes = name.encode("utf8") res = lib.dc_get_config(self._dc_context, namebytes) - assert res != ffi.NULL, "config value not found for: {!r}".format(name) + assert res != ffi.NULL, f"config value not found for: {name!r}" return from_dc_charpointer(res) def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None: @@ -296,7 +297,7 @@ class Account(object): addr, displayname = obj.get_config("addr"), obj.get_config("displayname") elif isinstance(obj, Contact): if obj.account != self: - raise ValueError("account mismatch {}".format(obj)) + raise ValueError(f"account mismatch {obj}") addr, displayname = obj.addr, obj.name elif isinstance(obj, str): displayname, addr = parseaddr(obj) @@ -368,7 +369,7 @@ class Account(object): def get_fresh_messages(self) -> Generator[Message, None, None]: """yield all fresh messages from all chats.""" dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref) - yield from iter_array(dc_array, lambda x: Message.from_db(self, x)) + return (x for x in iter_array(dc_array, lambda x: Message.from_db(self, x)) if x is not None) def create_chat(self, obj) -> Chat: """Create a 1:1 chat with Account, Contact or e-mail address.""" @@ -413,7 +414,7 @@ class Account(object): def get_device_chat(self) -> Chat: return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat() - def get_message_by_id(self, msg_id: int) -> Message: + def get_message_by_id(self, msg_id: int) -> Optional[Message]: """return Message instance. :param msg_id: integer id of this message. :returns: :class:`deltachat.message.Message` instance. @@ -428,7 +429,7 @@ class Account(object): """ res = lib.dc_get_chat(self._dc_context, chat_id) if res == ffi.NULL: - raise ValueError("cannot get chat with id={}".format(chat_id)) + raise ValueError(f"cannot get chat with id={chat_id}") lib.dc_chat_unref(res) return Chat(self, chat_id) @@ -545,7 +546,7 @@ class Account(object): res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref) lot = DCLot(res) if lot.state() == const.DC_QR_ERROR: - raise ValueError("invalid or unknown QR code: {}".format(lot.text1())) + raise ValueError(f"invalid or unknown QR code: {lot.text1()}") return ScannedQRCode(lot) def qr_setup_contact(self, qr): @@ -596,6 +597,8 @@ class Account(object): # def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False): + from .events import FFIEventLogger + """get the account running, configure it if necessary. add plugins if provided. :param addr: the email address of the account @@ -743,7 +746,7 @@ class Account(object): try: self._event_thread.wait(timeout=5) except RuntimeError as e: - self.log("Waiting for event thread failed: {}".format(e)) + self.log(f"Waiting for event thread failed: {e}") if self._event_thread.is_alive(): self.log("WARN: event thread did not terminate yet, ignoring.") diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index 783f25304..1548829ae 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -40,7 +40,7 @@ class Chat(object): return not self == other def __repr__(self) -> str: - return "".format(self.id, self.get_name()) + return f"" @property def _dc_chat(self): @@ -282,12 +282,20 @@ class Chat(object): if msg.is_out_preparing(): assert msg.id != 0 # get a fresh copy of dc_msg, the core needs it - msg = Message.from_db(self.account, msg.id) + maybe_msg = Message.from_db(self.account, msg.id) + if maybe_msg is not None: + msg = maybe_msg + else: + raise ValueError("message does not exist") + sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg) if sent_id == 0: raise ValueError("message could not be sent") # modify message in place to avoid bad state for the caller - msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg + sent_msg = Message.from_db(self.account, sent_id) + if sent_msg is None: + raise ValueError("cannot load just sent message from the database") + msg._dc_msg = sent_msg._dc_msg return msg def send_text(self, text): @@ -444,7 +452,7 @@ class Chat(object): contact = self.account.create_contact(obj) ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id) if ret != 1: - raise ValueError("could not add contact {!r} to chat".format(contact)) + raise ValueError(f"could not add contact {contact!r} to chat") return contact def remove_contact(self, obj): @@ -457,7 +465,7 @@ class Chat(object): contact = self.account.get_contact(obj) ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id) if ret != 1: - raise ValueError("could not remove contact {!r} from chat".format(contact)) + raise ValueError(f"could not remove contact {contact!r} from chat") def get_contacts(self): """get all contacts for this chat. @@ -493,7 +501,7 @@ class Chat(object): p = as_dc_charpointer(img_path) res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p) if res != 1: - raise ValueError("Setting Profile Image {!r} failed".format(p)) + raise ValueError(f"Setting Profile Image {p!r} failed") def remove_profile_image(self): """remove group profile image. diff --git a/python/src/deltachat/contact.py b/python/src/deltachat/contact.py index adf4cdbad..fc38cdd73 100644 --- a/python/src/deltachat/contact.py +++ b/python/src/deltachat/contact.py @@ -31,7 +31,7 @@ class Contact(object): return not self == other def __repr__(self): - return "".format(self.id, self.addr, self.account._dc_context) + return f"" @property def _dc_contact(self): diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index f2f4417c5..193924055 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -82,7 +82,7 @@ class DirectImap: configured, otherwise None. """ if "_" not in config_name: - config_name = "configured_{}_folder".format(config_name) + config_name = f"configured_{config_name}_folder" foldername = self.account.get_config(config_name) if foldername: return self.select_folder(foldername) @@ -203,7 +203,7 @@ class IdleManager: """(blocking) wait for next idle message from server.""" self.log("imap-direct: calling idle_check") res = self.direct_imap.conn.idle.poll(timeout=timeout) - self.log("imap-direct: idle_check returned {!r}".format(res)) + self.log(f"imap-direct: idle_check returned {res!r}") return res def wait_for_new_message(self, timeout=None) -> bytes: diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index 9cd1caefe..d88876004 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -13,6 +13,7 @@ from .capi import ffi, lib from .cutil import from_optional_dc_charpointer from .hookspec import account_hookimpl from .message import map_system_message +from .account import Account def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): @@ -31,11 +32,11 @@ class FFIEvent: def __str__(self): if self.name == "DC_EVENT_INFO": - return "INFO {data2}".format(data2=self.data2) + return f"INFO {self.data2}" if self.name == "DC_EVENT_WARNING": - return "WARNING {data2}".format(data2=self.data2) + return f"WARNING {self.data2}" if self.name == "DC_EVENT_ERROR": - return "ERROR {data2}".format(data2=self.data2) + return f"ERROR {self.data2}" return "{name} data1={data1} data2={data2}".format(**self.__dict__) @@ -68,7 +69,7 @@ class FFIEventLogger: locname = tname if self.logid: locname += "-" + self.logid - s = "{:2.2f} [{}] {}".format(elapsed, locname, message) + s = f"{elapsed:2.2f} [{locname}] {message}" if os.name == "posix": WARN = "\033[93m" @@ -103,7 +104,7 @@ class FFIEventTracker: timeout = timeout if timeout is not None else self._timeout ev = self._event_queue.get(timeout=timeout) if check_error and ev.name == "DC_EVENT_ERROR": - raise ValueError("unexpected event: {}".format(ev)) + raise ValueError(f"unexpected event: {ev}") return ev def iter_events(self, timeout=None, check_error=True): @@ -111,7 +112,7 @@ class FFIEventTracker: yield self.get(timeout=timeout, check_error=check_error) def get_matching(self, event_name_regex, check_error=True, timeout=None): - rex = re.compile("^(?:{})$".format(event_name_regex)) + rex = re.compile(f"^(?:{event_name_regex})$") for ev in self.iter_events(timeout=timeout, check_error=check_error): if rex.match(ev.name): return ev @@ -162,20 +163,20 @@ class FFIEventTracker: def ensure_event_not_queued(self, event_name_regex): __tracebackhide__ = True - rex = re.compile("(?:{}).*".format(event_name_regex)) + rex = re.compile(f"(?:{event_name_regex}).*") while 1: try: ev = self._event_queue.get(False) except Empty: break else: - assert not rex.match(ev.name), "event found {}".format(ev) + assert not rex.match(ev.name), f"event found {ev}" def wait_securejoin_inviter_progress(self, target): while 1: event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS") if event.data2 >= target: - print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account) + print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account) break def wait_idle_inbox_ready(self): @@ -221,7 +222,7 @@ class EventThread(threading.Thread): With each Account init this callback thread is started. """ - def __init__(self, account) -> None: + def __init__(self, account: Account) -> None: self.account = account super(EventThread, self).__init__(name="events") self.daemon = True @@ -247,36 +248,37 @@ class EventThread(threading.Thread): def run(self) -> None: """get and run events until shutdown.""" with self.log_execution("EVENT THREAD"): - self._inner_run() + event_emitter = ffi.gc( + lib.dc_get_event_emitter(self.account._dc_context), + lib.dc_event_emitter_unref, + ) + while not self._marked_for_shutdown: + with self.swallow_and_log_exception("Unexpected error in event thread"): + event = lib.dc_get_next_event(event_emitter) + if event == ffi.NULL or self._marked_for_shutdown: + break + self._process_event(event) - def _inner_run(self): - event_emitter = ffi.gc( - lib.dc_get_event_emitter(self.account._dc_context), - lib.dc_event_emitter_unref, - ) - while not self._marked_for_shutdown: - event = lib.dc_get_next_event(event_emitter) - if event == ffi.NULL or self._marked_for_shutdown: - break - evt = lib.dc_event_get_id(event) - data1 = lib.dc_event_get_data1_int(event) - # the following code relates to the deltachat/_build.py's helper - # function which provides us signature info of an event call - evt_name = get_dc_event_name(evt) - if lib.dc_event_has_string_data(evt): - data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event)) - else: - data2 = lib.dc_event_get_data2_int(event) + def _process_event(self, event) -> None: + evt = lib.dc_event_get_id(event) + data1 = lib.dc_event_get_data1_int(event) + # the following code relates to the deltachat/_build.py's helper + # function which provides us signature info of an event call + evt_name = get_dc_event_name(evt) + if lib.dc_event_has_string_data(evt): + data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event)) + else: + data2 = lib.dc_event_get_data2_int(event) - lib.dc_event_unref(event) - ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) - with self.swallow_and_log_exception("ac_process_ffi_event {}".format(ffi_event)): - self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event) - for name, kwargs in self._map_ffi_event(ffi_event): - hook = getattr(self.account._pm.hook, name) - info = "call {} kwargs={} failed".format(name, kwargs) - with self.swallow_and_log_exception(info): - hook(**kwargs) + lib.dc_event_unref(event) + ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) + with self.swallow_and_log_exception(f"ac_process_ffi_event {ffi_event}"): + self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event) + for name, kwargs in self._map_ffi_event(ffi_event): + hook = getattr(self.account._pm.hook, name) + info = f"call {name} kwargs={kwargs} failed" + with self.swallow_and_log_exception(info): + hook(**kwargs) @contextmanager def swallow_and_log_exception(self, info): @@ -285,7 +287,7 @@ class EventThread(threading.Thread): except Exception as ex: logfile = io.StringIO() traceback.print_exception(*sys.exc_info(), file=logfile) - self.account.log("{}\nException {}\nTraceback:\n{}".format(info, ex, logfile.getvalue())) + self.account.log(f"{info}\nException {ex}\nTraceback:\n{logfile.getvalue()}") def _map_ffi_event(self, ffi_event: FFIEvent): name = ffi_event.name @@ -298,20 +300,22 @@ class EventThread(threading.Thread): yield "ac_configure_completed", {"success": success, "comment": comment} elif name == "DC_EVENT_INCOMING_MSG": msg = account.get_message_by_id(ffi_event.data2) - yield map_system_message(msg) or ("ac_incoming_message", {"message": msg}) + if msg is not None: + yield map_system_message(msg) or ("ac_incoming_message", {"message": msg}) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = account.get_message_by_id(ffi_event.data2) - if msg.is_outgoing(): - res = map_system_message(msg) - if res and res[0].startswith("ac_member"): - yield res - yield "ac_outgoing_message", {"message": msg} - elif msg.is_in_fresh(): - yield map_system_message(msg) or ( - "ac_incoming_message", - {"message": msg}, - ) + if msg is not None: + if msg.is_outgoing(): + res = map_system_message(msg) + if res and res[0].startswith("ac_member"): + yield res + yield "ac_outgoing_message", {"message": msg} + elif msg.is_in_fresh(): + yield map_system_message(msg) or ( + "ac_incoming_message", + {"message": msg}, + ) elif name == "DC_EVENT_REACTIONS_CHANGED": assert ffi_event.data1 > 0 msg = account.get_message_by_id(ffi_event.data2) diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 0e083ad86..d84713bf8 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -36,21 +36,21 @@ class Message(object): def __repr__(self): c = self.get_sender_contact() typ = "outgoing" if self.is_outgoing() else "incoming" - return "".format( - typ, - self.is_system_message(), - repr(self.text[:10]), - self.id, - c.id, - c.addr, - self.chat.id, - self.chat.get_name(), + return ( + f"" ) @classmethod - def from_db(cls, account, id): + def from_db(cls, account, id) -> Optional["Message"]: + """Attempt to load the message from the database given its ID. + + None is returned if the message does not exist, i.e. deleted.""" assert id > 0 - return cls(account, ffi.gc(lib.dc_get_msg(account._dc_context, id), lib.dc_msg_unref)) + res = lib.dc_get_msg(account._dc_context, id) + if res == ffi.NULL: + return None + return cls(account, ffi.gc(res, lib.dc_msg_unref)) @classmethod def new_empty(cls, account, view_type): @@ -115,7 +115,7 @@ class Message(object): """set file for this message from path and mime_type.""" mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type) if not os.path.exists(path): - raise ValueError("path does not exist: {!r}".format(path)) + raise ValueError(f"path does not exist: {path!r}") lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype) @props.with_doc diff --git a/python/src/deltachat/reactions.py b/python/src/deltachat/reactions.py index 92cdf1905..968bcb0d5 100644 --- a/python/src/deltachat/reactions.py +++ b/python/src/deltachat/reactions.py @@ -18,7 +18,7 @@ class Reactions(object): self._dc_reactions = dc_reactions def __repr__(self): - return "".format(self._dc_reactions) + return f"" @classmethod def from_msg(cls, msg): diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index b61c1a0e4..fe1d85436 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -131,9 +131,9 @@ def pytest_report_header(config, startdir): if cfg: if "?" in cfg: url, token = cfg.split("?", 1) - summary.append("Liveconfig provider: {}?".format(url)) + summary.append(f"Liveconfig provider: {url}?") else: - summary.append("Liveconfig file: {}".format(cfg)) + summary.append(f"Liveconfig file: {cfg}") return summary @@ -178,13 +178,13 @@ class TestProcess: except IndexError: res = requests.post(liveconfig_opt, timeout=60) if res.status_code != 200: - pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text)) + pytest.fail(f"newtmpuser count={index} code={res.status_code}: '{res.text}'") d = res.json() config = {"addr": d["email"], "mail_pw": d["password"]} print("newtmpuser {}: addr={}".format(index, config["addr"])) self._configlist.append(config) yield config - pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS)) + pytest.fail(f"more than {MAX_LIVE_CREATED_ACCOUNTS} live accounts requested.") def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path): db_target_path = pathlib.Path(db_target_path) @@ -252,7 +252,7 @@ def data(request): fn = os.path.join(path, *bn.split("/")) if os.path.exists(fn): return fn - print("WARNING: path does not exist: {!r}".format(fn)) + print(f"WARNING: path does not exist: {fn!r}") return None def read_path(self, bn, mode="r"): @@ -285,7 +285,7 @@ class ACSetup: self.init_time = init_time def log(self, *args): - print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args) + print("[acsetup]", f"{time.time() - self.init_time:.3f}", *args) def add_configured(self, account): """add an already configured account.""" @@ -339,7 +339,7 @@ class ACSetup: def _pop_config_success(self): acc, success, comment = self._configured_events.get() if not success: - pytest.fail("configuring online account {} failed: {}".format(acc, comment)) + pytest.fail(f"configuring online account {acc} failed: {comment}") self._account2state[acc] = self.CONFIGURED return acc @@ -373,7 +373,7 @@ class ACSetup: imap.delete("1:*", expunge=True) else: imap.conn.folder.delete(folder) - acc.log("imap cleaned for addr {}".format(addr)) + acc.log(f"imap cleaned for addr {addr}") self._imap_cleaned.add(addr) @@ -397,7 +397,7 @@ class ACFactory: request.addfinalizer(self.finalize) def log(self, *args): - print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args) + print("[acfactory]", f"{time.time() - self.init_time:.3f}", *args) def finalize(self): while self._finalizers: @@ -439,7 +439,7 @@ class ACFactory: return self._getaccount(closed=closed) def _getaccount(self, try_cache_addr=None, closed=False): - logid = "ac{}".format(len(self._accounts) + 1) + logid = f"ac{len(self._accounts) + 1}" # we need to use fixed database basename for maybe_cache_* functions to work path = self.tmpdir.mkdir(logid).join("dc.db") if try_cache_addr: @@ -465,12 +465,12 @@ class ACFactory: except IndexError: pass else: - fname_pub = self.data.read_path("key/{name}-public.asc".format(name=keyname)) - fname_sec = self.data.read_path("key/{name}-secret.asc".format(name=keyname)) + fname_pub = self.data.read_path(f"key/{keyname}-public.asc") + fname_sec = self.data.read_path(f"key/{keyname}-secret.asc") if fname_pub and fname_sec: account._preconfigure_keypair(addr, fname_pub, fname_sec) return True - print("WARN: could not use preconfigured keys for {!r}".format(addr)) + print(f"WARN: could not use preconfigured keys for {addr!r}") def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account: # do a pseudo-configured account @@ -478,7 +478,7 @@ class ACFactory: if passphrase: ac.open(passphrase) acname = ac._logid - addr = "{}@offline.org".format(acname) + addr = f"{acname}@offline.org" ac.update_config( { "addr": addr, diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 199e5d5c9..24665ea64 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -38,7 +38,7 @@ class ImexTracker: if isinstance(ev, str): files_written.append(ev) elif ev == 0: - raise ImexFailed("export failed, exp-files: {}".format(files_written)) + raise ImexFailed(f"export failed, exp-files: {files_written}") elif ev == 1000: return files_written diff --git a/python/tests/stress_test_db.py b/python/tests/stress_test_db.py index eca48f2f4..f46bcdf65 100644 --- a/python/tests/stress_test_db.py +++ b/python/tests/stress_test_db.py @@ -59,15 +59,15 @@ def test_db_busy_error(acfactory, tmpdir): if report_type == ReportType.exit: replier.log("EXIT") elif report_type == ReportType.ffi_error: - replier.log("ERROR: {}".format(report_args[0])) + replier.log(f"ERROR: {report_args[0]}") elif report_type == ReportType.message_echo: continue else: - raise ValueError("{} unknown report type {}, args={}".format(addr, report_type, report_args)) + raise ValueError(f"{addr} unknown report type {report_type}, args={report_args}") alive_count -= 1 replier.log("shutting down") replier.account.shutdown() - replier.log("shut down complete, remaining={}".format(alive_count)) + replier.log(f"shut down complete, remaining={alive_count}") class ReportType: @@ -86,12 +86,12 @@ class AutoReplier: self.current_sent = 0 self.addr = self.account.get_self_contact().addr - self._thread = threading.Thread(name="Stats{}".format(self.account), target=self.thread_stats) + self._thread = threading.Thread(name=f"Stats{self.account}", target=self.thread_stats) self._thread.setDaemon(True) self._thread.start() def log(self, message): - self._log("{} {}".format(self.addr, message)) + self._log(f"{self.addr} {message}") def thread_stats(self): # XXX later use, for now we just quit @@ -107,17 +107,17 @@ class AutoReplier: return message.create_chat() message.mark_seen() - self.log("incoming message: {}".format(message)) + self.log(f"incoming message: {message}") self.current_sent += 1 # we are still alive, let's send a reply if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0: - message.chat.send_text("send big file as reply to: {}".format(message.text)) + message.chat.send_text(f"send big file as reply to: {message.text}") msg = message.chat.send_file(self.account.bigfile) else: - msg = message.chat.send_text("got message id {}, small text reply".format(message.id)) + msg = message.chat.send_text(f"got message id {message.id}, small text reply") assert msg.text - self.log("message-sent: {}".format(msg)) + self.log(f"message-sent: {msg}") self.report_func(self, ReportType.message_echo) if self.current_sent >= self.num_send: self.report_func(self, ReportType.exit) diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 42cb0f298..b38afe88c 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1249,7 +1249,7 @@ def test_send_mark_seen_clean_incoming_events(acfactory, lp): msg = ac2._evtracker.wait_next_incoming_message() assert msg.text == "hello" - lp.sec("ac2: mark seen {}".format(msg)) + lp.sec(f"ac2: mark seen {msg}") msg.mark_seen() for ev in ac1._evtracker.iter_events(): @@ -1437,9 +1437,8 @@ def test_import_export_online_all(acfactory, tmpdir, data, lp): backupdir = tmpdir.mkdir("backup") - lp.sec("export all to {}".format(backupdir)) + lp.sec(f"export all to {backupdir}") with ac1.temp_plugin(ImexTracker()) as imex_tracker: - ac1.stop_io() ac1.imex(backupdir.strpath, const.DC_IMEX_EXPORT_BACKUP) @@ -1475,7 +1474,7 @@ def test_import_export_online_all(acfactory, tmpdir, data, lp): assert_account_is_proper(ac1) assert_account_is_proper(ac2) - lp.sec("Second-time export all to {}".format(backupdir)) + lp.sec(f"Second-time export all to {backupdir}") ac1.stop_io() path2 = ac1.export_all(backupdir.strpath) assert os.path.exists(path2) diff --git a/python/tests/test_2_increation.py b/python/tests/test_2_increation.py index 2cc764ca3..33c264ef1 100644 --- a/python/tests/test_2_increation.py +++ b/python/tests/test_2_increation.py @@ -17,7 +17,7 @@ def wait_msg_delivered(account, msg_list): def wait_msgs_changed(account, msgs_list): """wait for one or more MSGS_CHANGED events to match msgs_list contents.""" - account.log("waiting for msgs_list={}".format(msgs_list)) + account.log(f"waiting for msgs_list={msgs_list}") msgs_list = list(msgs_list) while msgs_list: ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") @@ -27,7 +27,7 @@ def wait_msgs_changed(account, msgs_list): del msgs_list[i] break else: - account.log("waiting mismatch data1={} data2={}".format(data1, data2)) + account.log(f"waiting mismatch data1={data1} data2={data2}") return ev.data1, ev.data2 diff --git a/python/tests/test_3_offline.py b/python/tests/test_3_offline.py index 97dedfe37..8a01eb2a2 100644 --- a/python/tests/test_3_offline.py +++ b/python/tests/test_3_offline.py @@ -744,7 +744,7 @@ class TestOfflineChat: contacts = [] for i in range(10): lp.sec("create contact") - contact = ac1.create_contact("some{}@example.org".format(i)) + contact = ac1.create_contact(f"some{i}@example.org") contacts.append(contact) lp.sec("add contact") chat.add_contact(contact) diff --git a/python/tox.ini b/python/tox.ini index d01dd82e4..17905738f 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -56,7 +56,7 @@ deps = pygments restructuredtext_lint commands = - black --check setup.py install_python_bindings.py src/deltachat examples/ tests/ + black --check --diff setup.py install_python_bindings.py src/deltachat examples/ tests/ ruff src/deltachat tests/ examples/ rst-lint --encoding 'utf-8' README.rst diff --git a/scripts/README.md b/scripts/README.md index 2036d9fc7..9523e257c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,6 +6,8 @@ and an own build machine. ## Description of scripts +- `clippy.sh` runs `cargo clippy` for all Rust code in the project. + - `../.github/workflows` contains jobs run by GitHub Actions. - `remote_tests_python.sh` rsyncs to a build machine and runs diff --git a/scripts/clippy.sh b/scripts/clippy.sh new file mode 100755 index 000000000..d9cfcd5de --- /dev/null +++ b/scripts/clippy.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Run clippy for all Rust code in the project. +cargo clippy --workspace --tests --examples --benches -- -D warnings diff --git a/src/authres.rs b/src/authres.rs index eb267ddc9..2b2555544 100644 --- a/src/authres.rs +++ b/src/authres.rs @@ -644,7 +644,6 @@ Authentication-Results: dkim="; .unwrap(); } - #[ignore = "Disallowing keychanges is disabled for now"] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_handle_authres_fails() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -822,8 +821,7 @@ Authentication-Results: dkim="; .insert_str(0, "Authentication-Results: example.net; dkim=fail\n"); let rcvd = bob.recv_msg(&sent).await; - // Disallowing keychanges is disabled for now: - // assert!(rcvd.error.unwrap().contains("DKIM failed")); + assert!(rcvd.error.unwrap().contains("DKIM failed")); // The message info should contain a warning: assert!(message::get_msg_info(&bob, rcvd.id) .await diff --git a/src/chat.rs b/src/chat.rs index 60d0d4100..6d23b2048 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -19,7 +19,7 @@ use crate::color::str_to_color; use crate::config::Config; use crate::constants::{ Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_LAST_SPECIAL, - DC_CHAT_ID_TRASH, DC_GCM_ADDDAYMARKER, DC_GCM_INFO_ONLY, DC_RESEND_USER_AVATAR_DAYS, + DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS, }; use crate::contact::{Contact, ContactId, Origin, VerifiedStatus}; use crate::context::Context; @@ -2376,12 +2376,40 @@ pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Re send_msg(context, chat_id, &mut msg).await } -pub async fn get_chat_msgs( +/// Chat message list request options. +#[derive(Debug)] +pub struct MessageListOptions { + /// Return only info messages. + pub info_only: bool, + + /// Add day markers before each date regarding the local timezone. + pub add_daymarker: bool, +} + +/// Returns all messages belonging to the chat. +pub async fn get_chat_msgs(context: &Context, chat_id: ChatId) -> Result> { + get_chat_msgs_ex( + context, + chat_id, + MessageListOptions { + info_only: false, + add_daymarker: false, + }, + ) + .await +} + +/// Returns messages belonging to the chat according to the given options. +pub async fn get_chat_msgs_ex( context: &Context, chat_id: ChatId, - flags: u32, + options: MessageListOptions, ) -> Result> { - let process_row = if (flags & DC_GCM_INFO_ONLY) != 0 { + let MessageListOptions { + info_only, + add_daymarker, + } = options; + let process_row = if info_only { |row: &rusqlite::Row| { // is_info logic taken from Message.is_info() let params = row.get::<_, String>("param")?; @@ -2431,7 +2459,7 @@ pub async fn get_chat_msgs( let cnv_to_local = gm2local_offset(); for (ts, curr_id) in sorted_rows { - if (flags & DC_GCM_ADDDAYMARKER) != 0 { + if add_daymarker { let curr_local_timestamp = ts + cnv_to_local; let curr_day = curr_local_timestamp / 86400; if curr_day != last_day { @@ -2446,7 +2474,7 @@ pub async fn get_chat_msgs( Ok(ret) }; - let items = if (flags & DC_GCM_INFO_ONLY) != 0 { + let items = if info_only { context .sql .query_map( @@ -4942,7 +4970,7 @@ mod tests { assert!(chat.is_protected()); assert!(chat.is_unpromoted()); - let msgs = get_chat_msgs(&t, chat_id, 0).await?; + let msgs = get_chat_msgs(&t, chat_id).await?; assert_eq!(msgs.len(), 1); let msg = t.get_last_msg_in(chat_id).await; @@ -4971,7 +4999,7 @@ mod tests { assert!(!chat.is_protected()); assert!(!chat.is_unpromoted()); - let msgs = get_chat_msgs(&t, chat_id, 0).await?; + let msgs = get_chat_msgs(&t, chat_id).await?; assert_eq!(msgs.len(), 3); // enable protection on promoted chat, the info-message is sent via send_msg() this time @@ -5069,7 +5097,7 @@ mod tests { add_contact_to_chat(&alice, alice_chat_id, contact_id).await?; assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2); send_text_msg(&alice, alice_chat_id, "hi!".to_string()).await?; - assert_eq!(get_chat_msgs(&alice, alice_chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&alice, alice_chat_id).await?.len(), 1); // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com). let sent_msg = alice.pop_sent_msg().await; @@ -5102,7 +5130,7 @@ mod tests { let msg = alice.get_last_msg().await; assert_eq!(msg.chat_id, alice_chat_id); assert_eq!(msg.text, Some("ho!".to_string())); - assert_eq!(get_chat_msgs(&alice, alice_chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&alice, alice_chat_id).await?.len(), 2); Ok(()) } @@ -5130,7 +5158,7 @@ mod tests { assert_eq!(chat.id.get_fresh_msg_cnt(&t).await?, 1); assert_eq!(t.get_fresh_msgs().await?.len(), 1); - let msgs = get_chat_msgs(&t, chat.id, 0).await?; + let msgs = get_chat_msgs(&t, chat.id).await?; assert_eq!(msgs.len(), 1); let msg_id = match msgs.first().unwrap() { ChatItem::Message { msg_id } => *msg_id, @@ -5180,7 +5208,7 @@ mod tests { .is_contact_request()); assert_eq!(chat_id.get_msg_cnt(&t).await?, 1); assert_eq!(chat_id.get_fresh_msg_cnt(&t).await?, 1); - let msgs = get_chat_msgs(&t, chat_id, 0).await?; + let msgs = get_chat_msgs(&t, chat_id).await?; assert_eq!(msgs.len(), 1); let msg_id = match msgs.first().unwrap() { ChatItem::Message { msg_id } => *msg_id, @@ -5268,7 +5296,7 @@ mod tests { let chat_id = msg.chat_id; assert_eq!(chat_id.get_fresh_msg_cnt(&alice).await?, 1); - let msgs = get_chat_msgs(&alice, chat_id, 0).await?; + let msgs = get_chat_msgs(&alice, chat_id).await?; assert_eq!(msgs.len(), 1); // Alice disables receiving classic emails. @@ -5280,7 +5308,7 @@ mod tests { // Already received classic email should still be in the chat. assert_eq!(chat_id.get_fresh_msg_cnt(&alice).await?, 1); - let msgs = get_chat_msgs(&alice, chat_id, 0).await?; + let msgs = get_chat_msgs(&alice, chat_id).await?; assert_eq!(msgs.len(), 1); Ok(()) @@ -5427,7 +5455,7 @@ mod tests { assert!(msg1.get_text().unwrap().contains("bob@example.net")); let chat_id2 = ChatId::create_for_contact(&t, bob_id).await?; - assert_eq!(get_chat_msgs(&t, chat_id2, 0).await?.len(), 0); + assert_eq!(get_chat_msgs(&t, chat_id2).await?.len(), 0); forward_msgs(&t, &[msg1.id], chat_id2).await?; let msg2 = t.get_last_msg_in(chat_id2).await; assert!(!msg2.is_info()); // forwarded info-messages lose their info-state @@ -5596,15 +5624,15 @@ mod tests { let msg = bob.recv_msg(&sent1).await; assert_eq!(msg.get_text().unwrap(), "alice->bob"); assert_eq!(get_chat_contacts(&bob, msg.chat_id).await?.len(), 2); - assert_eq!(get_chat_msgs(&bob, msg.chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&bob, msg.chat_id).await?.len(), 1); bob.recv_msg(&sent2).await; assert_eq!(get_chat_contacts(&bob, msg.chat_id).await?.len(), 3); - assert_eq!(get_chat_msgs(&bob, msg.chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&bob, msg.chat_id).await?.len(), 2); let received = bob.recv_msg_opt(&sent3).await; // No message should actually be added since we already know this message: assert!(received.is_none()); assert_eq!(get_chat_contacts(&bob, msg.chat_id).await?.len(), 3); - assert_eq!(get_chat_msgs(&bob, msg.chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&bob, msg.chat_id).await?.len(), 2); // Claire does not receive the first message, however, due to resending, she has a similar view as Alice and Bob let claire = TestContext::new().await; @@ -5613,7 +5641,7 @@ mod tests { let msg = claire.recv_msg(&sent3).await; assert_eq!(msg.get_text().unwrap(), "alice->bob"); assert_eq!(get_chat_contacts(&claire, msg.chat_id).await?.len(), 3); - assert_eq!(get_chat_msgs(&claire, msg.chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&claire, msg.chat_id).await?.len(), 2); let msg_from = Contact::get_by_id(&claire, msg.get_from_id()).await?; assert_eq!(msg_from.get_addr(), "alice@example.org"); diff --git a/src/configure/read_url.rs b/src/configure/read_url.rs index b0cdd989f..d164a7007 100644 --- a/src/configure/read_url.rs +++ b/src/configure/read_url.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, format_err}; use crate::context::Context; +use crate::socks::Socks5Config; pub async fn read_url(context: &Context, url: &str) -> anyhow::Result { match read_url_inner(context, url).await { @@ -16,7 +17,8 @@ pub async fn read_url(context: &Context, url: &str) -> anyhow::Result { } pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result { - let client = crate::http::get_client()?; + let socks5_config = Socks5Config::from_database(&context.sql).await?; + let client = crate::http::get_client(socks5_config)?; let mut url = url.to_string(); // Follow up to 10 http-redirects diff --git a/src/configure/server_params.rs b/src/configure/server_params.rs index 9ecaa5e1c..e8e98a755 100644 --- a/src/configure/server_params.rs +++ b/src/configure/server_params.rs @@ -99,15 +99,6 @@ impl ServerParams { // Try common secure combinations. vec![ - // Try STARTTLS - Self { - socket: Socket::Starttls, - port: match self.protocol { - Protocol::Imap => 143, - Protocol::Smtp => 587, - }, - ..self.clone() - }, // Try TLS Self { socket: Socket::Ssl, @@ -115,6 +106,15 @@ impl ServerParams { Protocol::Imap => 993, Protocol::Smtp => 465, }, + ..self.clone() + }, + // Try STARTTLS + Self { + socket: Socket::Starttls, + port: match self.protocol { + Protocol::Imap => 143, + Protocol::Smtp => 587, + }, ..self }, ] @@ -343,5 +343,41 @@ mod tests { } ], ); + + // Test that TLS is preferred to STARTTLS + // when the port and security are not set. + let v = expand_param_vector( + vec![ServerParams { + protocol: Protocol::Smtp, + hostname: "example.net".to_string(), + port: 0, + socket: Socket::Automatic, + username: "foobar".to_string(), + strict_tls: Some(true), + }], + "foobar@example.net", + "example.net", + ); + assert_eq!( + v, + vec![ + ServerParams { + protocol: Protocol::Smtp, + hostname: "example.net".to_string(), + port: 465, + socket: Socket::Ssl, + username: "foobar".to_string(), + strict_tls: Some(true) + }, + ServerParams { + protocol: Protocol::Smtp, + hostname: "example.net".to_string(), + port: 587, + socket: Socket::Starttls, + username: "foobar".to_string(), + strict_tls: Some(true) + }, + ], + ); } } diff --git a/src/constants.rs b/src/constants.rs index a7c60a8dd..8f76c68ac 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -112,9 +112,6 @@ pub const DC_GCL_NO_SPECIALS: usize = 0x02; pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04; pub const DC_GCL_FOR_FORWARDING: usize = 0x08; -pub const DC_GCM_ADDDAYMARKER: u32 = 0x01; -pub const DC_GCM_INFO_ONLY: u32 = 0x02; - pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01; pub const DC_GCL_ADD_SELF: u32 = 0x02; diff --git a/src/context.rs b/src/context.rs index 2b3bb17dd..20413cc08 100644 --- a/src/context.rs +++ b/src/context.rs @@ -985,20 +985,20 @@ mod tests { assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0); receive_msg(&t, &bob).await; - assert_eq!(get_chat_msgs(&t, bob.id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1); assert_eq!(bob.id.get_fresh_msg_cnt(&t).await.unwrap(), 1); assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1); receive_msg(&t, &claire).await; receive_msg(&t, &claire).await; - assert_eq!(get_chat_msgs(&t, claire.id, 0).await.unwrap().len(), 2); + assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 2); assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2); assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3); receive_msg(&t, &dave).await; receive_msg(&t, &dave).await; receive_msg(&t, &dave).await; - assert_eq!(get_chat_msgs(&t, dave.id, 0).await.unwrap().len(), 3); + assert_eq!(get_chat_msgs(&t, dave.id).await.unwrap().len(), 3); assert_eq!(dave.id.get_fresh_msg_cnt(&t).await.unwrap(), 3); assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); @@ -1013,7 +1013,7 @@ mod tests { receive_msg(&t, &bob).await; receive_msg(&t, &claire).await; receive_msg(&t, &dave).await; - assert_eq!(get_chat_msgs(&t, claire.id, 0).await.unwrap().len(), 3); + assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 3); assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3); assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted @@ -1030,7 +1030,7 @@ mod tests { let t = TestContext::new_alice().await; let bob = t.create_chat_with_contact("", "bob@g.it").await; receive_msg(&t, &bob).await; - assert_eq!(get_chat_msgs(&t, bob.id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1); // chat is unmuted by default, here and in the following assert(), // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs() diff --git a/src/decrypt.rs b/src/decrypt.rs index 1165e3dd7..fa1232830 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -99,8 +99,7 @@ pub(crate) async fn prepare_decryption( from, autocrypt_header.as_ref(), message_time, - // Disallowing keychanges is disabled for now: - true, // dkim_results.allow_keychange, + dkim_results.allow_keychange, ) .await?; diff --git a/src/download.rs b/src/download.rs index 86f33a451..1701dd644 100644 --- a/src/download.rs +++ b/src/download.rs @@ -450,7 +450,7 @@ mod tests { .await?; let msg = bob.get_last_msg().await; let chat_id = msg.chat_id; - assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1); assert_eq!(msg.download_state(), DownloadState::Available); // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat @@ -464,7 +464,7 @@ mod tests { false, ) .await?; - assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0); + assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0); assert!(Message::load_from_db(&bob, msg.id) .await? .chat_id @@ -517,13 +517,13 @@ mod tests { .await?; let msg = bob.get_last_msg().await; let chat_id = msg.chat_id; - assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1); assert_eq!(msg.download_state(), DownloadState::Available); // downloading the mdn afterwards expands to nothing and deletes the placeholder directly // (usually mdn are too small for not being downloaded directly) receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?; - assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0); + assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0); assert!(Message::load_from_db(&bob, msg.id) .await? .chat_id diff --git a/src/ephemeral.rs b/src/ephemeral.rs index ea1754de6..2e38386ea 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -1080,7 +1080,7 @@ mod tests { } async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) { - let chat_items = chat::get_chat_msgs(t, chat.id, 0).await.unwrap(); + let chat_items = chat::get_chat_msgs(t, chat.id).await.unwrap(); // Check that the chat is empty except for possibly info messages: for item in &chat_items { if let ChatItem::Message { msg_id } = item { diff --git a/src/http.rs b/src/http.rs index 17e73a10d..8eed8b55e 100644 --- a/src/http.rs +++ b/src/http.rs @@ -4,10 +4,21 @@ use std::time::Duration; use anyhow::Result; +use crate::socks::Socks5Config; + const HTTP_TIMEOUT: Duration = Duration::from_secs(30); -pub(crate) fn get_client() -> Result { - Ok(reqwest::ClientBuilder::new() - .timeout(HTTP_TIMEOUT) - .build()?) +pub(crate) fn get_client(socks5_config: Option) -> Result { + let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT); + let builder = if let Some(socks5_config) = socks5_config { + let proxy = reqwest::Proxy::all(socks5_config.to_url())?; + builder.proxy(proxy) + } else { + // Disable usage of "system" proxy configured via environment variables. + // It is enabled by default in `reqwest`, see + // + // for documentation. + builder.no_proxy() + }; + Ok(builder.build()?) } diff --git a/src/login_param.rs b/src/login_param.rs index 297ffa77f..ee674c1fc 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -4,7 +4,6 @@ use std::fmt; use anyhow::{ensure, Result}; use async_native_tls::Certificate; -pub use async_smtp::ServerAddress; use once_cell::sync::Lazy; use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2}; @@ -163,7 +162,7 @@ impl LoginParam { .await? .and_then(|provider_id| get_provider_by_id(&provider_id)); - let socks5_config = Socks5Config::from_database(context).await?; + let socks5_config = Socks5Config::from_database(&context.sql).await?; Ok(LoginParam { addr, @@ -263,7 +262,7 @@ impl fmt::Display for LoginParam { write!( f, - "{} imap:{}:{}:{}:{}:cert_{}:{} smtp:{}:{}:{}:{}:cert_{}:{}", + "{} imap:{}:{}:{}:{}:{}:cert_{}:{} smtp:{}:{}:{}:{}:{}:cert_{}:{}", unset_empty(&self.addr), unset_empty(&self.imap.user), if !self.imap.password.is_empty() { @@ -273,6 +272,7 @@ impl fmt::Display for LoginParam { }, unset_empty(&self.imap.server), self.imap.port, + self.imap.security, self.imap.certificate_checks, if self.imap.oauth2 { "OAUTH2" @@ -287,6 +287,7 @@ impl fmt::Display for LoginParam { }, unset_empty(&self.smtp.server), self.smtp.port, + self.smtp.security, self.smtp.certificate_checks, if self.smtp.oauth2 { "OAUTH2" diff --git a/src/message.rs b/src/message.rs index e4df6c9d8..1b1a1118c 100644 --- a/src/message.rs +++ b/src/message.rs @@ -2144,7 +2144,7 @@ mod tests { .unwrap(); let mut has_image = false; - let chatitems = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let chatitems = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); for chatitem in chatitems { if let ChatItem::Message { msg_id } = chatitem { if let Ok(msg) = Message::load_from_db(&t, msg_id).await { @@ -2288,7 +2288,7 @@ mod tests { assert_eq!(msg1.chat_id, msg2.chat_id); let chats = Chatlist::try_load(&bob, 0, None, None).await?; assert_eq!(chats.len(), 1); - let msgs = chat::get_chat_msgs(&bob, bob_chat_id, 0).await?; + let msgs = chat::get_chat_msgs(&bob, bob_chat_id).await?; assert_eq!(msgs.len(), 2); assert_eq!(bob.get_fresh_msgs().await?.len(), 0); @@ -2299,7 +2299,7 @@ mod tests { let bob_chat = Chat::load_from_db(&bob, bob_chat_id).await?; assert_eq!(bob_chat.blocked, Blocked::Request); - let msgs = chat::get_chat_msgs(&bob, bob_chat_id, 0).await?; + let msgs = chat::get_chat_msgs(&bob, bob_chat_id).await?; assert_eq!(msgs.len(), 2); bob_chat_id.accept(&bob).await.unwrap(); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 5742e4cb1..d428fc080 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -325,8 +325,7 @@ impl MimeMessage { if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) { if message_time > peerstate.last_seen_autocrypt && mail.ctype.mimetype != "multipart/report" - // Disallowing keychanges is disabled for now: - // && decryption_info.dkim_results.allow_keychange + && decryption_info.dkim_results.allow_keychange { peerstate.degrade_encryption(message_time); } @@ -397,12 +396,11 @@ impl MimeMessage { parser.heuristically_parse_ndn(context).await; parser.parse_headers(context).await?; - // Disallowing keychanges is disabled for now - // if !decryption_info.dkim_results.allow_keychange { - // for part in parser.parts.iter_mut() { - // part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string()); - // } - // } + if !parser.decryption_info.dkim_results.allow_keychange { + for part in parser.parts.iter_mut() { + part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string()); + } + } if parser.is_mime_modified { parser.decoded_data = mail_raw; diff --git a/src/oauth2.rs b/src/oauth2.rs index bde9f2240..70c58c1d2 100644 --- a/src/oauth2.rs +++ b/src/oauth2.rs @@ -12,6 +12,7 @@ use crate::config::Config; use crate::context::Context; use crate::provider; use crate::provider::Oauth2Authorizer; +use crate::socks::Socks5Config; use crate::tools::time; const OAUTH2_GMAIL: Oauth2 = Oauth2 { @@ -158,7 +159,8 @@ pub async fn get_oauth2_access_token( } // ... and POST - let client = crate::http::get_client()?; + let socks5_config = Socks5Config::from_database(&context.sql).await?; + let client = crate::http::get_client(socks5_config)?; let response: Response = match client.post(post_url).form(&post_param).send().await { Ok(resp) => match resp.json().await { @@ -284,7 +286,8 @@ impl Oauth2 { // "verified_email": true, // "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg" // } - let client = match crate::http::get_client() { + let socks5_config = Socks5Config::from_database(&context.sql).await.ok()?; + let client = match crate::http::get_client(socks5_config) { Ok(cl) => cl, Err(err) => { warn!(context, "failed to get HTTP client: {}", err); diff --git a/src/qr.rs b/src/qr.rs index 4baabf487..00e063513 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -22,6 +22,7 @@ use crate::context::Context; use crate::key::Fingerprint; use crate::message::Message; use crate::peerstate::Peerstate; +use crate::socks::Socks5Config; use crate::tools::time; use crate::{token, EventType}; @@ -395,7 +396,11 @@ struct CreateAccountErrorResponse { #[allow(clippy::indexing_slicing)] async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> { let url_str = &qr[DCACCOUNT_SCHEME.len()..]; - let response = crate::http::get_client()?.post(url_str).send().await?; + let socks5_config = Socks5Config::from_database(&context.sql).await?; + let response = crate::http::get_client(socks5_config)? + .post(url_str) + .send() + .await?; let response_status = response.status(); let response_text = response.text().await.with_context(|| { format!("Cannot create account, request to {url_str:?} failed: empty response") diff --git a/src/reaction.rs b/src/reaction.rs index d75294f9a..bd1a98c10 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -443,24 +443,24 @@ Content-Disposition: reaction\n\ let chat_alice = alice.create_chat(&bob).await; let alice_msg = alice.send_text(chat_alice.id, "Hi!").await; let bob_msg = bob.recv_msg(&alice_msg).await; - assert_eq!(get_chat_msgs(&alice, chat_alice.id, 0).await?.len(), 1); - assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&alice, chat_alice.id).await?.len(), 1); + assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id).await?.len(), 1); let alice_msg2 = alice.send_text(chat_alice.id, "Hi again!").await; bob.recv_msg(&alice_msg2).await; - assert_eq!(get_chat_msgs(&alice, chat_alice.id, 0).await?.len(), 2); - assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&alice, chat_alice.id).await?.len(), 2); + assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id).await?.len(), 2); bob_msg.chat_id.accept(&bob).await?; send_reaction(&bob, bob_msg.id, "👍").await.unwrap(); expect_reactions_changed_event(&bob, bob_msg.chat_id, bob_msg.id, ContactId::SELF).await?; - assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id).await?.len(), 2); let bob_reaction_msg = bob.pop_sent_msg().await; let alice_reaction_msg = alice.recv_msg_opt(&bob_reaction_msg).await.unwrap(); assert_eq!(alice_reaction_msg.chat_id, DC_CHAT_ID_TRASH); - assert_eq!(get_chat_msgs(&alice, chat_alice.id, 0).await?.len(), 2); + assert_eq!(get_chat_msgs(&alice, chat_alice.id).await?.len(), 2); let reactions = get_msg_reactions(&alice, alice_msg.sender_msg_id).await?; assert_eq!(reactions.to_string(), "👍1"); diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index c89791982..342b2fe36 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -154,12 +154,12 @@ async fn test_adhoc_group_show_accepted_contact_accepted() { assert_eq!(chat.typ, Chattype::Single); assert_eq!(chat.name, "Bob"); assert_eq!(chat::get_chat_contacts(&t, chat_id).await.unwrap().len(), 1); - assert_eq!(chat::get_chat_msgs(&t, chat_id, 0).await.unwrap().len(), 1); + assert_eq!(chat::get_chat_msgs(&t, chat_id).await.unwrap().len(), 1); // receive a non-delta-message from Bob, shows up because of the show_emails setting receive_imf(&t, ONETOONE_NOREPLY_MAIL, false).await.unwrap(); - assert_eq!(chat::get_chat_msgs(&t, chat_id, 0).await.unwrap().len(), 2); + assert_eq!(chat::get_chat_msgs(&t, chat_id).await.unwrap().len(), 2); // let Bob create an adhoc-group by a non-delta-message, shows up because of the show_emails setting receive_imf(&t, GRP_MAIL, false).await.unwrap(); @@ -208,7 +208,7 @@ async fn test_read_receipt_and_unarchive() -> Result<()> { // create a group with bob, archive group let group_id = chat::create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?; chat::add_contact_to_chat(&t, group_id, bob_id).await?; - assert_eq!(chat::get_chat_msgs(&t, group_id, 0).await.unwrap().len(), 0); + assert_eq!(chat::get_chat_msgs(&t, group_id).await.unwrap().len(), 0); group_id .set_visibility(&t, ChatVisibility::Archived) .await?; @@ -289,7 +289,7 @@ async fn test_read_receipt_and_unarchive() -> Result<()> { false, ) .await?; - assert_eq!(chat::get_chat_msgs(&t, group_id, 0).await?.len(), 1); + assert_eq!(chat::get_chat_msgs(&t, group_id).await?.len(), 1); let msg = message::Message::load_from_db(&t, msg.id).await?; assert_eq!(msg.state, MessageState::OutMdnRcvd); @@ -705,7 +705,7 @@ async fn test_parse_ndn_group_msg() -> Result<()> { assert_eq!(msg.state, MessageState::OutFailed); - let msgs = chat::get_chat_msgs(&t, msg.chat_id, 0).await?; + let msgs = chat::get_chat_msgs(&t, msg.chat_id).await?; let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() { msg_id } else { @@ -966,7 +966,7 @@ async fn test_block_mailing_list() { assert_eq!(chats.len(), 0); // Test that the message is not shown // Both messages are in the same blocked chat. - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), 2); } @@ -997,7 +997,7 @@ async fn test_mailing_list_decide_block_then_unblock() { receive_imf(&t.ctx, DC_MAILINGLIST2, false).await.unwrap(); let msg = t.get_last_msg().await; - let msgs = chat::get_chat_msgs(&t, msg.chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, msg.chat_id).await.unwrap(); assert_eq!(msgs.len(), 2); } @@ -1019,14 +1019,14 @@ async fn test_mailing_list_decide_not_now() { let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); // Test that chat is still in the chatlist - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), 1); // ...and contains 1 message receive_imf(&t.ctx, DC_MAILINGLIST2, false).await.unwrap(); let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); // Test that the new mailing list message got into the same chat - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), 2); let chat = Chat::load_from_db(&t.ctx, chat_id).await.unwrap(); assert!(chat.is_contact_request()); @@ -1052,7 +1052,7 @@ async fn test_mailing_list_decide_accept() { receive_imf(&t.ctx, DC_MAILINGLIST2, false).await.unwrap(); - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), 2); let chat = chat::Chat::load_from_db(&t.ctx, chat_id).await.unwrap(); assert!(chat.can_send(&t.ctx).await.unwrap()); @@ -1110,7 +1110,7 @@ async fn test_majordomo_mailing_list() -> Result<()> { assert_eq!(chat.typ, Chattype::Mailinglist); assert_eq!(chat.grpid, "mylist@bar.org"); assert_eq!(chat.name, "ola"); - assert_eq!(chat::get_chat_msgs(&t, chat.id, 0).await.unwrap().len(), 1); + assert_eq!(chat::get_chat_msgs(&t, chat.id).await.unwrap().len(), 1); assert!(!chat.can_send(&t).await?); assert_eq!(chat.get_mailinglist_addr(), None); @@ -1131,7 +1131,7 @@ async fn test_majordomo_mailing_list() -> Result<()> { ) .await .unwrap(); - assert_eq!(chat::get_chat_msgs(&t, chat.id, 0).await.unwrap().len(), 2); + assert_eq!(chat::get_chat_msgs(&t, chat.id).await.unwrap().len(), 2); Ok(()) } @@ -1330,7 +1330,7 @@ async fn test_mailing_list_with_mimepart_footer() { ); assert!(msg.has_html()); let chat = Chat::load_from_db(&t, msg.chat_id).await.unwrap(); - assert_eq!(get_chat_msgs(&t, msg.chat_id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&t, msg.chat_id).await.unwrap().len(), 1); assert_eq!(chat.typ, Chattype::Mailinglist); assert_eq!(chat.blocked, Blocked::Request); assert_eq!(chat.grpid, "intern.lists.abc.de"); @@ -1350,7 +1350,7 @@ async fn test_mailing_list_with_mimepart_footer_signed() { .await .unwrap(); let msg = t.get_last_msg().await; - assert_eq!(get_chat_msgs(&t, msg.chat_id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&t, msg.chat_id).await.unwrap().len(), 1); let text = msg.text.clone().unwrap(); assert!(text.contains("content text")); assert!(!text.contains("footer text")); @@ -1547,7 +1547,7 @@ async fn test_many_images() { assert_eq!(msg.viewtype, Viewtype::Image); assert!(msg.has_html()); let chat = Chat::load_from_db(&t, msg.chat_id).await.unwrap(); - assert_eq!(get_chat_msgs(&t, chat.id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&t, chat.id).await.unwrap().len(), 1); } /// Test that classical MUA messages are assigned to group chats based on the `In-Reply-To` @@ -1597,7 +1597,7 @@ async fn test_in_reply_to() { assert_eq!(msg.get_text().unwrap(), "reply foo"); // Load the first message from the same chat. - let msgs = chat::get_chat_msgs(&t, msg.chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, msg.chat_id).await.unwrap(); let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() { msg_id } else { @@ -1824,7 +1824,7 @@ async fn create_test_alias(chat_request: bool, group_request: bool) -> (TestCont assert!(msg.get_text().unwrap().contains("hi support!")); let chat = Chat::load_from_db(&alice, msg.chat_id).await.unwrap(); assert_eq!(chat.typ, Chattype::Group); - assert_eq!(get_chat_msgs(&alice, chat.id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&alice, chat.id).await.unwrap().len(), 1); if group_request { assert_eq!(get_chat_contacts(&alice, chat.id).await.unwrap().len(), 4); } else { @@ -1857,7 +1857,7 @@ async fn create_test_alias(chat_request: bool, group_request: bool) -> (TestCont } else { assert_eq!(chat.typ, Chattype::Single); } - assert_eq!(get_chat_msgs(&claire, chat.id, 0).await.unwrap().len(), 1); + assert_eq!(get_chat_msgs(&claire, chat.id).await.unwrap().len(), 1); assert_eq!(msg.get_override_sender_name(), None); (claire, alice) @@ -1972,7 +1972,7 @@ async fn test_dont_assign_to_trash_by_parent() { println!("\n========= Delete the message =========="); msg.id.trash(&t).await.unwrap(); - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), 0); println!("\n========= Receive a message that is a reply to the deleted message =========="); @@ -2182,8 +2182,8 @@ Original signature updated", .await?; let bob = Contact::load_from_db(&t, bob_id).await?; assert_eq!(bob.get_status(), "Original signature updated"); - assert_eq!(get_chat_msgs(&t, one2one_chat_id, 0).await?.len(), 2); - assert_eq!(get_chat_msgs(&t, ml_chat_id, 0).await?.len(), 1); + assert_eq!(get_chat_msgs(&t, one2one_chat_id).await?.len(), 2); + assert_eq!(get_chat_msgs(&t, ml_chat_id).await?.len(), 1); assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 2); Ok(()) } diff --git a/src/securejoin.rs b/src/securejoin.rs index b4261fc88..834436136 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -779,7 +779,7 @@ mod tests { use crate::chat; use crate::chat::ProtectionStatus; use crate::chatlist::Chatlist; - use crate::constants::{Chattype, DC_GCM_ADDDAYMARKER}; + use crate::constants::Chattype; use crate::contact::ContactAddress; use crate::contact::VerifiedStatus; use crate::peerstate::Peerstate; @@ -922,7 +922,7 @@ mod tests { // Check Alice got the verified message in her 1:1 chat. { let chat = alice.create_chat(&bob).await; - let msg_id = chat::get_chat_msgs(&alice.ctx, chat.get_id(), DC_GCM_ADDDAYMARKER) + let msg_id = chat::get_chat_msgs(&alice.ctx, chat.get_id()) .await .unwrap() .into_iter() @@ -971,7 +971,7 @@ mod tests { // Check Bob got the verified message in his 1:1 chat. { let chat = bob.create_chat(&alice).await; - let msg_id = chat::get_chat_msgs(&bob.ctx, chat.get_id(), DC_GCM_ADDDAYMARKER) + let msg_id = chat::get_chat_msgs(&bob.ctx, chat.get_id()) .await .unwrap() .into_iter() @@ -1281,7 +1281,7 @@ mod tests { Blocked::Yes, "Alice's 1:1 chat with Bob is not hidden" ); - let msg_id = chat::get_chat_msgs(&alice.ctx, alice_chatid, DC_GCM_ADDDAYMARKER) + let msg_id = chat::get_chat_msgs(&alice.ctx, alice_chatid) .await .unwrap() .into_iter() @@ -1326,17 +1326,14 @@ mod tests { Blocked::Yes, "Bob's 1:1 chat with Alice is not hidden" ); - for item in chat::get_chat_msgs(&bob.ctx, bob_chatid, DC_GCM_ADDDAYMARKER) - .await - .unwrap() - { + for item in chat::get_chat_msgs(&bob.ctx, bob_chatid).await.unwrap() { if let chat::ChatItem::Message { msg_id } = item { let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap(); let text = msg.get_text().unwrap(); println!("msg {msg_id} text: {text}"); } } - let mut msg_iter = chat::get_chat_msgs(&bob.ctx, bob_chatid, DC_GCM_ADDDAYMARKER) + let mut msg_iter = chat::get_chat_msgs(&bob.ctx, bob_chatid) .await .unwrap() .into_iter(); diff --git a/src/smtp.rs b/src/smtp.rs index f123c8c6f..5cf0dd73f 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -5,9 +5,9 @@ pub mod send; use std::time::{Duration, SystemTime}; use anyhow::{bail, format_err, Context as _, Error, Result}; -use async_smtp::smtp::client::net::ClientTlsParameters; -use async_smtp::smtp::response::{Category, Code, Detail}; -use async_smtp::{smtp, EmailAddress, ServerAddress}; +use async_smtp::response::{Category, Code, Detail}; +use async_smtp::{self as smtp, EmailAddress, SmtpTransport}; +use tokio::io::BufWriter; use tokio::task; use crate::config::Config; @@ -17,18 +17,21 @@ use crate::login_param::{build_tls, CertificateChecks, LoginParam, ServerLoginPa use crate::message::Message; use crate::message::{self, MsgId}; use crate::mimefactory::MimeFactory; +use crate::net::connect_tcp; +use crate::net::session::SessionStream; use crate::oauth2::get_oauth2_access_token; use crate::provider::Socket; use crate::socks::Socks5Config; use crate::sql; use crate::{context::Context, scheduler::connectivity::ConnectivityStore}; -/// SMTP write and read timeout in seconds. -const SMTP_TIMEOUT: u64 = 30; +/// SMTP write and read timeout. +const SMTP_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Default)] pub(crate) struct Smtp { - transport: Option, + /// SMTP connection. + transport: Option>>, /// Email address we are sending from. from: Option, @@ -56,7 +59,7 @@ impl Smtp { // Closing connection with a QUIT command may take some time, especially if it's a // stale connection and an attempt to send the command times out. Send a command in a // separate task to avoid waiting for reply or timeout. - task::spawn(async move { transport.close().await }); + task::spawn(async move { transport.quit().await }); } self.last_success = None; } @@ -77,10 +80,7 @@ impl Smtp { /// Check whether we are connected. pub fn is_connected(&self) -> bool { - self.transport - .as_ref() - .map(|t| t.is_connected()) - .unwrap_or_default() + self.transport.is_some() } /// Connect using configured parameters. @@ -107,6 +107,127 @@ impl Smtp { .await } + async fn connect_secure_socks5( + &self, + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + socks5_config: Socks5Config, + ) -> Result>> { + let socks5_stream = socks5_config + .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls) + .await?; + let tls = build_tls(strict_tls); + let tls_stream = tls.connect(hostname, socks5_stream).await?; + let buffered_stream = BufWriter::new(tls_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + + async fn connect_starttls_socks5( + &self, + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + socks5_config: Socks5Config, + ) -> Result>> { + let socks5_stream = socks5_config + .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls) + .await?; + + // Run STARTTLS command and convert the client back into a stream. + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, socks5_stream).await?; + let tcp_stream = transport.starttls().await?; + let tls = build_tls(strict_tls); + let tls_stream = tls + .connect(hostname, tcp_stream) + .await + .context("STARTTLS upgrade failed")?; + let buffered_stream = BufWriter::new(tls_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true).without_greeting(); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + + async fn connect_insecure_socks5( + &self, + context: &Context, + hostname: &str, + port: u16, + socks5_config: Socks5Config, + ) -> Result>> { + let socks5_stream = socks5_config + .connect(context, hostname, port, SMTP_TIMEOUT, false) + .await?; + let buffered_stream = BufWriter::new(socks5_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + + async fn connect_secure( + &self, + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result>> { + let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?; + let tls = build_tls(strict_tls); + let tls_stream = tls.connect(hostname, tcp_stream).await?; + let buffered_stream = BufWriter::new(tls_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + + async fn connect_starttls( + &self, + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result>> { + let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, strict_tls).await?; + + // Run STARTTLS command and convert the client back into a stream. + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, tcp_stream).await?; + let tcp_stream = transport.starttls().await?; + let tls = build_tls(strict_tls); + let tls_stream = tls + .connect(hostname, tcp_stream) + .await + .context("STARTTLS upgrade failed")?; + let buffered_stream = BufWriter::new(tls_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true).without_greeting(); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + + async fn connect_insecure( + &self, + context: &Context, + hostname: &str, + port: u16, + ) -> Result>> { + let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?; + let buffered_stream = BufWriter::new(tcp_stream); + let session_stream: Box = Box::new(buffered_stream); + let client = smtp::SmtpClient::new().smtp_utf8(true); + let transport = SmtpTransport::new(client, session_stream).await?; + Ok(transport) + } + /// Connect using the provided login params. pub async fn connect( &mut self, @@ -139,61 +260,83 @@ impl Smtp { CertificateChecks::AcceptInvalidCertificates | CertificateChecks::AcceptInvalidCertificates2 => false, }; - let tls_config = build_tls(strict_tls); - let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config); - let (creds, mechanism) = if lp.oauth2 { - // oauth2 - let send_pw = &lp.password; - let access_token = get_oauth2_access_token(context, addr, send_pw, false).await?; - if access_token.is_none() { - bail!("SMTP OAuth 2 error {}", addr); + let mut transport = if let Some(socks5_config) = socks5_config { + match lp.security { + Socket::Automatic => bail!("SMTP port security is not configured"), + Socket::Ssl => { + self.connect_secure_socks5( + context, + domain, + port, + strict_tls, + socks5_config.clone(), + ) + .await? + } + Socket::Starttls => { + self.connect_starttls_socks5( + context, + domain, + port, + strict_tls, + socks5_config.clone(), + ) + .await? + } + Socket::Plain => { + self.connect_insecure_socks5(context, domain, port, socks5_config.clone()) + .await? + } } - let user = &lp.user; - ( - smtp::authentication::Credentials::new( - user.to_string(), - access_token.unwrap_or_default(), - ), - vec![smtp::authentication::Mechanism::Xoauth2], - ) } else { - // plain - let user = lp.user.clone(); - let pw = lp.password.clone(); - ( - smtp::authentication::Credentials::new(user, pw), - vec![ - smtp::authentication::Mechanism::Plain, - smtp::authentication::Mechanism::Login, - ], - ) + match lp.security { + Socket::Automatic => bail!("SMTP port security is not configured"), + Socket::Ssl => { + self.connect_secure(context, domain, port, strict_tls) + .await? + } + Socket::Starttls => { + self.connect_starttls(context, domain, port, strict_tls) + .await? + } + Socket::Plain => self.connect_insecure(context, domain, port).await?, + } }; - let security = match lp.security { - Socket::Plain => smtp::ClientSecurity::None, - Socket::Starttls => smtp::ClientSecurity::Required(tls_parameters), - _ => smtp::ClientSecurity::Wrapper(tls_parameters), - }; - - let client = - smtp::SmtpClient::with_security(ServerAddress::new(domain.to_string(), port), security); - - let mut client = client - .smtp_utf8(true) - .credentials(creds) - .authentication_mechanism(mechanism) - .connection_reuse(smtp::ConnectionReuseParameters::ReuseUnlimited) - .timeout(Some(Duration::from_secs(SMTP_TIMEOUT))); - - if let Some(socks5_config) = socks5_config { - client = client.use_socks5(socks5_config.to_async_smtp_socks5_config()); + // Authenticate. + { + let (creds, mechanism) = if lp.oauth2 { + // oauth2 + let send_pw = &lp.password; + let access_token = get_oauth2_access_token(context, addr, send_pw, false).await?; + if access_token.is_none() { + bail!("SMTP OAuth 2 error {}", addr); + } + let user = &lp.user; + ( + smtp::authentication::Credentials::new( + user.to_string(), + access_token.unwrap_or_default(), + ), + vec![smtp::authentication::Mechanism::Xoauth2], + ) + } else { + // plain + let user = lp.user.clone(); + let pw = lp.password.clone(); + ( + smtp::authentication::Credentials::new(user, pw), + vec![ + smtp::authentication::Mechanism::Plain, + smtp::authentication::Mechanism::Login, + ], + ) + }; + transport.try_login(&creds, &mechanism).await?; } - let mut trans = client.into_transport(); - trans.connect().await.context("SMTP failed to connect")?; - - self.transport = Some(trans); + self.transport = Some(transport); self.last_success = Some(SystemTime::now()); context.emit_event(EventType::SmtpConnected(format!( @@ -223,7 +366,6 @@ pub(crate) async fn smtp_send( message: &str, smtp: &mut Smtp, msg_id: MsgId, - rowid: i64, ) -> SendResult { if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { info!(context, "smtp-sending out mime message:"); @@ -241,9 +383,7 @@ pub(crate) async fn smtp_send( return SendResult::Retry; } - let send_result = smtp - .send(context, recipients, message.as_bytes(), rowid) - .await; + let send_result = smtp.send(context, recipients, message.as_bytes()).await; smtp.last_send_error = send_result.as_ref().err().map(|e| e.to_string()); let status = match send_result { @@ -252,7 +392,7 @@ pub(crate) async fn smtp_send( info!(context, "SMTP failed to send: {:?}", &err); let res = match err { - async_smtp::smtp::error::Error::Permanent(ref response) => { + async_smtp::error::Error::Permanent(ref response) => { // Workaround for incorrectly configured servers returning permanent errors // instead of temporary ones. let maybe_transient = match response.code { @@ -287,7 +427,7 @@ pub(crate) async fn smtp_send( SendResult::Failure(format_err!("Permanent SMTP error: {}", err)) } } - async_smtp::smtp::error::Error::Transient(ref response) => { + async_smtp::error::Error::Transient(ref response) => { // We got a transient 4xx response from SMTP server. // Give some time until the server-side error maybe goes away. @@ -337,7 +477,7 @@ pub(crate) async fn smtp_send( // Local error, job is invalid, do not retry. smtp.disconnect().await; warn!(context, "SMTP job is invalid: {}", err); - SendResult::Failure(err.into()) + SendResult::Failure(err) } Err(crate::smtp::send::Error::NoTransport) => { // Should never happen. @@ -445,15 +585,7 @@ pub(crate) async fn send_msg_to_smtp( return Ok(()); } - let status = smtp_send( - context, - &recipients_list, - body.as_str(), - smtp, - msg_id, - rowid, - ) - .await; + let status = smtp_send(context, &recipients_list, body.as_str(), smtp, msg_id).await; match status { SendResult::Retry => {} @@ -585,7 +717,7 @@ async fn send_mdn_msg_id( .map_err(|err| format_err!("invalid recipient: {} {:?}", addr, err))?; let recipients = vec![recipient]; - match smtp_send(context, &recipients, &body, smtp, msg_id, 0).await { + match smtp_send(context, &recipients, &body, smtp, msg_id).await { SendResult::Success => { info!(context, "Successfully sent MDN for {}", msg_id); context diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 08f023371..3cd5c0567 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -1,8 +1,6 @@ //! # SMTP message sending -use std::time::Duration; - -use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport}; +use async_smtp::{EmailAddress, Envelope, SendableEmail}; use super::Smtp; use crate::config::Config; @@ -19,9 +17,9 @@ pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50; #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Envelope error: {}", _0)] - Envelope(#[from] async_smtp::error::Error), + Envelope(anyhow::Error), #[error("Send error: {}", _0)] - SmtpSend(#[from] async_smtp::smtp::error::Error), + SmtpSend(async_smtp::error::Error), #[error("SMTP has no transport")] NoTransport, #[error("{}", _0)] @@ -36,7 +34,6 @@ impl Smtp { context: &Context, recipients: &[EmailAddress], message: &[u8], - rowid: i64, ) -> Result<()> { if !context.get_config_bool(Config::Bot).await? { // Notify ratelimiter about sent message regardless of whether quota is exceeded or not. @@ -62,19 +59,10 @@ impl Smtp { let envelope = Envelope::new(self.from.clone(), recipients_chunk.to_vec()) .map_err(Error::Envelope)?; - let mail = SendableEmail::new( - envelope, - rowid.to_string(), // only used for internal logging - message, - ); + let mail = SendableEmail::new(envelope, message); if let Some(ref mut transport) = self.transport { - // The timeout is 1min + 3min per MB. - let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64; - transport - .send_with_timeout(mail, Some(&Duration::from_secs(timeout))) - .await - .map_err(Error::SmtpSend)?; + transport.send(mail).await.map_err(Error::SmtpSend)?; context.emit_event(EventType::SmtpMessageSent(format!( "Message len={message_len_bytes} was smtp-sent to {recipients_display}" diff --git a/src/socks.rs b/src/socks.rs index 58d7aa1db..f409008ff 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -5,16 +5,17 @@ use std::pin::Pin; use std::time::Duration; use anyhow::Result; -pub use async_smtp::ServerAddress; use fast_socks5::client::{Config, Socks5Stream}; use fast_socks5::util::target_addr::ToTargetAddr; use fast_socks5::AuthenticationMethod; use fast_socks5::Socks5Command; +use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use tokio::net::TcpStream; use tokio_io_timeout::TimeoutStream; use crate::context::Context; use crate::net::connect_tcp; +use crate::sql::Sql; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct Socks5Config { @@ -25,9 +26,7 @@ pub struct Socks5Config { impl Socks5Config { /// Reads SOCKS5 configuration from the database. - pub async fn from_database(context: &Context) -> Result> { - let sql = &context.sql; - + pub async fn from_database(sql: &Sql) -> Result> { let enabled = sql.get_raw_config_bool("socks5_enabled").await?; if enabled { let host = sql.get_raw_config("socks5_host").await?.unwrap_or_default(); @@ -56,6 +55,20 @@ impl Socks5Config { } } + /// Converts SOCKS5 configuration into URL. + pub fn to_url(&self) -> String { + // `socks5h` means that hostname is resolved into address by the proxy + // and DNS requests should not leak. + let mut url = "socks5h://".to_string(); + if let Some((username, password)) = &self.user_password { + let username_urlencoded = utf8_percent_encode(username, NON_ALPHANUMERIC).to_string(); + let password_urlencoded = utf8_percent_encode(password, NON_ALPHANUMERIC).to_string(); + url += &format!("{username_urlencoded}:{password_urlencoded}@"); + } + url += &format!("{}:{}", self.host, self.port); + url + } + /// If `load_dns_cache` is true, loads cached DNS resolution results. /// Use this only if the connection is going to be protected with TLS checks. pub async fn connect( @@ -87,14 +100,6 @@ impl Socks5Config { Ok(socks_stream) } - - pub fn to_async_smtp_socks5_config(&self) -> async_smtp::smtp::Socks5Config { - async_smtp::smtp::Socks5Config { - host: self.host.clone(), - port: self.port, - user_password: self.user_password.clone(), - } - } } impl fmt::Display for Socks5Config { @@ -112,3 +117,35 @@ impl fmt::Display for Socks5Config { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_socks5h_url() { + let config = Socks5Config { + host: "127.0.0.1".to_string(), + port: 9050, + user_password: None, + }; + assert_eq!(config.to_url(), "socks5h://127.0.0.1:9050"); + + let config = Socks5Config { + host: "example.org".to_string(), + port: 1080, + user_password: Some(("root".to_string(), "toor".to_string())), + }; + assert_eq!(config.to_url(), "socks5h://root:toor@example.org:1080"); + + let config = Socks5Config { + host: "example.org".to_string(), + port: 1080, + user_password: Some(("root".to_string(), "foo/?\\@".to_string())), + }; + assert_eq!( + config.to_url(), + "socks5h://root:foo%2F%3F%5C%40@example.org:1080" + ); + } +} diff --git a/src/stock_str.rs b/src/stock_str.rs index 2ce6d03db..f91b5a783 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -1449,16 +1449,10 @@ mod tests { }; // delete self-talk first; this adds a message to device-chat about how self-talk can be restored - let device_chat_msgs_before = chat::get_chat_msgs(&t, device_chat_id, 0) - .await - .unwrap() - .len(); + let device_chat_msgs_before = chat::get_chat_msgs(&t, device_chat_id).await.unwrap().len(); self_talk_id.delete(&t).await.ok(); assert_eq!( - chat::get_chat_msgs(&t, device_chat_id, 0) - .await - .unwrap() - .len(), + chat::get_chat_msgs(&t, device_chat_id).await.unwrap().len(), device_chat_msgs_before + 1 ); diff --git a/src/sync.rs b/src/sync.rs index f31825ca8..8eba78308 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -476,7 +476,7 @@ mod tests { let chat = Chat::load_from_db(&alice, chat_id).await?; assert!(chat.is_self_talk()); assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1); - let msgs = chat::get_chat_msgs(&alice, chat_id, 0).await?; + let msgs = chat::get_chat_msgs(&alice, chat_id).await?; assert_eq!(msgs.len(), 0); // let alice's other device receive and execute the sync message, diff --git a/src/test_utils.rs b/src/test_utils.rs index 92ffd3baf..fd879c68a 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -18,11 +18,11 @@ use tokio::runtime::Handle; use tokio::sync::RwLock; use tokio::task; -use crate::chat::{self, Chat, ChatId}; +use crate::chat::{self, Chat, ChatId, MessageListOptions}; use crate::chatlist::Chatlist; use crate::config::Config; use crate::constants::Chattype; -use crate::constants::{DC_GCL_NO_SPECIALS, DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; +use crate::constants::{DC_GCL_NO_SPECIALS, DC_MSG_ID_DAYMARKER}; use crate::contact::{Contact, ContactAddress, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::{Event, EventType, Events}; @@ -469,9 +469,7 @@ impl TestContext { .await .unwrap(); - let chat_msgs = chat::get_chat_msgs(self, received.chat_id, 0) - .await - .unwrap(); + let chat_msgs = chat::get_chat_msgs(self, received.chat_id).await.unwrap(); assert!( chat_msgs.contains(&ChatItem::Message { msg_id: msg.id }), "received message is not shown in chat, maybe it's hidden (you may have \ @@ -496,7 +494,7 @@ impl TestContext { /// /// Panics on errors or if the most recent message is a marker. pub async fn get_last_msg_in(&self, chat_id: ChatId) -> Message { - let msgs = chat::get_chat_msgs(&self.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&self.ctx, chat_id).await.unwrap(); let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() { msg_id } else { @@ -624,9 +622,16 @@ impl TestContext { #[allow(dead_code)] #[allow(clippy::indexing_slicing)] pub async fn print_chat(&self, chat_id: ChatId) { - let msglist = chat::get_chat_msgs(self, chat_id, DC_GCM_ADDDAYMARKER) - .await - .unwrap(); + let msglist = chat::get_chat_msgs_ex( + self, + chat_id, + MessageListOptions { + info_only: false, + add_daymarker: true, + }, + ) + .await + .unwrap(); let msglist: Vec = msglist .into_iter() .map(|x| match x { @@ -923,7 +928,7 @@ pub(crate) async fn get_chat_msg( index: usize, asserted_msgs_count: usize, ) -> Message { - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id).await.unwrap(); assert_eq!(msgs.len(), asserted_msgs_count); let msg_id = if let ChatItem::Message { msg_id } = msgs[index] { msg_id diff --git a/src/tests/aeap.rs b/src/tests/aeap.rs index efb7954c8..86751a4ab 100644 --- a/src/tests/aeap.rs +++ b/src/tests/aeap.rs @@ -4,7 +4,6 @@ use anyhow::Result; use crate::chat; use crate::chat::ChatId; -use crate::constants; use crate::contact; use crate::contact::Contact; use crate::contact::ContactId; @@ -345,9 +344,16 @@ async fn mark_as_verified(this: &TestContext, other: &TestContext) { } async fn get_last_info_msg(t: &TestContext, chat_id: ChatId) -> Option { - let msgs = chat::get_chat_msgs(&t.ctx, chat_id, constants::DC_GCM_INFO_ONLY) - .await - .unwrap(); + let msgs = chat::get_chat_msgs_ex( + &t.ctx, + chat_id, + chat::MessageListOptions { + info_only: true, + add_daymarker: false, + }, + ) + .await + .unwrap(); let msg_id = if let chat::ChatItem::Message { msg_id } = msgs.last()? { msg_id } else { diff --git a/src/tools.rs b/src/tools.rs index a547f6042..b21665dc7 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -1180,7 +1180,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); let device_chat_id = chats.get_chat_id(0).unwrap(); - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), 1); // the message should be added only once a day - test that an hour later and nearly a day later @@ -1190,7 +1190,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; get_provider_update_timestamp(), ) .await; - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), 1); maybe_warn_on_bad_time( @@ -1199,7 +1199,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; get_provider_update_timestamp(), ) .await; - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), 1); // next day, there should be another device message @@ -1212,7 +1212,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap()); - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), 2); } @@ -1242,7 +1242,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); let device_chat_id = chats.get_chat_id(0).unwrap(); - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), 1); // do not repeat the warning every day ... @@ -1262,7 +1262,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); let device_chat_id = chats.get_chat_id(0).unwrap(); - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); let test_len = msgs.len(); assert!(test_len == 1 || test_len == 2); @@ -1277,7 +1277,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap(); assert_eq!(chats.len(), 1); let device_chat_id = chats.get_chat_id(0).unwrap(); - let msgs = chat::get_chat_msgs(&t, device_chat_id, 0).await.unwrap(); + let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap(); assert_eq!(msgs.len(), test_len + 1); }