feat: Try SQLite FTS5 (#5052 )

Merge pull request #5305 from deltachat/dependabot/cargo/tempfile-3.10.1
chore(cargo): bump image from 0.24.8 to 0.24.9
2026-04-06 15:42:10 +03:00 · 2024-03-04 15:28:51 -03:00 · 2024-03-03 20:44:06 +00:00 · 2024-03-03 14:46:53 +00:00 · 2024-03-03 08:05:01 +00:00 · 2024-03-03 08:01:51 +00:00
42 changed files with 1099 additions and 1175 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
    name: Lint Rust
    runs-on: ubuntu-latest
    env:
-      RUSTUP_TOOLCHAIN: 1.75.0
+      RUSTUP_TOOLCHAIN: 1.76.0
    steps:
      - uses: actions/checkout@v4
        with:
@@ -83,11 +83,11 @@ jobs:
      matrix:
        include:
          - os: ubuntu-latest
-            rust: 1.75.0
+            rust: 1.76.0
          - os: windows-latest
-            rust: 1.75.0
+            rust: 1.76.0
          - os: macos-latest
-            rust: 1.75.0
+            rust: 1.76.0

          # Minimum Supported Rust Version = 1.70.0
          - os: ubuntu-latest
@@ -218,7 +218,7 @@ jobs:
          path: target/debug

      - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

@@ -264,7 +264,7 @@ jobs:
          show-progress: false

      - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

--- a/.github/workflows/deltachat-rpc-server.yml
+++ b/.github/workflows/deltachat-rpc-server.yml
@@ -174,7 +174,7 @@ jobs:

      # Python 3.11 is needed for tomllib used in scripts/wheel-rpc-server.py
      - name: Install python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: 3.12

--- a/.github/workflows/upload-docs.yml
+++ b/.github/workflows/upload-docs.yml
@@ -1,4 +1,4 @@
-name: Build & Deploy Documentation on rs.delta.chat
+name: Build & Deploy Documentation on rs.delta.chat, c.delta.chat, py.delta.chat

 on:
  push:
@@ -6,7 +6,7 @@ on:
      - main

 jobs:
-  build:
+  build-rs:
    runs-on: ubuntu-latest

    steps:
@@ -24,3 +24,41 @@ jobs:
          HOST: "delta.chat"
          SOURCE: "target/doc"
          TARGET: "/var/www/html/rs/"
+
+  build-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build Python documentation
+        run: nix build .#python-docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+
+  build-c:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build C documentation
+        run: nix build .#docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
--- a/.github/workflows/upload-ffi-docs.yml
+++ b/.github/workflows/upload-ffi-docs.yml
@@ -21,10 +21,8 @@ jobs:
        run: |
          cargo doc --package deltachat_ffi --no-deps
      - name: Upload to cffi.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/cffi/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"
--- a/.github/workflows/upload-python-docs.yml
+++ b/.github/workflows/upload-python-docs.yml
@@ -1,26 +0,0 @@
-name: Build & Deploy Documentation on py.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          show-progress: false
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Build Python documentation
-        run: scripts/build-python-docs.sh
-      - name: Upload to py.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: delta
-          KEY: ${{ secrets.CODESPEAK_KEY }}
-          HOST: "lists.codespeak.net"
-          SOURCE: "dist/html/"
-          TARGET: "/home/delta/build/master"
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -121,9 +121,9 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"

 [[package]]
 name = "anyhow"
-version = "1.0.79"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
+checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1"
 dependencies = [
 "backtrace",
 ]
@@ -290,7 +290,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -991,7 +991,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -1128,7 +1128,6 @@ dependencies = [
 "num-traits",
 "num_cpus",
 "once_cell",
- "openssl-src",
 "parking_lot",
 "percent-encoding",
 "pgp",
@@ -1233,7 +1232,7 @@ name = "deltachat_derive"
 version = "2.0.0"
 dependencies = [
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -1442,7 +1441,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -1468,6 +1467,22 @@ dependencies = [
 "syn 1.0.109",
 ]

+[[package]]
+name = "dsa"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48bc224a9084ad760195584ce5abb3c2c34a225fa312a128ad245a6b412b7689"
+dependencies = [
+ "digest 0.10.7",
+ "num-bigint-dig",
+ "num-traits",
+ "pkcs8 0.10.2",
+ "rfc6979 0.4.0",
+ "sha2 0.10.8",
+ "signature 2.2.0",
+ "zeroize",
+]
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.16"
@@ -1735,7 +1750,7 @@ dependencies = [
 "heck",
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -1748,7 +1763,7 @@ dependencies = [
 "num-traits",
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -2047,7 +2062,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -2117,9 +2132,9 @@ dependencies = [

 [[package]]
 name = "gif"
-version = "0.12.0"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045"
+checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2"
 dependencies = [
 "color_quant",
 "weezl",
@@ -2213,9 +2228,9 @@ dependencies = [

 [[package]]
 name = "hashlink"
-version = "0.8.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
+checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"
 dependencies = [
 "hashbrown",
 ]
@@ -2550,9 +2565,9 @@ dependencies = [

 [[package]]
 name = "image"
-version = "0.24.8"
+version = "0.24.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "034bbe799d1909622a74d1193aa50147769440040ff36cb2baa947609b0a4e23"
+checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
 dependencies = [
 "bytemuck",
 "byteorder",
@@ -2664,6 +2679,12 @@ dependencies = [
 "windows-sys 0.52.0",
 ]

+[[package]]
+name = "iter-read"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a598c1abae8e3456ebda517868b254b6bc2a9bb6501ffd5b9d0875bf332e048b"
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2694,6 +2715,20 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "k256"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b"
+dependencies = [
+ "cfg-if",
+ "ecdsa 0.16.9",
+ "elliptic-curve 0.13.8",
+ "once_cell",
+ "sha2 0.10.8",
+ "signature 2.2.0",
+]
+
 [[package]]
 name = "kamadak-exif"
 version = "0.5.5"
@@ -2770,9 +2805,9 @@ dependencies = [

 [[package]]
 name = "libsqlite3-sys"
-version = "0.27.0"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
+checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
 dependencies = [
 "cc",
 "openssl-sys",
@@ -2804,9 +2839,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"

 [[package]]
 name = "lru-cache"
@@ -3078,7 +3113,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -3121,6 +3156,27 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "num_enum"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "object"
 version = "0.32.2"
@@ -3159,9 +3215,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

 [[package]]
 name = "openssl"
-version = "0.10.63"
+version = "0.10.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8"
+checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
 dependencies = [
 "bitflags 2.4.2",
 "cfg-if",
@@ -3180,7 +3236,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -3191,18 +3247,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

 [[package]]
 name = "openssl-src"
-version = "300.1.6+3.1.4"
+version = "300.2.3+3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085"
+checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843"
 dependencies = [
 "cc",
 ]

 [[package]]
 name = "openssl-sys"
-version = "0.9.99"
+version = "0.9.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae"
+checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff"
 dependencies = [
 "cc",
 "libc",
@@ -3350,9 +3406,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"

 [[package]]
 name = "pgp"
-version = "0.10.2"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27e1f8e085bfa9b85763fe3ddaacbe90a09cd847b3833129153a6cb063bbe132"
+checksum = "031fa1e28c4cb54c90502ef0642a44ef10ec8349349ebe6372089f1b1ef4f297"
 dependencies = [
 "aes",
 "base64 0.21.7",
@@ -3367,23 +3423,27 @@ dependencies = [
 "cfb-mode",
 "chrono",
 "cipher",
+ "const-oid",
 "crc24",
 "curve25519-dalek 4.1.2",
 "derive_builder",
 "des",
 "digest 0.10.7",
+ "dsa",
 "ed25519-dalek 2.1.1",
 "elliptic-curve 0.13.8",
 "flate2",
 "generic-array",
 "hex",
 "idea",
+ "iter-read",
+ "k256",
 "log",
 "md-5",
 "nom",
 "num-bigint-dig",
- "num-derive",
 "num-traits",
+ "num_enum",
 "p256 0.13.2",
 "p384 0.13.0",
 "rand 0.8.5",
@@ -3417,7 +3477,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -3599,6 +3659,15 @@ dependencies = [
 "elliptic-curve 0.13.8",
 ]

+[[package]]
+name = "proc-macro-crate"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
+dependencies = [
+ "toml_edit 0.21.1",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -4102,9 +4171,9 @@ dependencies = [

 [[package]]
 name = "rusqlite"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
+checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
 dependencies = [
 "bitflags 2.4.2",
 "fallible-iterator",
@@ -4381,9 +4450,9 @@ dependencies = [

 [[package]]
 name = "serde"
-version = "1.0.196"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
 dependencies = [
 "serde_derive",
 ]
@@ -4408,13 +4477,13 @@ dependencies = [

 [[package]]
 name = "serde_derive"
-version = "1.0.196"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -4430,9 +4499,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.113"
+version = "1.0.114"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
 dependencies = [
 "itoa",
 "ryu",
@@ -4699,7 +4768,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "rustversion",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -4721,9 +4790,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.48"
+version = "2.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
+checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -4791,9 +4860,9 @@ checksum = "094c9f64d6de9a8506b1e49b63a29333b37ed9e821ee04be694d431b3264c3c5"

 [[package]]
 name = "tempfile"
-version = "3.10.0"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
 dependencies = [
 "cfg-if",
 "fastrand",
@@ -4826,9 +4895,9 @@ dependencies = [

 [[package]]
 name = "textwrap"
-version = "0.16.0"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
 dependencies = [
 "smawk",
 "unicode-linebreak",
@@ -4852,7 +4921,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -4969,7 +5038,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -5058,7 +5127,7 @@ dependencies = [
 "serde",
 "serde_spanned",
 "toml_datetime",
- "toml_edit",
+ "toml_edit 0.22.4",
 ]

 [[package]]
@@ -5070,6 +5139,17 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "toml_edit"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.22.4"
@@ -5131,7 +5211,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -5363,9 +5443,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

 [[package]]
 name = "walkdir"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
 dependencies = [
 "same-file",
 "winapi-util",
@@ -5419,7 +5499,7 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 "wasm-bindgen-shared",
 ]

@@ -5453,7 +5533,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
@@ -5839,7 +5919,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]

 [[package]]
@@ -5859,5 +5939,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.48",
+ "syn 2.0.52",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,7 +56,7 @@ futures-lite = "2.2.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.8", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.24.9", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
@@ -69,7 +69,7 @@ num-traits = "0.2"
 once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.10", default-features = false }
+pgp = { version = "0.11", default-features = false }
 pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
@@ -78,7 +78,7 @@ quoted_printable = "0.5"
 rand = "0.8"
 regex = "1.10"
 reqwest = { version = "0.11.24", features = ["json"] }
-rusqlite = { version = "0.30", features = ["sqlcipher"] }
+rusqlite = { version = "0.31", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1"
@@ -89,7 +89,7 @@ smallvec = "1"
 strum = "0.26"
 strum_macros = "0.26"
 tagger = "4.3.4"
-textwrap = "0.16.0"
+textwrap = "0.16.1"
 thiserror = "1"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
@@ -100,14 +100,6 @@ toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }

-# Pin OpenSSL to 3.1 releases.
-# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
-# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
-# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
-# According to <https://www.openssl.org/policies/releasestrat.html>
-# 3.1 branch will be supported until 2025-03-14.
-openssl-src = "~300.1"
-
 [dev-dependencies]
 ansi_term = "0.12.0"
 anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
--- a/deltachat-ffi/src/lib.rs
+++ b/deltachat-ffi/src/lib.rs
@@ -26,7 +26,7 @@ use anyhow::Context as _;
 use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, ContactId, Origin};
-use deltachat::context::Context;
+use deltachat::context::{Context, ContextBuilder};
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
@@ -34,7 +34,6 @@ use deltachat::message::MsgId;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
-use deltachat::stock_str::StockStrings;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
@@ -104,12 +103,11 @@ pub unsafe extern "C" fn dc_context_new(
    let ctx = if blobdir.is_null() || *blobdir == 0 {
        // generate random ID as this functionality is not yet available on the C-api.
        let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            as_path(dbfile),
-            id,
-            Events::new(),
-            StockStrings::new(),
-        ))
+        block_on(
+            ContextBuilder::new(as_path(dbfile).to_path_buf())
+                .with_id(id)
+                .open(),
+        )
    } else {
        eprintln!("blobdir can not be defined explicitly anymore");
        return ptr::null_mut();
@@ -133,12 +131,11 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
    }

    let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile),
-        id,
-        Events::new(),
-        StockStrings::new(),
-    )) {
+    match block_on(
+        ContextBuilder::new(as_path(dbfile).to_path_buf())
+            .with_id(id)
+            .build(),
+    ) {
        Ok(context) => Box::into_raw(Box::new(context)),
        Err(err) => {
            eprintln!("failed to create context: {err:#}");
--- a/deltachat-jsonrpc/Cargo.toml
+++ b/deltachat-jsonrpc/Cargo.toml
@@ -17,7 +17,7 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.9.0"
+tempfile = "3.10.1"
 log = "0.4"
 async-channel = { version = "2.0.0" }
 futures = { version = "0.3.30" }
@@ -26,7 +26,7 @@ yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }
 sanitize-filename = "0.5"
-walkdir = "2.3.3"
+walkdir = "2.5.0"
 base64 = "0.21"

 # optional dependencies
--- a/deltachat-repl/Cargo.toml
+++ b/deltachat-repl/Cargo.toml
@@ -9,9 +9,9 @@ ansi_term = "0.12.1"
 anyhow = "1"
 deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
-log = "0.4.20"
+log = "0.4.21"
 pretty_env_logger = "0.5"
-rusqlite = "0.30"
+rusqlite = "0.31"
 rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }

--- a/deltachat-repl/src/main.rs
+++ b/deltachat-repl/src/main.rs
@@ -9,7 +9,6 @@ extern crate deltachat;

 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;

 use ansi_term::Color;
@@ -20,8 +19,7 @@ use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::stock_str::StockStrings;
-use deltachat::{EventType, Events};
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
@@ -312,7 +310,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
        println!("Error: Bad arguments, expected [db-name].");
        bail!("No db-name specified");
    }
-    let context = Context::new(Path::new(&args[1]), 0, Events::new(), StockStrings::new()).await?;
+    let context = ContextBuilder::new(args[1].clone().into())
+        .with_id(1)
+        .open()
+        .await?;

    let events = context.get_event_emitter();
    tokio::task::spawn(async move {
--- a/deny.toml
+++ b/deny.toml
@@ -1,5 +1,4 @@
 [advisories]
-unmaintained = "allow"
 ignore = [
    "RUSTSEC-2020-0071",
    "RUSTSEC-2022-0093",
@@ -10,6 +9,15 @@ ignore = [
    # There is no fix at the time of writing this (2023-11-28).
    # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
    "RUSTSEC-2023-0071",
+
+    # Unmaintained ansi_term
+    "RUSTSEC-2021-0139",
+
+    # Unmaintained encoding
+    "RUSTSEC-2021-0153",
+
+    # Unmaintained safemem
+    "RUSTSEC-2023-0081",
 ]

 [bans]
@@ -56,6 +64,7 @@ skip = [
     { name = "spki", version = "0.6.0" },
     { name = "syn", version = "1.0.109" },
     { name = "time", version = "<0.3" },
+     { name = "toml_edit", version = "0.21.1" },
     { name = "untrusted", version = "0.7.1" },
     { name = "wasi", version = "<0.11" },
     { name = "windows_aarch64_gnullvm", version = "<0.52" },
--- a/flake.nix
+++ b/flake.nix
@@ -250,6 +250,7 @@
                  ./deltachat-ratelimit
                  ./deltachat-repl
                  ./deltachat-rpc-client
+                  ./deltachat-time
                  ./deltachat-rpc-server
                  ./format-flowed
                  ./release-date.in
--- a/python/src/deltachat/account.py
+++ b/python/src/deltachat/account.py
@@ -375,22 +375,6 @@ class Account:
        dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
        return (x for x in iter_array(dc_array, lambda x: Message.from_db(self, x)) if x is not None)

-    def _wait_next_message_ids(self) -> List[int]:
-        """Return IDs of all next messages from all chats."""
-        dc_array = ffi.gc(lib.dc_wait_next_msgs(self._dc_context), lib.dc_array_unref)
-        return [lib.dc_array_get_id(dc_array, i) for i in range(lib.dc_array_get_cnt(dc_array))]
-
-    def wait_next_incoming_message(self) -> Message:
-        """Waits until the next incoming message
-        with ID higher than given is received and returns it."""
-        while True:
-            message_ids = self._wait_next_message_ids()
-            for msg_id in message_ids:
-                message = Message.from_db(self, msg_id)
-                if message and not message.is_from_self() and not message.is_from_device():
-                    self.set_config("last_msg_id", str(msg_id))
-                    return message
-
    def create_chat(self, obj) -> Chat:
        """Create a 1:1 chat with Account, Contact or e-mail address."""
        return self.create_contact(obj).create_chat()
--- a/python/src/deltachat/events.py
+++ b/python/src/deltachat/events.py
@@ -182,6 +182,12 @@ class FFIEventTracker:
                print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)
                break

+    def wait_securejoin_joiner_progress(self, target):
+        while True:
+            event = self.get_matching("DC_EVENT_SECUREJOIN_JOINER_PROGRESS")
+            if event.data2 >= target:
+                break
+
    def wait_idle_inbox_ready(self):
        """Has to be called after start_io() to wait for fetch_existing_msgs to run
        so that new messages are not mistaken for old ones:
--- a/python/src/deltachat/testplugin.py
+++ b/python/src/deltachat/testplugin.py
@@ -10,7 +10,6 @@ import time
 import weakref
 import random
 from queue import Queue
-from threading import Event
 from typing import Callable, Dict, List, Optional, Set

 import pytest
@@ -592,23 +591,16 @@ class ACFactory:
        return ac1.create_chat(ac2)

    def get_protected_chat(self, ac1: Account, ac2: Account):
-        class SetupPlugin:
-            def __init__(self) -> None:
-                self.member_added = Event()
-
-            @account_hookimpl
-            def ac_member_added(self, chat: deltachat.Chat, contact, actor, message):
-                self.member_added.set()
-
-        setupplugin = SetupPlugin()
-        ac1.add_account_plugin(setupplugin)
        chat = ac1.create_group_chat("Protected Group", verified=True)
        qr = chat.get_join_qr()
        ac2.qr_join_chat(qr)
-        setupplugin.member_added.wait()
-        msg = ac2.wait_next_incoming_message()
+        ac2._evtracker.wait_securejoin_joiner_progress(1000)
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev.data2)
+        assert msg is not None
        assert msg.text == "Messages are guaranteed to be end-to-end encrypted from now on."
-        msg = ac2.wait_next_incoming_message()
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg is not None
        assert "Member Me " in msg.text and " added by " in msg.text
        return chat

--- a/python/tests/test_1_online.py
+++ b/python/tests/test_1_online.py
@@ -44,21 +44,21 @@ def test_configure_generate_key(acfactory, lp):
    lp.sec("ac1: send unencrypted message to ac2")
    chat.send_text("message1")
    lp.sec("ac2: waiting for message from ac1")
-    msg_in = ac2.wait_next_incoming_message()
+    msg_in = ac2._evtracker.wait_next_incoming_message()
    assert msg_in.text == "message1"
    assert not msg_in.is_encrypted()

    lp.sec("ac2: send encrypted message to ac1")
    msg_in.chat.send_text("message2")
    lp.sec("ac1: waiting for message from ac2")
-    msg2_in = ac1.wait_next_incoming_message()
+    msg2_in = ac1._evtracker.wait_next_incoming_message()
    assert msg2_in.text == "message2"
    assert msg2_in.is_encrypted()

    lp.sec("ac1: send encrypted message to ac2")
    msg2_in.chat.send_text("message3")
    lp.sec("ac2: waiting for message from ac1")
-    msg3_in = ac2.wait_next_incoming_message()
+    msg3_in = ac2._evtracker.wait_next_incoming_message()
    assert msg3_in.text == "message3"
    assert msg3_in.is_encrypted()

@@ -520,7 +520,7 @@ def test_forward_encrypted_to_unencrypted(acfactory, lp):
    lp.sec("ac1: send encrypted message to ac2")
    txt = "This should be encrypted"
    chat.send_text(txt)
-    msg = ac2.wait_next_incoming_message()
+    msg = ac2._evtracker.wait_next_incoming_message()
    assert msg.text == txt
    assert msg.is_encrypted()

--- a/python/tests/test_3_offline.py
+++ b/python/tests/test_3_offline.py
@@ -52,8 +52,8 @@ class TestOfflineAccountBasic:
    def test_wrong_db(self, tmp_path):
        p = tmp_path / "hello.db"
        p.write_text("123")
-        account = Account(str(p))
-        assert not account.is_open()
+        with pytest.raises(ValueError):
+            _account = Account(str(p))

    def test_os_name(self, tmp_path):
        p = tmp_path / "hello.db"
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -37,8 +37,6 @@ and an own build machine.

 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.

- `build-python-docs.sh` builds Python documentation into `dist/html/`.
-
 ## Triggering runs on the build machine locally (fast!)

 There is experimental support for triggering a remote Python or Rust test run 
--- a/scripts/build-python-docs.sh
+++ b/scripts/build-python-docs.sh
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export DCC_RS_TARGET=debug
-export DCC_RS_DEV="$PWD"
-cargo build -p deltachat_ffi --features jsonrpc
-
-python3 -m venv venv
-venv/bin/pip install ./python
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/pip install sphinx breathe sphinx_rtd_theme
-venv/bin/sphinx-build -b html -a python/doc/ dist/html
--- a/scripts/concourse/docs_wheels.yml
+++ b/scripts/concourse/docs_wheels.yml
@@ -15,55 +15,6 @@ resources:
      tag_filter: "v*"

 jobs:
-  - name: doxygen
-    plan:
-      - get: deltachat-core-rust
-        trigger: true
-
-      # Build Doxygen documentation
-      - task: build-doxygen
-        config:
-          inputs:
-            - name: deltachat-core-rust
-          outputs:
-            - name: c-docs
-          image_resource:
-            source:
-              repository: alpine
-            type: registry-image
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache doxygen git
-                cd deltachat-core-rust
-                scripts/run-doxygen.sh
-                cd ..
-                cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
-
-      - task: upload-c-docs
-        config:
-          inputs:
-            - name: c-docs
-          image_resource:
-            type: registry-image
-            source:
-              repository: alpine
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache rsync openssh-client
-                mkdir -p ~/.ssh
-                chmod 700 ~/.ssh
-                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-                chmod 600 ~/.ssh/id_ed25519
-                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
  - name: python-x86_64
    plan:
      - get: deltachat-core-rust
--- a/scripts/wheel-rpc-server.py
+++ b/scripts/wheel-rpc-server.py
@@ -157,12 +157,12 @@ def main():
    build_wheel(
        version,
        "dist/deltachat-rpc-server-armv7l-linux",
-        "py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
+        "py3-none-linux_armv7l.manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
    )
    build_wheel(
        version,
        "dist/deltachat-rpc-server-armv6l-linux",
-        "py3-none-manylinux_2_17_armv6l.manylinux2014_armv6l.musllinux_1_1_armv6l",
+        "py3-none-linux_armv6l",
    )
    build_wheel(
        version,
--- a/src/accounts.rs
+++ b/src/accounts.rs
@@ -17,7 +17,7 @@ use tokio::sync::oneshot;
 #[cfg(not(target_os = "ios"))]
 use tokio::time::{sleep, Duration};

-use crate::context::Context;
+use crate::context::{Context, ContextBuilder};
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::stock_str::StockStrings;

@@ -120,13 +120,16 @@ impl Accounts {
        let account_config = self.config.new_account().await?;
        let dbfile = account_config.dbfile(&self.dir);

-        let ctx = Context::new(
-            &dbfile,
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .build()
+            .await?;
+        // Try to open without a passphrase,
+        // but do not return an error if account is passphare-protected.
+        ctx.open("".to_string()).await?;
+
        self.accounts.insert(account_config.id, ctx);

        Ok(account_config.id)
@@ -135,14 +138,14 @@ impl Accounts {
    /// Adds a new closed account.
    pub async fn add_closed_account(&mut self) -> Result<u32> {
        let account_config = self.config.new_account().await?;
+        let dbfile = account_config.dbfile(&self.dir);

-        let ctx = Context::new_closed(
-            &account_config.dbfile(&self.dir),
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .build()
+            .await?;
        self.accounts.insert(account_config.id, ctx);

        Ok(account_config.id)
@@ -527,19 +530,17 @@ impl Config {
        let mut accounts = BTreeMap::new();

        for account_config in &self.inner.accounts {
-            let ctx = Context::new(
-                &account_config.dbfile(dir),
-                account_config.id,
-                events.clone(),
-                stockstrings.clone(),
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "failed to create context from file {:?}",
-                    account_config.dbfile(dir)
-                )
-            })?;
+            let dbfile = account_config.dbfile(dir);
+            let ctx = ContextBuilder::new(dbfile.clone())
+                .with_id(account_config.id)
+                .with_events(events.clone())
+                .with_stock_strings(stockstrings.clone())
+                .build()
+                .await
+                .with_context(|| format!("failed to create context from file {:?}", &dbfile))?;
+            // Try to open without a passphrase,
+            // but do not return an error if account is passphare-protected.
+            ctx.open("".to_string()).await?;

            accounts.insert(account_config.id, ctx);
        }
--- a/src/config.rs
+++ b/src/config.rs
@@ -779,7 +779,7 @@ mod tests {

    use super::*;
    use crate::constants;
-    use crate::test_utils::{sync, TestContext};
+    use crate::test_utils::{sync, TestContext, TestContextManager};

    #[test]
    fn test_to_string() {
@@ -1007,6 +1007,65 @@ mod tests {
        Ok(())
    }

+    /// Sync message mustn't be sent if self-{status,avatar} is changed by a self-sent message.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_no_sync_on_self_sent_msg() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice0 = &tcm.alice().await;
+        let alice1 = &tcm.alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let status = "Synced via usual message";
+        alice0.set_config(Config::Selfstatus, Some(status)).await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let status1 = "Synced via sync message";
+        alice1.set_config(Config::Selfstatus, Some(status1)).await?;
+        tcm.send_recv(alice0, alice1, "hi Alice!").await;
+        assert_eq!(
+            alice1.get_config(Config::Selfstatus).await?,
+            Some(status.to_string())
+        );
+        sync(alice1, alice0).await;
+        assert_eq!(
+            alice0.get_config(Config::Selfstatus).await?,
+            Some(status1.to_string())
+        );
+
+        // Need a chat with another contact to send self-avatar.
+        let bob = &tcm.bob().await;
+        let a0b_chat_id = tcm.send_recv_accept(bob, alice0, "hi").await.chat_id;
+        let file = alice0.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        tokio::fs::write(&file, bytes).await?;
+        alice0
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let file = alice1.dir.path().join("avatar.jpg");
+        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        tokio::fs::write(&file, bytes).await?;
+        alice1
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        let sent_msg = alice0.send_text(a0b_chat_id, "hi").await;
+        alice1.recv_msg(&sent_msg).await;
+        assert!(alice1
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".png"))
+            .is_some());
+        sync(alice1, alice0).await;
+        assert!(alice0
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".jpg"))
+            .is_some());
+
+        Ok(())
+    }
+
    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn test_event_config_synced() -> Result<()> {
        let alice0 = TestContext::new_alice().await;
--- a/src/configure.rs
+++ b/src/configure.rs
@@ -25,7 +25,7 @@ use tokio::task;
 use crate::config::{self, Config};
 use crate::contact::addr_cmp;
 use crate::context::Context;
-use crate::imap::Imap;
+use crate::imap::{session::Session as ImapSession, Imap};
 use crate::log::LogExt;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
@@ -395,7 +395,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {

    // Configure IMAP

-    let mut imap: Option<Imap> = None;
+    let mut imap: Option<(Imap, ImapSession)> = None;
    let imap_servers: Vec<&ServerParams> = servers
        .iter()
        .filter(|params| params.protocol == Protocol::Imap)
@@ -433,7 +433,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
        );
    }
-    let mut imap = match imap {
+    let (mut imap, mut imap_session) = match imap {
        Some(imap) => imap,
        None => bail!(nicer_configuration_error(ctx, errors).await),
    };
@@ -454,9 +454,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {

    let create_mvbox = ctx.should_watch_mvbox().await?;

-    imap.configure_folders(ctx, create_mvbox).await?;
+    imap.configure_folders(ctx, &mut imap_session, create_mvbox)
+        .await?;

-    imap.select_with_uidvalidity(ctx, "INBOX")
+    imap_session
+        .select_with_uidvalidity(ctx, "INBOX")
        .await
        .context("could not read INBOX status")?;

@@ -576,7 +578,7 @@ async fn try_imap_one_param(
    socks5_config: &Option<Socks5Config>,
    addr: &str,
    provider_strict_tls: bool,
-) -> Result<Imap, ConfigurationError> {
+) -> Result<(Imap, ImapSession), ConfigurationError> {
    let inf = format!(
        "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
        param.user,
@@ -614,9 +616,9 @@ async fn try_imap_one_param(
                msg: format!("{err:#}"),
            })
        }
-        Ok(()) => {
+        Ok(session) => {
            info!(context, "success: {}", inf);
-            Ok(imap)
+            Ok((imap, session))
        }
    }
 }
--- a/src/contact.rs
+++ b/src/contact.rs
@@ -1537,7 +1537,7 @@ WHERE type=? AND id IN (
 /// as profile images can be set only by receiving messages, this should be always the case, however.
 ///
 /// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
-/// this typically happens if we see message with our own profile image, sent from another device.
+/// this typically happens if we see message with our own profile image.
 pub(crate) async fn set_profile_image(
    context: &Context,
    contact_id: ContactId,
@@ -1550,7 +1550,7 @@ pub(crate) async fn set_profile_image(
            if contact_id == ContactId::SELF {
                if was_encrypted {
                    context
-                        .set_config_internal(Config::Selfavatar, Some(profile_image))
+                        .set_config_ex(Nosync, Config::Selfavatar, Some(profile_image))
                        .await?;
                } else {
                    info!(context, "Do not use unencrypted selfavatar.");
@@ -1564,7 +1564,7 @@ pub(crate) async fn set_profile_image(
            if contact_id == ContactId::SELF {
                if was_encrypted {
                    context
-                        .set_config_internal(Config::Selfavatar, None)
+                        .set_config_ex(Nosync, Config::Selfavatar, None)
                        .await?;
                } else {
                    info!(context, "Do not use unencrypted selfavatar deletion.");
@@ -1597,7 +1597,7 @@ pub(crate) async fn set_status(
    if contact_id == ContactId::SELF {
        if encrypted && has_chat_version {
            context
-                .set_config_internal(Config::Selfstatus, Some(&status))
+                .set_config_ex(Nosync, Config::Selfstatus, Some(&status))
                .await?;
        }
    } else {
--- a/src/context.rs
+++ b/src/context.rs
@@ -153,11 +153,19 @@ impl ContextBuilder {
        self
    }

-    /// Opens the [`Context`].
-    pub async fn open(self) -> Result<Context> {
+    /// Builds the [`Context`] without opening it.
+    pub async fn build(self) -> Result<Context> {
        let context =
            Context::new_closed(&self.dbfile, self.id, self.events, self.stock_strings).await?;
-        let password = self.password.unwrap_or_default();
+        Ok(context)
+    }
+
+    /// Builds the [`Context`] and opens it.
+    ///
+    /// Returns error if context cannot be opened with the given passphrase.
+    pub async fn open(self) -> Result<Context> {
+        let password = self.password.clone().unwrap_or_default();
+        let context = self.build().await?;
        match context.open(password).await? {
            true => Ok(context),
            false => bail!("database could not be decrypted, incorrect or missing password"),
@@ -461,13 +469,13 @@ impl Context {

        // connection
        let mut connection = Imap::new_configured(self, channel::bounded(1).1).await?;
-        connection.prepare(self).await?;
+        let mut session = connection.prepare(self).await?;

        // fetch imap folders
        for folder_meaning in [FolderMeaning::Inbox, FolderMeaning::Mvbox] {
            let (_, watch_folder) = convert_folder_meaning(self, folder_meaning).await?;
            connection
-                .fetch_move_delete(self, &watch_folder, folder_meaning)
+                .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
                .await?;
        }

@@ -484,7 +492,7 @@ impl Context {
        };

        if quota_needs_update {
-            if let Err(err) = self.update_recent_quota(&mut connection).await {
+            if let Err(err) = self.update_recent_quota(&mut session).await {
                warn!(self, "Failed to update quota: {err:#}.");
            }
        }
@@ -1091,12 +1099,14 @@ impl Context {
                .query_map(
                    "SELECT m.id AS id
                 FROM msgs m
+                 INNER JOIN msgs_search ms
+                        ON m.id=ms.rowid
                 LEFT JOIN contacts ct
                        ON m.from_id=ct.id
                 WHERE m.chat_id=?
                   AND m.hidden=0
                   AND ct.blocked=0
-                   AND txt LIKE ?
+                   AND ms.txt LIKE ?
                 ORDER BY m.timestamp,m.id;",
                    (chat_id, str_like_in_text),
                    |row| row.get::<_, MsgId>("id"),
@@ -1124,6 +1134,8 @@ impl Context {
                .query_map(
                    "SELECT m.id AS id
                 FROM msgs m
+                 INNER JOIN msgs_search ms
+                        ON m.id=ms.rowid
                 LEFT JOIN contacts ct
                        ON m.from_id=ct.id
                 LEFT JOIN chats c
@@ -1132,7 +1144,7 @@ impl Context {
                   AND m.hidden=0
                   AND c.blocked!=1
                   AND ct.blocked=0
-                   AND m.txt LIKE ?
+                   AND ms.txt LIKE ?
                 ORDER BY m.id DESC LIMIT 1000",
                    (str_like_in_text,),
                    |row| row.get::<_, MsgId>("id"),
@@ -1529,6 +1541,8 @@ mod tests {
        msg2.set_text("barbaz".to_string());
        send_msg(&alice, chat.id, &mut msg2).await?;

+        alice.send_text(chat.id, "Δ-Chat").await;
+
        // Global search with a part of text finds the message.
        let res = alice.search_msgs(None, "ob").await?;
        assert_eq!(res.len(), 1);
@@ -1541,6 +1555,12 @@ mod tests {
        assert_eq!(res.first(), Some(&msg2.id));
        assert_eq!(res.get(1), Some(&msg1.id));

+        // Search is case-insensitive.
+        for chat_id in [None, Some(chat.id)] {
+            let res = alice.search_msgs(chat_id, "δ-chat").await?;
+            assert_eq!(res.len(), 1);
+        }
+
        // Global search with longer text does not find any message.
        let res = alice.search_msgs(None, "foobarbaz").await?;
        assert!(res.is_empty());
--- a/src/download.rs
+++ b/src/download.rs
@@ -3,13 +3,13 @@
 use std::cmp::max;
 use std::collections::BTreeMap;

-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};

 use crate::config::Config;
 use crate::context::Context;
-use crate::imap::{Imap, ImapActionResult};
+use crate::imap::session::Session;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::tools::time;
@@ -129,9 +129,11 @@ impl Message {
 /// Actually download a message partially downloaded before.
 ///
 /// Most messages are downloaded automatically on fetch instead.
-pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Imap) -> Result<()> {
-    imap.prepare(context).await?;
-
+pub(crate) async fn download_msg(
+    context: &Context,
+    msg_id: MsgId,
+    session: &mut Session,
+) -> Result<()> {
    let msg = Message::load_from_db(context, msg_id).await?;
    let row = context
        .sql
@@ -152,7 +154,7 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
        return Err(anyhow!("Call download_full() again to try over."));
    };

-    match imap
+    session
        .fetch_single_msg(
            context,
            &server_folder,
@@ -160,16 +162,11 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
            server_uid,
            msg.rfc724_mid.clone(),
        )
-        .await
-    {
-        ImapActionResult::RetryLater | ImapActionResult::Failed => {
-            Err(anyhow!("Call download_full() again to try over."))
-        }
-        ImapActionResult::Success => Ok(()),
-    }
+        .await?;
+    Ok(())
 }

-impl Imap {
+impl Session {
    /// Download a single message and pipe it to receive_imf().
    ///
    /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
@@ -181,20 +178,19 @@ impl Imap {
        uidvalidity: u32,
        uid: u32,
        rfc724_mid: String,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
+    ) -> Result<()> {
+        if uid == 0 {
+            bail!("Attempt to fetch UID 0");
        }

+        self.select_folder(context, Some(folder)).await?;
+
        // we are connected, and the folder is selected
        info!(context, "Downloading message {}/{} fully...", folder, uid);

        let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
        uid_message_ids.insert(uid, rfc724_mid);
-        let (last_uid, _received) = match self
+        let (last_uid, _received) = self
            .fetch_many_msgs(
                context,
                folder,
@@ -204,16 +200,11 @@ impl Imap {
                false,
                false,
            )
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return ImapActionResult::Failed,
-        };
+            .await?;
        if last_uid.is_none() {
-            ImapActionResult::Failed
-        } else {
-            ImapActionResult::Success
+            bail!("Failed to fetch UID {uid}");
        }
+        Ok(())
    }
 }

--- a/src/imap.rs
+++ b/src/imap.rs
--- a/src/imap/idle.rs
+++ b/src/imap/idle.rs
@@ -4,13 +4,12 @@ use anyhow::{bail, Context as _, Result};
 use async_channel::Receiver;
 use async_imap::extensions::idle::IdleResponse;
 use futures_lite::FutureExt;
+use tokio::time::timeout;

 use super::session::Session;
 use super::Imap;
-use crate::config::Config;
 use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
-use crate::log::LogExt;
 use crate::tools::{self, time_elapsed};

 /// Timeout after which IDLE is finished
@@ -98,97 +97,38 @@ impl Session {
 }

 impl Imap {
+    /// Idle using polling.
    pub(crate) async fn fake_idle(
        &mut self,
        context: &Context,
-        watch_folder: Option<String>,
+        session: &mut Session,
+        watch_folder: String,
        folder_meaning: FolderMeaning,
-    ) {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
-
+    ) -> Result<()> {
        let fake_idle_start_time = tools::Time::now();

-        // Do not poll, just wait for an interrupt when no folder is passed in.
-        let watch_folder = if let Some(watch_folder) = watch_folder {
-            watch_folder
-        } else {
-            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
-            self.idle_interrupt_receiver.recv().await.ok();
-            return;
-        };
        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);

-        const TIMEOUT_INIT_MS: u64 = 60_000;
-        let mut timeout_ms: u64 = TIMEOUT_INIT_MS;
-        enum Event {
-            Tick,
-            Interrupt,
-        }
-        // loop until we are interrupted or if we fetched something
+        // Loop until we are interrupted or until we fetch something.
        loop {
-            use futures::future::FutureExt;
-            use rand::Rng;
-
-            let mut interval = tokio::time::interval(Duration::from_millis(timeout_ms));
-            timeout_ms = timeout_ms
-                .saturating_add(rand::thread_rng().gen_range((timeout_ms / 2)..=timeout_ms));
-            interval.tick().await; // The first tick completes immediately.
-            match interval
-                .tick()
-                .map(|_| Event::Tick)
-                .race(
-                    self.idle_interrupt_receiver
-                        .recv()
-                        .map(|_| Event::Interrupt),
-                )
-                .await
-            {
-                Event::Tick => {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.prepare(context).await {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if let Some(session) = &self.session {
-                        if session.can_idle()
-                            && !context
-                                .get_config_bool(Config::DisableIdle)
-                                .await
-                                .context("Failed to get disable_idle config")
-                                .log_err(context)
-                                .unwrap_or_default()
-                        {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
+            match timeout(Duration::from_secs(60), self.idle_interrupt_receiver.recv()).await {
+                Err(_) => {
+                    // Let's see if fetching messages results
                    // in anything.  If so, we behave as if IDLE had data but
                    // will have already fetched the messages so perform_*_fetch
                    // will not find any new.
-                    match self
-                        .fetch_new_messages(context, &watch_folder, folder_meaning, false)
-                        .await
-                    {
-                        Ok(res) => {
-                            info!(context, "fetch_new_messages returned {:?}", res);
-                            timeout_ms = TIMEOUT_INIT_MS;
-                            if res {
-                                break;
-                            }
-                        }
-                        Err(err) => {
-                            error!(context, "could not fetch from folder: {:#}", err);
-                            self.trigger_reconnect(context);
-                        }
+                    let res = self
+                        .fetch_new_messages(context, session, &watch_folder, folder_meaning, false)
+                        .await?;
+
+                    info!(context, "fetch_new_messages returned {:?}", res);
+
+                    if res {
+                        break;
                    }
                }
-                Event::Interrupt => {
-                    info!(context, "Fake IDLE interrupted");
+                Ok(_) => {
+                    info!(context, "Fake IDLE interrupted.");
                    break;
                }
            }
@@ -199,5 +139,6 @@ impl Imap {
            "IMAP-fake-IDLE done after {:.4}s",
            time_elapsed(&fake_idle_start_time).as_millis() as f64 / 1000.,
        );
+        Ok(())
    }
 }
--- a/src/imap/scan_folders.rs
+++ b/src/imap/scan_folders.rs
@@ -1,18 +1,21 @@
 use std::collections::BTreeMap;

 use anyhow::{Context as _, Result};
-use futures::TryStreamExt;

 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
-use crate::imap::Imap;
+use crate::imap::{session::Session, Imap};
 use crate::log::LogExt;
 use crate::tools::{self, time_elapsed};
 use crate::{context::Context, imap::FolderMeaning};

 impl Imap {
    /// Returns true if folders were scanned, false if scanning was postponed.
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
+    pub(crate) async fn scan_folders(
+        &mut self,
+        context: &Context,
+        session: &mut Session,
+    ) -> Result<bool> {
        // First of all, debounce to once per minute:
        let mut last_scan = context.last_full_folder_scan.lock().await;
        if let Some(last_scan) = *last_scan {
@@ -27,8 +30,7 @@ impl Imap {
        }
        info!(context, "Starting full folder scan");

-        self.prepare(context).await?;
-        let folders = self.list_folders().await?;
+        let folders = session.list_folders().await?;
        let watched_folders = get_watched_folders(context).await?;

        let mut folder_configs = BTreeMap::new();
@@ -64,18 +66,16 @@ impl Imap {
                && folder_meaning != FolderMeaning::Drafts
                && folder_meaning != FolderMeaning::Trash
            {
-                let session = self.session.as_mut().context("no session")?;
                // Drain leftover unsolicited EXISTS messages
                session.server_sent_unsolicited_exists(context)?;

                loop {
-                    self.fetch_move_delete(context, folder.name(), folder_meaning)
+                    self.fetch_move_delete(context, session, folder.name(), folder_meaning)
                        .await
                        .context("Can't fetch new msgs in scanned folder")
                        .log_err(context)
                        .ok();

-                    let session = self.session.as_mut().context("no session")?;
                    // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
                    if !session.server_sent_unsolicited_exists(context)? {
                        break;
@@ -97,18 +97,6 @@ impl Imap {
        last_scan.replace(tools::Time::now());
        Ok(true)
    }
-
-    /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
-        let session = self.session.as_mut();
-        let session = session.context("No IMAP connection")?;
-        let list = session
-            .list(Some(""), Some("*"))
-            .await?
-            .try_collect()
-            .await?;
-        Ok(list)
-    }
 }

 pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {
--- a/src/imap/select_folder.rs
+++ b/src/imap/select_folder.rs
@@ -3,6 +3,7 @@
 use anyhow::Context as _;

 use super::session::Session as ImapSession;
+use super::{get_uid_next, get_uidvalidity, set_modseq, set_uid_next, set_uidvalidity};
 use crate::context::Context;

 type Result<T> = std::result::Result<T, Error>;
@@ -51,7 +52,7 @@ impl ImapSession {
    /// Selects a folder, possibly updating uid_validity and, if needed,
    /// expunging the folder to remove delete-marked messages.
    /// Returns whether a new folder was selected.
-    pub(super) async fn select_folder(
+    pub(crate) async fn select_folder(
        &mut self,
        context: &Context,
        folder: Option<&str>,
@@ -127,10 +128,135 @@ impl ImapSession {
            },
        }
    }
+
+    /// Selects a folder and takes care of UIDVALIDITY changes.
+    ///
+    /// When selecting a folder for the first time, sets the uid_next to the current
+    /// mailbox.uid_next so that no old emails are fetched.
+    ///
+    /// Returns Result<new_emails> (i.e. whether new emails arrived),
+    /// if in doubt, returns new_emails=true so emails are fetched.
+    pub(crate) async fn select_with_uidvalidity(
+        &mut self,
+        context: &Context,
+        folder: &str,
+    ) -> Result<bool> {
+        let newly_selected = self
+            .select_or_create_folder(context, folder)
+            .await
+            .with_context(|| format!("failed to select or create folder {folder}"))?;
+        let mailbox = self
+            .selected_mailbox
+            .as_mut()
+            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
+
+        let old_uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
+        let old_uid_next = get_uid_next(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+
+        let new_uid_validity = mailbox
+            .uid_validity
+            .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
+        let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
+            Some(uid_next)
+        } else {
+            warn!(
+                context,
+                "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
+            );
+
+            // RFC 3501 says STATUS command SHOULD NOT be used
+            // on the currently selected mailbox because the same
+            // information can be obtained by other means,
+            // such as reading SELECT response.
+            //
+            // However, it also says that UIDNEXT is REQUIRED
+            // in the SELECT response and if we are here,
+            // it is actually not returned.
+            //
+            // In particular, Winmail Pro Mail Server 5.1.0616
+            // never returns UIDNEXT in SELECT response,
+            // but responds to "STATUS INBOX (UIDNEXT)" command.
+            let status = self
+                .inner
+                .status(folder, "(UIDNEXT)")
+                .await
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+
+            if status.uid_next.is_none() {
+                // This happens with mail.163.com as of 2023-11-26.
+                // It does not return UIDNEXT on SELECT and returns invalid
+                // `* STATUS "INBOX" ()` response on explicit request for UIDNEXT.
+                warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT.");
+            }
+            status.uid_next
+        };
+        mailbox.uid_next = new_uid_next;
+
+        if new_uid_validity == old_uid_validity {
+            let new_emails = if newly_selected == NewlySelected::No {
+                // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next
+                // was not updated and may contain an incorrect value. So, just return true so that
+                // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
+                // new messages is only one command, just as a SELECT command)
+                true
+            } else if let Some(new_uid_next) = new_uid_next {
+                if new_uid_next < old_uid_next {
+                    warn!(
+                        context,
+                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                    );
+                    set_uid_next(context, folder, new_uid_next).await?;
+                    context.schedule_resync().await?;
+                }
+                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
+            } else {
+                // We have no UIDNEXT and if in doubt, return true.
+                true
+            };
+
+            return Ok(new_emails);
+        }
+
+        // UIDVALIDITY is modified, reset highest seen MODSEQ.
+        set_modseq(context, folder, 0).await?;
+
+        // ==============  uid_validity has changed or is being set the first time.  ==============
+
+        let new_uid_next = new_uid_next.unwrap_or_default();
+        set_uid_next(context, folder, new_uid_next).await?;
+        set_uidvalidity(context, folder, new_uid_validity).await?;
+
+        // Collect garbage entries in `imap` table.
+        context
+            .sql
+            .execute(
+                "DELETE FROM imap WHERE folder=? AND uidvalidity!=?",
+                (&folder, new_uid_validity),
+            )
+            .await?;
+
+        if old_uid_validity != 0 || old_uid_next != 0 {
+            context.schedule_resync().await?;
+        }
+        info!(
+            context,
+            "uid/validity change folder {}: new {}/{} previous {}/{}.",
+            folder,
+            new_uid_next,
+            new_uid_validity,
+            old_uid_next,
+            old_uid_validity,
+        );
+        Ok(false)
+    }
 }

 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
-pub(super) enum NewlySelected {
+pub(crate) enum NewlySelected {
    /// The folder was newly selected during this call to select_folder().
    Yes,
    /// No SELECT command was run because the folder already was selected
--- a/src/imap/session.rs
+++ b/src/imap/session.rs
@@ -1,11 +1,32 @@
+use std::cmp;
+use std::collections::BTreeMap;
 use std::ops::{Deref, DerefMut};

+use anyhow::{Context as _, Result};
 use async_imap::types::Mailbox;
 use async_imap::Session as ImapSession;
+use futures::TryStreamExt;

+use crate::constants::DC_FETCH_EXISTING_MSGS_COUNT;
 use crate::imap::capabilities::Capabilities;
 use crate::net::session::SessionStream;

+/// Prefetch:
+/// - Message-ID to check if we already have the message.
+/// - In-Reply-To and References to check if message is a reply to chat message.
+/// - Chat-Version to check if a message is a chat message
+/// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
+///   not necessarily sent by Delta Chat.
+const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
+                              MESSAGE-ID \
+                              DATE \
+                              X-MICROSOFT-ORIGINAL-MESSAGE-ID \
+                              FROM \
+                              IN-REPLY-TO REFERENCES \
+                              CHAT-VERSION \
+                              AUTOCRYPT-SETUP-MESSAGE\
+                              )])";
+
 #[derive(Debug)]
 pub(crate) struct Session {
    pub(super) inner: ImapSession<Box<dyn SessionStream>>,
@@ -68,4 +89,71 @@ impl Session {
    pub fn can_metadata(&self) -> bool {
        self.capabilities.can_metadata
    }
+
+    /// Returns the names of all folders on the IMAP server.
+    pub async fn list_folders(&mut self) -> Result<Vec<async_imap::types::Name>> {
+        let list = self.list(Some(""), Some("*")).await?.try_collect().await?;
+        Ok(list)
+    }
+
+    /// Prefetch all messages greater than or equal to `uid_next`. Returns a list of fetch results
+    /// in the order of ascending delivery time to the server (INTERNALDATE).
+    pub(crate) async fn prefetch(
+        &mut self,
+        uid_next: u32,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        // fetch messages with larger UID than the last one seen
+        let set = format!("{uid_next}:*");
+        let mut list = self
+            .uid_fetch(set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                // If the mailbox is not empty, results always include
+                // at least one UID, even if last_seen_uid+1 is past
+                // the last UID in the mailbox.  It happens because
+                // uid:* is interpreted the same way as *:uid.
+                // See <https://tools.ietf.org/html/rfc3501#page-61> for
+                // standard reference. Therefore, sometimes we receive
+                // already seen messages and have to filter them out.
+                if msg_uid >= uid_next {
+                    msgs.insert((msg.internal_date(), msg_uid), msg);
+                }
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
+
+    /// Like prefetch(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
+    pub(crate) async fn prefetch_existing_msgs(
+        &mut self,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        let exists: i64 = {
+            let mailbox = self.selected_mailbox.as_ref().context("no mailbox")?;
+            mailbox.exists.into()
+        };
+
+        // Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
+        // Sequence numbers are sequential. If there are 1000 messages in the inbox,
+        // we can fetch the sequence numbers 900-1000 and get the last 100 messages.
+        let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT + 1);
+        let set = format!("{first}:{exists}");
+        let mut list = self
+            .fetch(&set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP Could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert((msg.internal_date(), msg_uid), msg);
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
 }
--- a/src/key.rs
+++ b/src/key.rs
@@ -79,7 +79,7 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 }

 pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPublicKey> {
-    match context
+    let public_key = context
        .sql
        .query_row_optional(
            "SELECT public_key
@@ -91,8 +91,8 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
                Ok(bytes)
            },
        )
-        .await?
-    {
+        .await?;
+    match public_key {
        Some(bytes) => SignedPublicKey::from_slice(&bytes),
        None => {
            let keypair = generate_keypair(context).await?;
@@ -102,7 +102,7 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
 }

 pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecretKey> {
-    match context
+    let private_key = context
        .sql
        .query_row_optional(
            "SELECT private_key
@@ -114,8 +114,8 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
                Ok(bytes)
            },
        )
-        .await?
-    {
+        .await?;
+    match private_key {
        Some(bytes) => SignedSecretKey::from_slice(&bytes),
        None => {
            let keypair = generate_keypair(context).await?;
@@ -255,7 +255,7 @@ pub(crate) async fn load_keypair(

 /// Use of a key pair for encryption or decryption.
 ///
-/// This is used by [store_self_keypair] to know what kind of key is
+/// This is used by `store_self_keypair` to know what kind of key is
 /// being saved.
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum KeyPairUse {
@@ -277,7 +277,7 @@ pub enum KeyPairUse {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub async fn store_self_keypair(
+pub(crate) async fn store_self_keypair(
    context: &Context,
    keypair: &KeyPair,
    default: KeyPairUse,
--- a/src/mimefactory.rs
+++ b/src/mimefactory.rs
@@ -933,7 +933,6 @@ impl<'a> MimeFactory<'a> {
        };
        let command = self.msg.param.get_cmd();
        let mut placeholdertext = None;
-        let mut meta_part = None;

        let send_verified_headers = match chat.typ {
            Chattype::Single => true,
@@ -1119,17 +1118,13 @@ impl<'a> MimeFactory<'a> {

        if let Some(grpimage) = grpimage {
            info!(context, "setting group image '{}'", grpimage);
-            let mut meta = Message {
-                viewtype: Viewtype::Image,
-                ..Default::default()
-            };
-            meta.param.set(Param::File, grpimage);
-
-            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
-            meta_part = Some(mail);
-            headers
-                .protected
-                .push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            let avatar = build_avatar_file(context, grpimage)
+                .await
+                .context("Cannot attach group image")?;
+            headers.hidden.push(Header::new(
+                "Chat-Group-Avatar".into(),
+                format!("base64:{avatar}"),
+            ));
        }

        if self.msg.viewtype == Viewtype::Sticker {
@@ -1253,10 +1248,6 @@ impl<'a> MimeFactory<'a> {
            parts.push(file_part);
        }

-        if let Some(meta_part) = meta_part {
-            parts.push(meta_part);
-        }
-
        if let Some(msg_kml_part) = self.get_message_kml_part() {
            parts.push(msg_kml_part);
        }
@@ -1288,7 +1279,7 @@ impl<'a> MimeFactory<'a> {

        if self.attach_selfavatar {
            match context.get_config(Config::Selfavatar).await? {
-                Some(path) => match build_selfavatar_file(context, &path).await {
+                Some(path) => match build_avatar_file(context, &path).await {
                    Ok(avatar) => headers.hidden.push(Header::new(
                        "Chat-User-Avatar".into(),
                        format!("base64:{avatar}"),
@@ -1497,8 +1488,11 @@ async fn build_body_file(
    Ok((mail, filename_to_send))
 }

-async fn build_selfavatar_file(context: &Context, path: &str) -> Result<String> {
-    let blob = BlobObject::from_path(context, path.as_ref())?;
+async fn build_avatar_file(context: &Context, path: &str) -> Result<String> {
+    let blob = match path.starts_with("$BLOBDIR/") {
+        true => BlobObject::from_name(context, path.to_string())?,
+        false => BlobObject::from_path(context, path.as_ref())?,
+    };
    let body = fs::read(blob.to_abs_path()).await?;
    let encoded_body = wrapped_base64_encode(&body);
    Ok(encoded_body)
--- a/src/mimeparser.rs
+++ b/src/mimeparser.rs
@@ -265,8 +265,10 @@ impl MimeMessage {
                for field in &part.headers {
                    let key = field.get_key().to_lowercase();

-                    // For now only Chat-User-Avatar can be hidden.
-                    if !headers.contains_key(&key) && key == "chat-user-avatar" {
+                    // For now only avatar headers can be hidden.
+                    if !headers.contains_key(&key)
+                        && (key == "chat-user-avatar" || key == "chat-group-avatar")
+                    {
                        headers.insert(key.to_string(), field.get_value());
                    }
                }
--- a/src/pgp.rs
+++ b/src/pgp.rs
@@ -151,7 +151,7 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
    let user_id = format!("<{addr}>");
    let key_params = SecretKeyParamsBuilder::default()
        .key_type(signing_key_type)
-        .can_create_certificates(true)
+        .can_certify(true)
        .can_sign(true)
        .primary_user_id(user_id)
        .passphrase(None)
--- a/src/quota.rs
+++ b/src/quota.rs
@@ -10,7 +10,6 @@ use crate::config::Config;
 use crate::context::Context;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::session::Session as ImapSession;
-use crate::imap::Imap;
 use crate::message::{Message, Viewtype};
 use crate::tools;
 use crate::{stock_str, EventType};
@@ -111,13 +110,7 @@ impl Context {
    /// As the message is added only once, the user is not spammed
    /// in case for some providers the quota is always at ~100%
    /// and new space is allocated as needed.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> {
-        if let Err(err) = imap.prepare(self).await {
-            warn!(self, "could not connect: {:#}", err);
-            return Ok(());
-        }
-
-        let session = imap.session.as_mut().context("no session")?;
+    pub(crate) async fn update_recent_quota(&self, session: &mut ImapSession) -> Result<()> {
        let quota = if session.can_check_quota() {
            let folders = get_watched_folders(self).await?;
            get_unique_quota_roots_and_usage(session, folders).await
--- a/src/scheduler.rs
+++ b/src/scheduler.rs
@@ -19,7 +19,7 @@ use crate::context::Context;
 use crate::download::{download_msg, DownloadState};
 use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::events::EventType;
-use crate::imap::{FolderMeaning, Imap};
+use crate::imap::{session::Session, FolderMeaning, Imap};
 use crate::location;
 use crate::log::LogExt;
 use crate::message::MsgId;
@@ -330,7 +330,7 @@ pub(crate) struct Scheduler {
    recently_seen_loop: RecentlySeenLoop,
 }

-async fn download_msgs(context: &Context, imap: &mut Imap) -> Result<()> {
+async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
    let msg_ids = context
        .sql
        .query_map(
@@ -349,7 +349,7 @@ async fn download_msgs(context: &Context, imap: &mut Imap) -> Result<()> {
        .await?;

    for msg_id in msg_ids {
-        if let Err(err) = download_msg(context, msg_id, imap).await {
+        if let Err(err) = download_msg(context, msg_id, session).await {
            warn!(context, "Failed to download message {msg_id}: {:#}.", err);

            // Update download state to failure
@@ -392,84 +392,26 @@ async fn inbox_loop(
            return;
        };

+        let mut old_session: Option<Session> = None;
        loop {
-            {
-                // Update quota no more than once a minute.
-                let quota_needs_update = {
-                    let quota = ctx.quota.read().await;
-                    quota
-                        .as_ref()
-                        .filter(|quota| time_elapsed(&quota.modified) > Duration::from_secs(60))
-                        .is_none()
-                };
-
-                if quota_needs_update {
-                    if let Err(err) = ctx.update_recent_quota(&mut connection).await {
-                        warn!(ctx, "Failed to update quota: {:#}.", err);
+            let session = if let Some(session) = old_session.take() {
+                session
+            } else {
+                match connection.prepare(&ctx).await {
+                    Err(err) => {
+                        warn!(ctx, "Failed to prepare connection: {:#}.", err);
+                        continue;
                    }
-                }
-            }
-
-            let resync_requested = ctx.resync_request.swap(false, Ordering::Relaxed);
-            if resync_requested {
-                if let Err(err) = connection.resync_folders(&ctx).await {
-                    warn!(ctx, "Failed to resync folders: {:#}.", err);
-                    ctx.resync_request.store(true, Ordering::Relaxed);
-                }
-            }
-
-            maybe_add_time_based_warnings(&ctx).await;
-
-            match ctx.get_config_i64(Config::LastHousekeeping).await {
-                Ok(last_housekeeping_time) => {
-                    let next_housekeeping_time =
-                        last_housekeeping_time.saturating_add(60 * 60 * 24);
-                    if next_housekeeping_time <= time() {
-                        sql::housekeeping(&ctx).await.log_err(&ctx).ok();
-                    }
-                }
-                Err(err) => {
-                    warn!(ctx, "Failed to get last housekeeping time: {}", err);
+                    Ok(session) => session,
                }
            };

-            match ctx.get_config_bool(Config::FetchedExistingMsgs).await {
-                Ok(fetched_existing_msgs) => {
-                    if !fetched_existing_msgs {
-                        // Consider it done even if we fail.
-                        //
-                        // This operation is not critical enough to retry,
-                        // especially if the error is persistent.
-                        if let Err(err) = ctx
-                            .set_config_internal(
-                                Config::FetchedExistingMsgs,
-                                config::from_bool(true),
-                            )
-                            .await
-                        {
-                            warn!(ctx, "Can't set Config::FetchedExistingMsgs: {:#}", err);
-                        }
-
-                        if let Err(err) = connection.fetch_existing_msgs(&ctx).await {
-                            warn!(ctx, "Failed to fetch existing messages: {:#}", err);
-                            connection.trigger_reconnect(&ctx);
-                        }
-                    }
-                }
-                Err(err) => {
-                    warn!(ctx, "Can't get Config::FetchedExistingMsgs: {:#}", err);
+            match inbox_fetch_idle(&ctx, &mut connection, session).await {
+                Err(err) => warn!(ctx, "Failed fetch_idle: {err:#}"),
+                Ok(session) => {
+                    old_session = Some(session);
                }
            }
-
-            if let Err(err) = download_msgs(&ctx, &mut connection).await {
-                warn!(ctx, "Failed to download messages: {:#}", err);
-            }
-
-            if let Err(err) = connection.fetch_metadata(&ctx).await {
-                warn!(ctx, "Failed to fetch metadata: {err:#}.");
-            }
-
-            fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await;
        }
    };

@@ -515,89 +457,123 @@ pub async fn convert_folder_meaning(
    Ok((folder_config, watch_folder))
 }

-/// Implement a single iteration of IMAP loop.
-///
-/// This function performs all IMAP operations on a single folder, selecting it if necessary and
-/// handling all the errors. In case of an error, it is logged, but not propagated upwards. If
-/// critical operation fails such as fetching new messages fails, connection is reset via
-/// `trigger_reconnect`, so a fresh one can be opened.
-async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: FolderMeaning) {
-    let create_mvbox = true;
-    if let Err(err) = connection
-        .ensure_configured_folders(ctx, create_mvbox)
-        .await
-    {
-        warn!(
-            ctx,
-            "Cannot watch {folder_meaning}, ensure_configured_folders() failed: {:#}", err,
-        );
-        connection
-            .fake_idle(ctx, None, FolderMeaning::Unknown)
-            .await;
-        return;
+async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) -> Result<Session> {
+    // Update quota no more than once a minute.
+    let quota_needs_update = {
+        let quota = ctx.quota.read().await;
+        quota
+            .as_ref()
+            .filter(|quota| time_elapsed(&quota.modified) > Duration::from_secs(60))
+            .is_none()
+    };
+    if quota_needs_update {
+        if let Err(err) = ctx.update_recent_quota(&mut session).await {
+            warn!(ctx, "Failed to update quota: {:#}.", err);
+        }
    }
-    let (folder_config, watch_folder) = match convert_folder_meaning(ctx, folder_meaning).await {
-        Ok(meaning) => meaning,
-        Err(error) => {
-            // Warning instead of error because the folder may not be configured.
-            // For example, this happens if the server does not have Sent folder
-            // but watching Sent folder is enabled.
-            warn!(ctx, "Error converting IMAP Folder name: {:?}", error);
-            connection.connectivity.set_not_configured(ctx).await;
-            connection
-                .fake_idle(ctx, None, FolderMeaning::Unknown)
-                .await;
-            return;
+
+    let resync_requested = ctx.resync_request.swap(false, Ordering::Relaxed);
+    if resync_requested {
+        if let Err(err) = session.resync_folders(ctx).await {
+            warn!(ctx, "Failed to resync folders: {:#}.", err);
+            ctx.resync_request.store(true, Ordering::Relaxed);
+        }
+    }
+
+    maybe_add_time_based_warnings(ctx).await;
+
+    match ctx.get_config_i64(Config::LastHousekeeping).await {
+        Ok(last_housekeeping_time) => {
+            let next_housekeeping_time = last_housekeeping_time.saturating_add(60 * 60 * 24);
+            if next_housekeeping_time <= time() {
+                sql::housekeeping(ctx).await.log_err(ctx).ok();
+            }
+        }
+        Err(err) => {
+            warn!(ctx, "Failed to get last housekeeping time: {}", err);
        }
    };

-    // connect and fake idle if unable to connect
-    if let Err(err) = connection
-        .prepare(ctx)
-        .await
-        .context("prepare IMAP connection")
-    {
-        warn!(ctx, "{:#}", err);
-        connection.trigger_reconnect(ctx);
-        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
-    }
+    match ctx.get_config_bool(Config::FetchedExistingMsgs).await {
+        Ok(fetched_existing_msgs) => {
+            if !fetched_existing_msgs {
+                // Consider it done even if we fail.
+                //
+                // This operation is not critical enough to retry,
+                // especially if the error is persistent.
+                if let Err(err) = ctx
+                    .set_config_internal(Config::FetchedExistingMsgs, config::from_bool(true))
+                    .await
+                {
+                    warn!(ctx, "Can't set Config::FetchedExistingMsgs: {:#}", err);
+                }

-    if folder_config == Config::ConfiguredInboxFolder {
-        if let Some(session) = connection.session.as_mut() {
-            session
-                .store_seen_flags_on_imap(ctx)
-                .await
-                .context("store_seen_flags_on_imap")
-                .log_err(ctx)
-                .ok();
-        } else {
-            warn!(ctx, "No session even though we just prepared it");
+                if let Err(err) = imap.fetch_existing_msgs(ctx, &mut session).await {
+                    warn!(ctx, "Failed to fetch existing messages: {:#}", err);
+                }
+            }
+        }
+        Err(err) => {
+            warn!(ctx, "Can't get Config::FetchedExistingMsgs: {:#}", err);
        }
    }

-    // Fetch the watched folder.
-    if let Err(err) = connection
-        .fetch_move_delete(ctx, &watch_folder, folder_meaning)
+    download_msgs(ctx, &mut session)
        .await
-        .context("fetch_move_delete")
-    {
-        connection.trigger_reconnect(ctx);
-        warn!(ctx, "{:#}", err);
-        return;
+        .context("Failed to download messages")?;
+    session
+        .fetch_metadata(ctx)
+        .await
+        .context("Failed to fetch metadata")?;
+
+    let session = fetch_idle(ctx, imap, session, FolderMeaning::Inbox).await?;
+    Ok(session)
+}
+
+/// Implement a single iteration of IMAP loop.
+///
+/// This function performs all IMAP operations on a single folder, selecting it if necessary and
+/// handling all the errors. In case of an error, an error is returned and connection is dropped,
+/// otherwise connection is returned.
+async fn fetch_idle(
+    ctx: &Context,
+    connection: &mut Imap,
+    mut session: Session,
+    folder_meaning: FolderMeaning,
+) -> Result<Session> {
+    let (folder_config, watch_folder) = match convert_folder_meaning(ctx, folder_meaning).await {
+        Ok(meaning) => meaning,
+        Err(err) => {
+            // Warning instead of error because the folder may not be configured.
+            // For example, this happens if the server does not have Sent folder
+            // but watching Sent folder is enabled.
+            warn!(ctx, "Error converting IMAP Folder name: {err:#}.");
+            connection.connectivity.set_not_configured(ctx).await;
+            connection.idle_interrupt_receiver.recv().await.ok();
+            return Err(err);
+        }
+    };
+
+    if folder_config == Config::ConfiguredInboxFolder {
+        session
+            .store_seen_flags_on_imap(ctx)
+            .await
+            .context("store_seen_flags_on_imap")?;
    }

+    // Fetch the watched folder.
+    connection
+        .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
+        .await
+        .context("fetch_move_delete")?;
+
    // Mark expired messages for deletion. Marked messages will be deleted from the server
    // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
    // called right before `fetch_move_delete` because it is not well optimized and would
    // otherwise slow down message fetching.
    delete_expired_imap_messages(ctx)
        .await
-        .context("delete_expired_imap_messages")
-        .log_err(ctx)
-        .ok();
+        .context("delete_expired_imap_messages")?;

    // Scan additional folders only after finishing fetching the watched folder.
    //
@@ -605,7 +581,11 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
    // be able to scan all folders before time is up if there are many of them.
    if folder_config == Config::ConfiguredInboxFolder {
        // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
-        match connection.scan_folders(ctx).await.context("scan_folders") {
+        match connection
+            .scan_folders(ctx, &mut session)
+            .await
+            .context("scan_folders")
+        {
            Err(err) => {
                // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
                // but maybe just one folder can't be selected or something
@@ -618,22 +598,17 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
                // In most cases this will select the watched folder and return because there are
                // no new messages. We want to select the watched folder anyway before going IDLE
                // there, so this does not take additional protocol round-trip.
-                if let Err(err) = connection
-                    .fetch_move_delete(ctx, &watch_folder, folder_meaning)
+                connection
+                    .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
                    .await
-                    .context("fetch_move_delete after scan_folders")
-                {
-                    connection.trigger_reconnect(ctx);
-                    warn!(ctx, "{:#}", err);
-                    return;
-                }
+                    .context("fetch_move_delete after scan_folders")?;
            }
            Ok(false) => {}
        }
    }

    // Synchronize Seen flags.
-    connection
+    session
        .sync_seen_flags(ctx, &watch_folder)
        .await
        .context("sync_seen_flags")
@@ -643,13 +618,6 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
    connection.connectivity.set_idle(ctx).await;

    ctx.emit_event(EventType::ImapInboxIdle);
-    let Some(session) = connection.session.take() else {
-        warn!(ctx, "No IMAP session, going to fake idle.");
-        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
-    };

    if !session.can_idle() {
        info!(
@@ -657,9 +625,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
            "IMAP session does not support IDLE, going to fake idle."
        );
        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
+            .fake_idle(ctx, &mut session, watch_folder, folder_meaning)
+            .await?;
+        return Ok(session);
    }

    if ctx
@@ -671,29 +639,22 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
    {
        info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
+            .fake_idle(ctx, &mut session, watch_folder, folder_meaning)
+            .await?;
+        return Ok(session);
    }

    info!(ctx, "IMAP session supports IDLE, using it.");
-    match session
+    let session = session
        .idle(
            ctx,
            connection.idle_interrupt_receiver.clone(),
            &watch_folder,
        )
        .await
-        .context("idle")
-    {
-        Ok(session) => {
-            connection.session = Some(session);
-        }
-        Err(err) => {
-            connection.trigger_reconnect(ctx);
-            warn!(ctx, "{:#}", err);
-        }
-    }
+        .context("idle")?;
+
+    Ok(session)
 }

 async fn simple_imap_loop(
@@ -719,8 +680,26 @@ async fn simple_imap_loop(
            return;
        }

+        let mut old_session: Option<Session> = None;
        loop {
-            fetch_idle(&ctx, &mut connection, folder_meaning).await;
+            let session = if let Some(session) = old_session.take() {
+                session
+            } else {
+                match connection.prepare(&ctx).await {
+                    Err(err) => {
+                        warn!(ctx, "Failed to prepare connection: {:#}.", err);
+                        continue;
+                    }
+                    Ok(session) => session,
+                }
+            };
+
+            match fetch_idle(&ctx, &mut connection, session, folder_meaning).await {
+                Err(err) => warn!(ctx, "Failed fetch_idle: {err:#}"),
+                Ok(session) => {
+                    old_session = Some(session);
+                }
+            }
        }
    };

--- a/src/smtp.rs
+++ b/src/smtp.rs
@@ -804,7 +804,7 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
        .sql
        .execute("DELETE FROM smtp_mdns WHERE retries > 6", [])
        .await?;
-    let msg_row = match context
+    let Some(msg_row) = context
        .sql
        .query_row_optional(
            "SELECT msg_id, from_id FROM smtp_mdns ORDER BY retries LIMIT 1",
@@ -816,9 +816,8 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
            },
        )
        .await?
-    {
-        Some(msg_row) => msg_row,
-        None => return Ok(false),
+    else {
+        return Ok(false);
    };
    let (msg_id, contact_id) = msg_row;

--- a/src/sql/migrations.rs
+++ b/src/sql/migrations.rs
@@ -911,6 +911,24 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
        .await?;
    }

+    if dbversion < 111 {
+        sql.execute_migration(
+            "CREATE VIRTUAL TABLE msgs_search USING fts5(txt, content='msgs', content_rowid='id'); \
+            CREATE TRIGGER msgs_search_insert AFTER INSERT ON msgs BEGIN \
+              INSERT INTO msgs_search (rowid, txt) VALUES (new.id, new.txt); \
+            END; \
+            CREATE TRIGGER msgs_search_delete AFTER DELETE ON msgs BEGIN \
+              INSERT INTO msgs_search (msgs_search, rowid, txt) VALUES ('delete', old.id, old.txt); \
+            END; \
+            CREATE TRIGGER msgs_search_update AFTER UPDATE ON msgs BEGIN \
+              INSERT INTO msgs_search (msgs_search, rowid, txt) VALUES ('delete', old.id, old.txt); \
+              INSERT INTO msgs_search (rowid, txt) VALUES (new.id, new.txt); \
+            END; \
+            INSERT INTO msgs_search (msgs_search) VALUES ('rebuild');",
+            111,
+        ).await?;
+    }
+
    let new_version = sql
        .get_raw_config_int(VERSION_CFG)
        .await?
--- a/src/tests/verified_chats.rs
+++ b/src/tests/verified_chats.rs
@@ -43,7 +43,7 @@ async fn check_verified_oneonone_chat(broken_by_classical_email: bool) {
            b"Subject: Re: Message from alice\r\n\
          From: <bob@example.net>\r\n\
          To: <alice@example.org>\r\n\
-          Date: Mon, 12 Dec 2022 14:33:39 +0000\r\n\
+          Date: Mon, 12 Dec 3000 14:33:39 +0000\r\n\
          Message-ID: <abcd@example.net>\r\n\
          \r\n\
          Heyho!\r\n",
@@ -52,26 +52,28 @@ async fn check_verified_oneonone_chat(broken_by_classical_email: bool) {
        .await
        .unwrap()
        .unwrap();
+        // Bob's contact is still verified, but the chat isn't marked as protected anymore
+        let contact = alice.add_or_lookup_contact(&bob).await;
+        assert_eq!(contact.is_verified(&alice).await.unwrap(), true);
+        assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
    } else {
        tcm.section("Bob sets up another Delta Chat device");
        let bob2 = TestContext::new().await;
-        enable_verified_oneonone_chats(&[&bob2]).await;
        bob2.set_name("bob2");
        bob2.configure_addr("bob@example.net").await;

+        SystemTime::shift(std::time::Duration::from_secs(3600));
        tcm.send_recv(&bob2, &alice, "Using another device now")
            .await;
+        let contact = alice.add_or_lookup_contact(&bob).await;
+        assert_eq!(contact.is_verified(&alice).await.unwrap(), false);
+        assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
    }

-    // Bob's contact is still verified, but the chat isn't marked as protected anymore
-    assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
-
    tcm.section("Bob sends another message from DC");
+    SystemTime::shift(std::time::Duration::from_secs(3600));
    tcm.send_recv(&bob, &alice, "Using DC again").await;

-    let contact = alice.add_or_lookup_contact(&bob).await;
-    assert_eq!(contact.is_verified(&alice.ctx).await.unwrap(), true);
-
    // Bob's chat is marked as verified again
    assert_verified(&alice, &bob, ProtectionStatus::Protected).await;
 }
@@ -816,8 +818,10 @@ async fn test_create_protected_grp_multidev() -> Result<()> {
 // ============== Helper Functions ==============

 async fn assert_verified(this: &TestContext, other: &TestContext, protected: ProtectionStatus) {
-    let contact = this.add_or_lookup_contact(other).await;
-    assert_eq!(contact.is_verified(this).await.unwrap(), true);
+    if protected != ProtectionStatus::ProtectionBroken {
+        let contact = this.add_or_lookup_contact(other).await;
+        assert_eq!(contact.is_verified(this).await.unwrap(), true);
+    }

    let chat = this.get_chat(other).await;
    let (expect_protected, expect_broken) = match protected {
Author	SHA1	Message	Date
iequidoo	15df57900b	feat: Try SQLite FTS5 (#5052 )	2024-03-04 15:28:51 -03:00
dependabot[bot]	12587684ca	Merge pull request #5305 from deltachat/dependabot/cargo/tempfile-3.10.1	2024-03-03 20:44:06 +00:00
dependabot[bot]	288eccf722	chore(cargo): bump image from 0.24.8 to 0.24.9 Bumps [image](https://github.com/image-rs/image) from 0.24.8 to 0.24.9. - [Changelog](https://github.com/image-rs/image/blob/master/CHANGES.md) - [Commits](https://github.com/image-rs/image/compare/v0.24.8...v0.24.9) --- updated-dependencies: - dependency-name: image dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-03 14:46:53 +00:00
dependabot[bot]	0c1d578207	chore(cargo): bump tempfile from 3.10.0 to 3.10.1 Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.10.0 to 3.10.1. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.10.0...v3.10.1) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-03 08:05:01 +00:00
dependabot[bot]	99ee769580	chore(cargo): bump textwrap from 0.16.0 to 0.16.1 Bumps [textwrap](https://github.com/mgeisler/textwrap) from 0.16.0 to 0.16.1. - [Release notes](https://github.com/mgeisler/textwrap/releases) - [Changelog](https://github.com/mgeisler/textwrap/blob/master/CHANGELOG.md) - [Commits](https://github.com/mgeisler/textwrap/compare/0.16.0...0.16.1) --- updated-dependencies: - dependency-name: textwrap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-03 08:01:51 +00:00
dependabot[bot]	345759d653	chore(cargo): bump syn from 2.0.48 to 2.0.52 Bumps [syn](https://github.com/dtolnay/syn) from 2.0.48 to 2.0.52. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.48...2.0.52) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-03 04:15:51 +00:00
link2xt	db0143f01a	build: remove deprecated `unmaintained` field from deny.toml	2024-03-03 03:49:27 +00:00
link2xt	4da0c19766	test: fixup tests/test_3_offline.py::TestOfflineAccountBasic::test_wrong_db	2024-03-03 03:28:47 +00:00
link2xt	08247a5d37	refactor: build contexts using ContextBuilder	2024-03-02 17:19:50 +00:00
link2xt	ceadd8928e	api: add ContextBuilder.build() to build Context without opening	2024-03-02 17:19:50 +00:00
dependabot[bot]	c3d96814ca	chore(cargo): bump walkdir from 2.4.0 to 2.5.0 Bumps [walkdir](https://github.com/BurntSushi/walkdir) from 2.4.0 to 2.5.0. - [Commits](https://github.com/BurntSushi/walkdir/compare/2.4.0...2.5.0) --- updated-dependencies: - dependency-name: walkdir dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-02 17:08:47 +00:00
dependabot[bot]	c2953623b9	chore(cargo): bump serde from 1.0.196 to 1.0.197 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.196 to 1.0.197. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.196...v1.0.197) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-02 17:07:36 +00:00
dependabot[bot]	1907d1859e	chore(cargo): bump anyhow from 1.0.79 to 1.0.80 Bumps [anyhow](https://github.com/dtolnay/anyhow) from 1.0.79 to 1.0.80. - [Release notes](https://github.com/dtolnay/anyhow/releases) - [Commits](https://github.com/dtolnay/anyhow/compare/1.0.79...1.0.80) --- updated-dependencies: - dependency-name: anyhow dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-02 16:00:57 +00:00
dependabot[bot]	a1970e998f	chore(cargo): bump log from 0.4.20 to 0.4.21 Bumps [log](https://github.com/rust-lang/log) from 0.4.20 to 0.4.21. - [Release notes](https://github.com/rust-lang/log/releases) - [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/log/compare/0.4.20...0.4.21) --- updated-dependencies: - dependency-name: log dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-02 16:00:33 +00:00
dependabot[bot]	1e9baefca0	chore(cargo): bump serde_json from 1.0.113 to 1.0.114 Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.113 to 1.0.114. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.113...v1.0.114) --- updated-dependencies: - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-02 15:59:55 +00:00
iequidoo	e16322d99d	test: get_protected_chat(): Use FFIEventTracker instead of dc_wait_next_msgs() (#5207 ) The way it was implemented it threw out all remaining messages after finding the next incoming message. Better use FFIEventTracker functions, they are used in all the tests anyway.	2024-03-02 12:12:26 -03:00
dependabot[bot]	ecfe3898c6	Merge pull request #5311 from deltachat/dependabot/cargo/rusqlite-0.31.0	2024-03-02 03:12:56 +00:00
link2xt	5499ca52bf	refactor: get rid of ImapActionResult	2024-03-02 01:31:29 +00:00
link2xt	4e8979f7c8	refactor: merge ImapConfig into Imap	2024-03-01 21:12:21 +00:00
dependabot[bot]	417db31098	chore(cargo): bump rusqlite from 0.30.0 to 0.31.0 Bumps [rusqlite](https://github.com/rusqlite/rusqlite) from 0.30.0 to 0.31.0. - [Release notes](https://github.com/rusqlite/rusqlite/releases) - [Changelog](https://github.com/rusqlite/rusqlite/blob/master/Changelog.md) - [Commits](https://github.com/rusqlite/rusqlite/compare/v0.30.0...v0.31.0) --- updated-dependencies: - dependency-name: rusqlite dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2024-03-01 21:11:09 +00:00
link2xt	cd9f6c3d5b	ci: build c.delta.chat docs with nix	2024-03-01 20:28:20 +00:00
link2xt	07870a6d69	refactor(imap): remove Session from Imap structure Connection establishment now happens only in one place in each IMAP loop. Now all connection establishment happens in one place and is limited by the ratelimit. Backoff was removed from fake_idle as it does not establish connections anymore. If connection fails, fake_idle will return an error. We then drop the connection and get back to the beginning of IMAP loop. Backoff may be still nice to have to delay retries in case of constant connection failures so we don't immediately hit ratelimit if the network is unusable and returns immediate error on each connection attempt (e.g. ICMP network unreachable error), but adding backoff for connection failures is out of scope for this change.	2024-03-01 18:36:03 +00:00
link2xt	b08a4d6fcf	ci: upload cffi docs without GH actions	2024-03-01 02:30:55 +00:00
link2xt	b3a82b416f	ci: upload python docs without GH actions	2024-03-01 02:21:35 +00:00
link2xt	4e5d7fb821	ci: build Python docs with Nix	2024-03-01 02:00:53 +00:00
link2xt	1d73f97ef3	nix: add deltachat-time to sources	2024-03-01 01:58:11 +00:00
link2xt	f5601e7683	Merge pull request #5296 from deltachat/link2xt/imap-session refactor: move more methods from Imap into Session	2024-02-29 02:29:52 +00:00
link2xt	0000c09ad3	fix(imap): allow maybe_network to interrupt connection ratelimit ratelimit can be exhausted quickly if the network is not available, i.e. if every connection attempt returns "network unreachable" error. When the network becomes available, we want to retry connecting as soon as maybe_network is called without waiting for ratelimiter.	2024-02-29 02:29:18 +00:00
link2xt	a83884d7e9	refactor(imap): require watch_folder for fake_idle()	2024-02-28 23:18:30 +00:00
link2xt	9e00e8627f	refactor(imap): pass Session to add_all_recipients_as_contacts()	2024-02-28 22:51:07 +00:00
link2xt	85c9622675	refactor(imap): move fetch_many_msgs() into Session	2024-02-28 22:48:23 +00:00
link2xt	30432d8fa5	refactor(imap): move fetch_metadata() to Session	2024-02-28 22:42:54 +00:00
link2xt	8b9f19be70	refactor(imap): move get_all_recipients() to Session	2024-02-28 22:40:37 +00:00
link2xt	39c317e211	refator(imap): move sync_seen_flags() to Session	2024-02-28 22:39:06 +00:00
link2xt	36ab7bdf47	refactor(imap): do not get Session twice in fetch_new_messages()	2024-02-28 22:32:38 +00:00
link2xt	f8f0ca08da	refactor(imap): pass Imap Session to update_recent_quota()	2024-02-28 22:30:19 +00:00
link2xt	2a0a05d03c	refactor(imap): move resync_folders() to Session	2024-02-28 22:26:25 +00:00
link2xt	7bc2f0cb6b	refactor(imap): move select_with_uidvalidity() to Session	2024-02-28 22:15:33 +00:00
link2xt	4355bd77a9	refactor(imap): move resync_folder_uids() to Session	2024-02-28 22:02:36 +00:00
link2xt	f0091696c2	refactor(imap): move prefetch() to Session	2024-02-28 21:56:28 +00:00
link2xt	d2e86c5852	refactor(imap): move prefetch_existing_msgs to Session	2024-02-28 21:49:44 +00:00
link2xt	d4a505b52e	refactor(imap): move list_folders() to Session	2024-02-28 21:43:25 +00:00
iequidoo	08a30031eb	fix: Don't send sync messages on self-{status,avatar} update from self-sent messages (#5289 ) Sync messages should only be sent in response to user actions.	2024-02-26 12:28:03 -03:00
link2xt	44686d6caa	ci: update to Rust 1.76 and fix clippy warnings	2024-02-25 10:43:01 +00:00
iequidoo	9862d40f89	feat: Send Chat-Group-Avatar as inline base64 (#5253 ) Before group avatar was sent as an attachment. Let's do the same as with user avatar and send group avatar as base64. Receiver code uses the same functions for user and chat avatars, so base64 avatars are supported for most receivers already.	2024-02-24 20:24:04 -03:00
link2xt	256c8c13f1	build: unpin OpenSSL `deltachat-rpc-server` releases are built with Nix and LLVM/clang toolchains now that fully support atomics. Zig toolchain that required disabling atomics and resulted in problems with OpenSSL 3.2 releases is not used anymore.	2024-02-24 13:32:42 +00:00
link2xt	0b3a56c3c4	api: make store_self_keypair private It is not useful as public API because input argument types are not public. Use `imex` instead.	2024-02-23 19:29:45 +00:00
iequidoo	89024bbf37	test: Fix test_verified_oneonone_chat_broken_by_device_change() (#5280 ) It was broken completely and before "fix: apply Autocrypt headers if timestamp is unchanged" that didn't show up because the message from the second Bob's device never had "Date" greater than one from the message sent before from the first device.	2024-02-23 15:23:02 -03:00
link2xt	cf16671d8d	fix(imap): set connectivity to "connecting" only after ratelimit	2024-02-22 13:08:44 +00:00
link2xt	671feb68a4	fix: do not fake idle after trigger_reconnect() In this case connection failure may be a connection timeout (currently 1 minute), so it does not make sense to fake idle for another minute immediately after. However, failure may be immediate if the port is closed and the server refuses connection every time. To prevent busy loop in this case we apply ratelimit to connection attempts rather than login attempts. This partially reverts `ccec26ffa7`	2024-02-22 13:08:44 +00:00
link2xt	ccd5158109	ci: upgrade setup-python GitHub Action	2024-02-22 06:18:34 +00:00
link2xt	0a18e32d62	chore(cargo): update rpgp to 0.11 <https://github.com/rpgp/rpgp/releases/tag/v0.11.0>	2024-02-22 05:08:00 +00:00
iequidoo	e9fadc0785	feat: Recognise Trash folder by name (#5275 ) If a folder is named "Trash" or like this, it should be recognised as such even if it does not have a \Trash attribute.	2024-02-20 18:29:04 -03:00
link2xt	cfa13f0669	build: tag armv6 wheels with tags accepted by PyPI See <https://github.com/pypi/warehouse/blob/main/warehouse/forklift/legacy.py> for the tag checking code. linux_armv6l and linux_armv7l are accepted, but {many,musl}linux_*_armv6l are not.	2024-02-20 18:10:52 +00:00