Do not use IDLE in test_mdn_asymmetric

Fixes test flakyness.

.github/workflows/ci.yml

@@ -77,6 +77,9 @@ jobs:
           - os: windows-latest
             rust: 1.68.0
             python: false # Python bindings compilation on Windows is not supported.
+          - os: macos-latest
+            rust: 1.68.0
+            python: 3.9
 
           # Minimum Supported Rust Version = 1.64.0
           #

.github/workflows/deltachat-rpc-server.yml

@@ -84,35 +84,15 @@ jobs:
       contents: write
     runs-on: "ubuntu-latest"
     steps:
-      - name: Download deltachat-rpc-server-x86_64
+      - name: Download built binaries
         uses: "actions/download-artifact@v3"
-        with:
-          name: "deltachat-rpc-server-x86_64"
-          path: "dist/deltachat-rpc-server-x86_64"
 
-      - name: Download deltachat-rpc-server-aarch64
-        uses: "actions/download-artifact@v3"
-        with:
-          name: "deltachat-rpc-server-aarch64"
-          path: "dist/deltachat-rpc-server-aarch64"
-
-      - name: Download deltachat-rpc-server-armv7
-        uses: "actions/download-artifact@v3"
-        with:
-          name: "deltachat-rpc-server-armv7"
-          path: "dist/deltachat-rpc-server-armv7"
-
-      - name: Download deltachat-rpc-server-win32.exe
-        uses: "actions/download-artifact@v3"
-        with:
-          name: "deltachat-rpc-server-win32.exe"
-          path: "dist/deltachat-rpc-server-win32.exe"
-
-      - name: Download deltachat-rpc-server-win64.exe
-        uses: "actions/download-artifact@v3"
-        with:
-          name: "deltachat-rpc-server-win64.exe"
-          path: "dist/deltachat-rpc-server-win64.exe"
+      - name: Compose dist/ directory
+        run: |
+          mkdir dist
+          for x in x86_64 aarch64 armv7 win32.exe win64.exe; do
+            mv "deltachat-rpc-server-$x"/* "dist/deltachat-rpc-server-$x"
+          done
 
       - name: List downloaded artifacts
         run: ls -l dist/
@@ -123,4 +103,4 @@ jobs:
         run: |
           gh release upload ${{ github.ref_name }} \
             --repo ${{ github.repository }} \
-            dist/deltachat-rpc-server-*
+            dist/*

CHANGELOG.md

@@ -3,16 +3,23 @@
 ## [Unreleased]
 
 ### Changes
-- "full message view" not needed because of footers that go to contact status #4151
-- Pick up system's light/dark mode in generated message HTML #4150
-- Support non-persistent configuration with DELTACHAT_* env
-- Print deltachat-repl errors with causes. #4166
+
+### Fixes
+
+
+## [1.112.0] - 2023-03-23
+
+### Changes
 - Increase MSRV to 1.64. #4167
 - Core takes care of stopping and re-starting IO itself where needed,
-  e.g. during backup creation.  It is no longer needed to call
-  dc_stop_io().  dc_start_io() can now be called at any time without
-  harm. #4138
-- More accurate maybe_add_bcc_self device message text #4175
+  e.g. during backup creation.
+  It is no longer needed to call `dc_stop_io()`.
+  `dc_start_io()` can now be called at any time without harm. #4138
+- Pick up system's light/dark mode in generated message HTML. #4150
+- More accurate `maybe_add_bcc_self` device message text. #4175
+- "Full message view" not needed because of footers that go to contact status. #4151
+- Support non-persistent configuration with `DELTACHAT_*` env. #4154
+- Print deltachat-repl errors with causes. #4166
 
 ### Fixes
 - Fix segmentation fault if `dc_context_unref()` is called during
@@ -23,6 +30,11 @@
 - Do not emit "Failed to run incremental vacuum" warnings on success. #4160
 - Ability to send backup over network and QR code to setup second device #4007
 - Disable buffering during STARTTLS setup. #4190
+- Add `DC_EVENT_IMAP_INBOX_IDLE` event to wait until the account
+  is ready for testing.
+  It is used to fix race condition between fetching
+  existing messages and starting the test. #4208
+
 
 ## [1.111.0] - 2023-03-05
 
@@ -2314,5 +2326,6 @@ For a full list of changes, please see our closed Pull Requests:
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 
-[unreleased]: https://github.com/deltachat/deltachat-core-rust/compare/v1.111.0...HEAD
+[unreleased]: https://github.com/deltachat/deltachat-core-rust/compare/v1.112.0...HEAD
 [1.111.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.110.0...v1.111.0
+[1.112.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.111.0...v1.112.0

Cargo.lock

@@ -270,25 +270,16 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "async_io_utilities"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0"
-dependencies = [
- "tokio",
-]
-
 [[package]]
 name = "async_zip"
-version = "0.0.9"
+version = "0.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a36d43bdefc7215b2b3a97edd03b1553b7969ad76551025eedd3b913c645f6e"
+checksum = "c50d29ab7e2f9e808cca1a69ea56a36f4ff216f54a41a23aae1fd4afc05cc020"
 dependencies = [
  "async-compression",
- "async_io_utilities",
- "chrono",
  "crc32fast",
+ "log",
+ "pin-project",
  "thiserror",
  "tokio",
 ]
@@ -1145,7 +1136,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.111.0"
+version = "1.112.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1218,7 +1209,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.111.0"
+version = "1.112.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -1241,7 +1232,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "1.111.0"
+version = "1.112.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1256,7 +1247,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.111.0"
+version = "1.112.0"
 dependencies = [
  "anyhow",
  "deltachat-jsonrpc",
@@ -1279,7 +1270,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.111.0"
+version = "1.112.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1288,7 +1279,7 @@ dependencies = [
  "libc",
  "num-traits",
  "once_cell",
- "rand 0.7.3",
+ "rand 0.8.5",
  "serde_json",
  "thiserror",
  "tokio",
@@ -2427,7 +2418,7 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
 [[package]]
 name = "iroh"
 version = "0.3.0"
-source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#aacd1d84e11b218e55e9622c4500973224904fe9"
+source = "git+https://github.com/n0-computer/iroh?branch=main#9ac4cf6e770879c8b2ec0dc6666fe531469e68e3"
 dependencies = [
  "abao",
  "anyhow",
@@ -2465,6 +2456,7 @@ dependencies = [
  "tracing",
  "tracing-futures",
  "tracing-subscriber",
+ "walkdir",
  "webpki",
  "x509-parser",
  "zeroize",
@@ -3026,9 +3018,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.45"
+version = "0.10.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
+checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -3067,9 +3059,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.80"
+version = "0.9.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
+checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b"
 dependencies = [
  "autocfg",
  "cc",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.111.0"
+version = "1.112.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.64"
@@ -39,7 +39,7 @@ async-channel = "1.8.0"
 async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
-async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
+async_zip = { version = "0.0.11", default-features = false, features = ["deflate", "fs"] }
 backtrace = "0.3"
 base64 = "0.21"
 bitflags = "1.3"
@@ -54,7 +54,7 @@ hex = "0.4.0"
 humansize = "2"
 image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 # iroh = { version = "0.3.0", default-features = false }
-iroh = { git = 'https://github.com/n0-computer/iroh', branch = "flub/ticket-multiple-addrs" }
+iroh = { git = 'https://github.com/n0-computer/iroh', branch = "main" }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.111.0"
+version = "1.112.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -23,7 +23,7 @@ serde_json = "1.0"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
-rand = "0.7"
+rand = "0.8"
 once_cell = "1.17.0"
 
 [features]

deltachat-ffi/deltachat.h

@@ -5705,6 +5705,14 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
+/**
+ * Emitted before going into IDLE on the Inbox folder.
+ *
+ * @param data1 0
+ * @param data2 0
+ */
+#define DC_EVENT_IMAP_INBOX_IDLE 106
+
 /**
  * Emitted when a new blob file was successfully written
  *
@@ -6990,6 +6998,11 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by name and address of the contact.
 #define DC_STR_PROTECTION_DISABLED_BY_OTHER 161
 
+/// "Scan to set up second device for %1$s"
+///
+/// `%1$s` will be replaced by name and address of the account.
+#define DC_STR_BACKUP_TRANSFER_QR 162
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -500,6 +500,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SmtpMessageSent(_) => 103,
         EventType::ImapMessageDeleted(_) => 104,
         EventType::ImapMessageMoved(_) => 105,
+        EventType::ImapInboxIdle => 106,
         EventType::NewBlobFile(_) => 150,
         EventType::DeletedBlobFile(_) => 151,
         EventType::Warning(_) => 300,
@@ -544,6 +545,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
+        | EventType::ImapInboxIdle
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -594,6 +596,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
+        | EventType::ImapInboxIdle
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -653,6 +656,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         EventType::MsgsChanged { .. }
         | EventType::ReactionsChanged { .. }
         | EventType::IncomingMsg { .. }
+        | EventType::ImapInboxIdle
         | EventType::MsgsNoticed(_)
         | EventType::MsgDelivered { .. }
         | EventType::MsgFailed { .. }
@@ -4240,17 +4244,28 @@ pub unsafe extern "C" fn dc_receive_backup(
     }
     let ctx = &*context;
     let qr_text = to_string_lossy(qr);
-    let qr = match block_on(qr::check_qr(ctx, &qr_text)).log_err(ctx, "Invalid QR code") {
+    receive_backup(ctx.clone(), qr_text)
+}
+
+// Because this is a long-running operation make sure we own the Context.  This stops a FFI
+// user from deallocating it by calling unref on the object while we are using it.
+fn receive_backup(ctx: Context, qr_text: String) -> libc::c_int {
+    let qr = match block_on(qr::check_qr(&ctx, &qr_text))
+        .log_err(&ctx, "Invalid QR code")
+        .context("Invalid QR code")
+        .set_last_error(&ctx)
+    {
         Ok(qr) => qr,
         Err(_) => return 0,
     };
-    spawn(async move {
-        imex::get_backup(ctx, qr)
-            .await
-            .log_err(ctx, "Get backup failed")
-            .ok();
-    });
-    1
+    match block_on(imex::get_backup(&ctx, qr))
+        .log_err(&ctx, "Get backup failed")
+        .context("Get backup failed")
+        .set_last_error(&ctx)
+    {
+        Ok(_) => 1,
+        Err(_) => 0,
+    }
 }
 
 trait ResultExt<T, E> {

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.111.0"
+version = "1.112.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/src/api/events.rs

@@ -47,6 +47,9 @@ pub enum JSONRPCEventType {
         msg: String,
     },
 
+    /// Emitted before going into IDLE on the Inbox folder.
+    ImapInboxIdle,
+
     /// Emitted when an new file in the $BLOBDIR was created
     NewBlobFile {
         file: String,
@@ -293,6 +296,7 @@ impl From<EventType> for JSONRPCEventType {
             EventType::SmtpMessageSent(msg) => SmtpMessageSent { msg },
             EventType::ImapMessageDeleted(msg) => ImapMessageDeleted { msg },
             EventType::ImapMessageMoved(msg) => ImapMessageMoved { msg },
+            EventType::ImapInboxIdle => ImapInboxIdle,
             EventType::NewBlobFile(file) => NewBlobFile { file },
             EventType::DeletedBlobFile(file) => DeletedBlobFile { file },
             EventType::Warning(msg) => Warning { msg },

deltachat-jsonrpc/typescript/package.json

@@ -55,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.111.0"
+  "version": "1.112.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.111.0"
+version = "1.112.0"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -31,6 +31,7 @@ class EventType(str, Enum):
     SMTP_MESSAGE_SENT = "SmtpMessageSent"
     IMAP_MESSAGE_DELETED = "ImapMessageDeleted"
     IMAP_MESSAGE_MOVED = "ImapMessageMoved"
+    IMAP_INBOX_IDLE = "ImapInboxIdle"
     NEW_BLOB_FILE = "NewBlobFile"
     DELETED_BLOB_FILE = "DeletedBlobFile"
     WARNING = "Warning"

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -55,6 +55,11 @@ class ACFactory:
     async def get_online_account(self) -> Account:
         account = await self.new_configured_account()
         await account.start_io()
+        while True:
+            event = await account.wait_for_event()
+            print(event)
+            if event.type == EventType.IMAP_INBOX_IDLE:
+                break
         return account
 
     async def get_online_accounts(self, num: int) -> List[Account]:

deltachat-rpc-client/tox.ini

@@ -6,7 +6,7 @@ envlist =
 
 [testenv]
 commands =
-    pytest --exitfirst {posargs}
+    pytest {posargs}
 setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
     RUST_MIN_STACK=8388608
@@ -15,6 +15,7 @@ passenv =
 deps =
     pytest
     pytest-asyncio
+    pytest-timeout
     aiohttp
     aiodns
 
@@ -27,3 +28,6 @@ deps =
 commands =
     black --quiet --check --diff src/ examples/ tests/
     ruff src/ examples/ tests/
+
+[pytest]
+timeout = 60

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.111.0"
+version = "1.112.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -9,9 +9,6 @@ license = "MPL-2.0"
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
 
-[[bin]]
-name = "deltachat-rpc-server"
-
 [dependencies]
 deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
 

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.111.0"
+  "version": "1.112.0"
 }

python/src/deltachat/events.py

@@ -190,7 +190,7 @@ class FFIEventTracker:
         - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
         - therefore no DC_EVENT_INCOMING_MSG is sent
         """
-        self.get_info_contains("INBOX: Idle entering")
+        self.get_matching("DC_EVENT_IMAP_INBOX_IDLE")
 
     def wait_next_incoming_message(self):
         """wait for and return next incoming message."""

python/tests/auditwheels.py

@@ -1,19 +0,0 @@
-import os
-import subprocess
-import sys
-
-if __name__ == "__main__":
-    assert len(sys.argv) == 2
-    workspacedir = sys.argv[1]
-    for relpath in os.listdir(workspacedir):
-        if relpath.startswith("deltachat"):
-            p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(
-                [
-                    "auditwheel",
-                    "repair",
-                    p,
-                    "-w",
-                    workspacedir,
-                ],
-            )

python/tests/test_1_online.py

@@ -134,22 +134,27 @@ def test_one_account_send_bcc_setting(acfactory, lp):
     ac1.set_config("bcc_self", "1")
 
     lp.sec("send out message with bcc to ourselves")
-    with ac1.direct_imap.idle() as idle1:
-        msg_out = chat.send_text("message2")
+    msg_out = chat.send_text("message2")
 
-        # wait for send out (BCC)
-        ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
-        assert ac1.get_config("bcc_self") == "1"
+    # wait for send out (BCC)
+    ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+    assert ac1.get_config("bcc_self") == "1"
 
-        # now make sure we are sending message to ourselves too
-        assert self_addr in ev.data2
-        assert other_addr in ev.data2
-        assert idle1.wait_for_seen()
-
-    # Second client receives only second message, but not the first
+    # Second client receives only second message, but not the first.
     ev_msg = ac1_clone._evtracker.wait_next_messages_changed()
     assert ev_msg.text == msg_out.text
 
+    # now make sure we are sending message to ourselves too
+    assert self_addr in ev.data2
+    assert other_addr in ev.data2
+
+    # BCC-self messages are marked as seen by the sender device.
+    ac1._evtracker.get_info_contains("Marked messages [0-9]+ in folder INBOX as seen.")
+
+    # Check that the message is marked as seen on IMAP.
+    ac1.direct_imap.select_folder("Inbox")
+    assert len(list(ac1.direct_imap.conn.fetch(AND(seen=True)))) == 1
+
 
 def test_send_file_twice_unicode_filename_mangling(tmpdir, acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
@@ -678,23 +683,24 @@ def test_mdn_asymmetric(acfactory, lp):
 
     assert len(msg.chat.get_messages()) == 1
 
-    ac1.direct_imap.select_config_folder("mvbox")
-    with ac1.direct_imap.idle() as idle1:
-        lp.sec("ac2: mark incoming message as seen")
-        ac2.mark_seen_messages([msg])
+    lp.sec("ac2: mark incoming message as seen")
+    ac2.mark_seen_messages([msg])
 
-        lp.sec("ac1: waiting for incoming activity")
-        # MDN should be moved even though MDNs are already disabled
-        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+    lp.sec("ac1: waiting for incoming activity")
+    # MDN should be moved even though MDNs are already disabled
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
-        assert len(chat.get_messages()) == 1
+    assert len(chat.get_messages()) == 1
 
-        # Wait for the message to be marked as seen on IMAP.
-        assert idle1.wait_for_seen()
+    # Wait for the message to be marked as seen on IMAP.
+    ac1._evtracker.get_info_contains("Marked messages 1 in folder DeltaChat as seen.")
 
     # MDN is received even though MDNs are already disabled
     assert msg_out.is_out_mdn_received()
 
+    ac1.direct_imap.select_config_folder("mvbox")
+    assert len(list(ac1.direct_imap.conn.fetch(AND(seen=True)))) == 1
+
 
 def test_send_and_receive_will_encrypt_decrypt(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --exitfirst --extra-info -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     pip wheel . -w {toxworkdir}/wheelhouse --no-deps
 setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
@@ -21,6 +21,7 @@ passenv =
     RUSTC_WRAPPER
 deps = 
     pytest
+    pytest-rerunfailures
     pytest-timeout
     pytest-xdist
     pdbpp
@@ -33,18 +34,6 @@ passenv =
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 
-[testenv:auditwheels]
-skipsdist = True
-deps = auditwheel
-passenv =
-    DCC_RS_DEV
-    DCC_RS_TARGET
-    AUDITWHEEL_ARCH
-    AUDITWHEEL_PLAT
-    AUDITWHEEL_POLICY
-commands = 
-    python tests/auditwheels.py {toxworkdir}/wheelhouse
-
 [testenv:lint]
 skipsdist = True
 skip_install = True

scripts/run_all.sh

@@ -31,7 +31,9 @@ unset DCC_NEW_TMP_EMAIL
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,pypy37,pypy38,pypy39,auditwheels --skip-missing-interpreters true
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,pypy37,pypy38,pypy39 --skip-missing-interpreters true
+
+auditwheel repair "$TOXWORKDIR/wheelhouse/deltachat*" -w "$TOXWORKDIR/wheelhouse"
 
 
 echo -----------------------

scripts/set_core_version.py

@@ -93,7 +93,7 @@ def main():
     if "alpha" not in newversion:
         for line in open("CHANGELOG.md"):
             ## 1.25.0
-            if line.startswith("## ") and line[2:].strip().startswith(newversion):
+            if line.startswith("## [") and line[4:].strip().startswith(newversion):
                 break
         else:
             raise SystemExit(

src/events.rs

@@ -133,6 +133,9 @@ pub enum EventType {
     /// Emitted when an IMAP message has been moved
     ImapMessageMoved(String),
 
+    /// Emitted before going into IDLE on the Inbox folder.
+    ImapInboxIdle,
+
     /// Emitted when an new file in the $BLOBDIR was created
     NewBlobFile(String),
 

src/imex.rs

@@ -91,15 +91,13 @@ pub async fn imex(
     let cancel = context.alloc_ongoing().await?;
 
     let res = {
-        let mut guard = context.scheduler.pause(context.clone()).await;
-        let res = imex_inner(context, what, path, passphrase)
+        let _guard = context.scheduler.pause(context.clone()).await;
+        imex_inner(context, what, path, passphrase)
             .race(async {
                 cancel.recv().await.ok();
                 Err(format_err!("canceled"))
             })
-            .await;
-        guard.resume().await;
-        res
+            .await
     };
     context.free_ongoing().await;
 

src/imex/transfer.rs

@@ -91,7 +91,7 @@ impl BackupProvider {
 
         // Acquire global "ongoing" mutex.
         let cancel_token = context.alloc_ongoing().await?;
-        let mut paused_guard = context.scheduler.pause(context.clone()).await;
+        let paused_guard = context.scheduler.pause(context.clone()).await;
         let context_dir = context
             .get_blobdir()
             .parent()
@@ -119,7 +119,6 @@ impl BackupProvider {
             Ok((provider, ticket)) => (provider, ticket),
             Err(err) => {
                 context.free_ongoing().await;
-                paused_guard.resume().await;
                 return Err(err);
             }
         };
@@ -128,7 +127,9 @@ impl BackupProvider {
             tokio::spawn(async move {
                 let res = Self::watch_provider(&context, provider, cancel_token).await;
                 context.free_ongoing().await;
-                paused_guard.resume().await;
+
+                // Explicit drop to move the guards into this future
+                drop(paused_guard);
                 drop(dbfile);
                 res
             })
@@ -369,7 +370,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> {
         !context.is_configured().await?,
         "Cannot import backups to accounts in use."
     );
-    let mut guard = context.scheduler.pause(context.clone()).await;
+    let _guard = context.scheduler.pause(context.clone()).await;
 
     // Acquire global "ongoing" mutex.
     let cancel_token = context.alloc_ongoing().await?;
@@ -381,7 +382,6 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> {
         }
         _ = cancel_token.recv() => Err(format_err!("cancelled")),
     };
-    guard.resume().await;
     res
 }
 

src/mimeparser.rs

@@ -1976,10 +1976,10 @@ pub(crate) fn get_list_post(headers: &[MailHeader]) -> Option<String> {
         .map(|s| s.addr)
 }
 
-fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
-where
-    F: Fn(String) -> bool,
-{
+fn get_all_addresses_from_header(
+    headers: &[MailHeader],
+    pred: fn(String) -> bool,
+) -> Vec<SingleInfo> {
     let mut result: Vec<SingleInfo> = Default::default();
 
     headers

src/scheduler.rs

@@ -5,7 +5,7 @@ use anyhow::{bail, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
 use futures::future::try_join_all;
 use futures_lite::FutureExt;
-use tokio::sync::{RwLock, RwLockWriteGuard};
+use tokio::sync::{oneshot, RwLock, RwLockWriteGuard};
 use tokio::task;
 
 use self::connectivity::ConnectivityStore;
@@ -13,6 +13,7 @@ use crate::config::Config;
 use crate::contact::{ContactId, RecentlySeenLoop};
 use crate::context::Context;
 use crate::ephemeral::{self, delete_expired_imap_messages};
+use crate::events::EventType;
 use crate::imap::{FolderMeaning, Imap};
 use crate::job;
 use crate::location;
@@ -89,20 +90,28 @@ impl SchedulerState {
 
     /// Pauses the IO scheduler.
     ///
-    /// If it is currently running the scheduler will be stopped.  When
-    /// [`IoPausedGuard::resume`] is called the scheduler is started again.
+    /// If it is currently running the scheduler will be stopped.  When the
+    /// [`IoPausedGuard`] is dropped the scheduler is started again.
     ///
     /// If in the meantime [`SchedulerState::start`] or [`SchedulerState::stop`] is called
     /// resume will do the right thing and restore the scheduler to the state requested by
     /// the last call.
     pub(crate) async fn pause<'a>(&'_ self, context: Context) -> IoPausedGuard {
-        let mut inner = self.inner.write().await;
-        inner.paused = true;
-        Self::do_stop(inner, &context).await;
-        IoPausedGuard {
-            context,
-            done: false,
+        {
+            let mut inner = self.inner.write().await;
+            inner.paused = true;
+            Self::do_stop(inner, &context).await;
         }
+        let (tx, rx) = oneshot::channel();
+        tokio::spawn(async move {
+            rx.await.ok();
+            let mut inner = context.scheduler.inner.write().await;
+            inner.paused = false;
+            if inner.started && inner.scheduler.is_none() {
+                SchedulerState::do_start(inner, context.clone()).await;
+            }
+        });
+        IoPausedGuard { sender: Some(tx) }
     }
 
     /// Restarts the scheduler, only if it is running.
@@ -194,31 +203,21 @@ struct InnerSchedulerState {
     paused: bool,
 }
 
+/// Guard to make sure the IO Scheduler is resumed.
+///
+/// Returned by [`SchedulerState::pause`].  To resume the IO scheduler simply drop this
+/// guard.
 #[derive(Debug)]
 pub(crate) struct IoPausedGuard {
-    context: Context,
-    done: bool,
-}
-
-impl IoPausedGuard {
-    pub(crate) async fn resume(&mut self) {
-        self.done = true;
-        let mut inner = self.context.scheduler.inner.write().await;
-        inner.paused = false;
-        if inner.started && inner.scheduler.is_none() {
-            SchedulerState::do_start(inner, self.context.clone()).await;
-        }
-    }
+    sender: Option<oneshot::Sender<()>>,
 }
 
 impl Drop for IoPausedGuard {
     fn drop(&mut self) {
-        if self.done {
-            return;
+        if let Some(sender) = self.sender.take() {
+            // Can only fail if receiver is dropped, but then we're already resumed.
+            sender.send(()).ok();
         }
-
-        // Async .resume() should be called manually due to lack of async drop.
-        error!(self.context, "Pause guard dropped without resuming.");
     }
 }
 
@@ -479,6 +478,7 @@ async fn fetch_idle(
 
     connection.connectivity.set_connected(ctx).await;
 
+    ctx.emit_event(EventType::ImapInboxIdle);
     if let Some(session) = connection.session.take() {
         if !session.can_idle() {
             info!(

src/webxdc.rs

@@ -165,6 +165,19 @@ pub(crate) struct StatusUpdateItemAndSerial {
     max_serial: StatusUpdateSerial,
 }
 
+/// Returns an entry index and a reference.
+fn find_zip_entry<'a>(
+    file: &'a async_zip::ZipFile,
+    name: &str,
+) -> Option<(usize, &'a async_zip::StoredZipEntry)> {
+    for (i, ent) in file.entries().iter().enumerate() {
+        if ent.entry().filename() == name {
+            return Some((i, ent));
+        }
+    }
+    None
+}
+
 impl Context {
     /// check if a file is an acceptable webxdc for sending or receiving.
     pub(crate) async fn is_webxdc_file(&self, filename: &str, file: &[u8]) -> Result<bool> {
@@ -180,7 +193,7 @@ impl Context {
             return Ok(false);
         }
 
-        let archive = match async_zip::read::mem::ZipFileReader::new(file).await {
+        let archive = match async_zip::read::mem::ZipFileReader::new(file.to_vec()).await {
             Ok(archive) => archive,
             Err(_) => {
                 info!(self, "{} cannot be opened as zip-file", &filename);
@@ -188,7 +201,7 @@ impl Context {
             }
         };
 
-        if archive.entry("index.html").is_none() {
+        if find_zip_entry(archive.file(), "index.html").is_none() {
             info!(self, "{} misses index.html", &filename);
             return Ok(false);
         }
@@ -215,7 +228,7 @@ impl Context {
 
         let valid = match async_zip::read::fs::ZipFileReader::new(path).await {
             Ok(archive) => {
-                if archive.entry("index.html").is_none() {
+                if find_zip_entry(archive.file(), "index.html").is_none() {
                     info!(self, "{} misses index.html", filename);
                     false
                 } else {
@@ -649,10 +662,9 @@ fn parse_webxdc_manifest(bytes: &[u8]) -> Result<WebxdcManifest> {
 }
 
 async fn get_blob(archive: &mut async_zip::read::fs::ZipFileReader, name: &str) -> Result<Vec<u8>> {
-    let (i, _) = archive
-        .entry(name)
+    let (i, _) = find_zip_entry(archive.file(), name)
         .ok_or_else(|| anyhow!("no entry found for {}", name))?;
-    let mut reader = archive.entry_reader(i).await?;
+    let mut reader = archive.entry(i).await?;
     let mut buf = Vec::new();
     reader.read_to_end(&mut buf).await?;
     Ok(buf)
@@ -754,9 +766,9 @@ impl Message {
             } else {
                 self.get_filename().unwrap_or_default()
             },
-            icon: if archive.entry("icon.png").is_some() {
+            icon: if find_zip_entry(archive.file(), "icon.png").is_some() {
                 "icon.png".to_string()
-            } else if archive.entry("icon.jpg").is_some() {
+            } else if find_zip_entry(archive.file(), "icon.jpg").is_some() {
                 "icon.jpg".to_string()
             } else {
                 WEBXDC_DEFAULT_ICON.to_string()