Prepare release 1.91.0

`coredeps` dockerfile is not outdated.

Add `run_all.sh` description.

CHANGELOG.md

@@ -2,11 +2,52 @@
 
 ## Unreleased
 
+### Added
+
 ### Changes
 
 ### Fixes
 
 
+## 1.91.0
+
+### Added
+- python bindings: extra method to get an account running
+
+### Changes
+- refactorings #3437
+
+### Fixes
+- mark "group image changed" as system message on receiver side #3517
+
+
+## 1.90.0
+
+### Changes
+- handle drafts from mailto links in scanned QR #3492
+- do not overflow ratelimiter leaky bucket #3496
+- (AEAP) Add device message after you changed your address #3505
+- (AEAP) Revert #3491, instead only replace contacts in verified groups #3510
+- improve python bindings and tests #3502 #3503
+
+### Fixes
+- don't squash text parts of NDN into attachments #3497
+- do not treat non-failed DSNs as NDNs #3506
+
+
+## 1.89.0
+
+### Changes
+
+- (AEAP) When one of your contacts changed their address, they are
+  only replaced in the chat where you got a message from them
+  for now #3491
+
+### Fixes
+- replace musl libc name resolution errors with a better message #3485
+- handle updates for not yet downloaded webxdc instances #3487
+
+
 ## 1.88.0
 
 ### Changes

Cargo.lock

@@ -752,7 +752,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.88.0"
+version = "1.91.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -832,7 +832,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.88.0"
+version = "1.91.0"
 dependencies = [
  "anyhow",
  "deltachat",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.88.0"
+version = "1.91.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.88.0"
+version = "1.91.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -2292,7 +2292,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  * - DC_QR_FPR_MISMATCH with dc_lot_t::id=Contact ID:
  *   scanned fingerprint does not match last seen fingerprint.
  *
- * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::test1=Formatted fingerprint
+ * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::text1=Formatted fingerprint
  *   the scanned QR code contains a fingerprint but no e-mail address;
  *   suggest the user to establish an encrypted connection first.
  *
@@ -2305,7 +2305,8 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr().
  *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
- *   e-mail address scanned,
+ *   e-mail address scanned, optionally, a draft message could be set in
+ *   dc_lot_t::text1 in which case dc_lot_t::text1_meaning will be DC_TEXT1_DRAFT;
  *   ask the user if they want to start chatting;
  *   if so, call dc_create_chat_by_contact_id().
  *
@@ -6364,6 +6365,24 @@ void dc_event_unref(dc_event_t* event);
 /// Used as status in the connectivity view.
 #define DC_STR_NOT_CONNECTED              121
 
+/// %1$s changed their address from %2$s to %3$s"
+///
+/// Used as an info message to chats with contacts that changed their address.
+#define DC_STR_AEAP_ADDR_CHANGED          122
+
+/// "You changed your email address from %1$s to %2$s.
+/// If you now send a message to a group, contacts there will automatically
+/// replace the old with your new address.\n\nIt's highly advised to set up 
+/// your old email provider to forward all emails to your new email address. 
+/// Otherwise you might miss messages of contacts who did not get your new 
+/// address yet." + the link to the AEAP blog post
+/// 
+/// As soon as there is a post about AEAP, the UIs should add it:
+/// set_stock_translation(123, getString(aeap_explanation) + "\n\n" + AEAP_BLOG_LINK)
+///
+/// Used in a device message that explains AEAP.
+#define DC_STR_AEAP_EXPLANATION_AND_LINK  123
+
 /**
  * @}
  */

deltachat-ffi/src/lot.rs

@@ -51,7 +51,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
+                Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
                 Qr::Text { text } => Some(text),
                 Qr::WithdrawVerifyContact { .. } => None,
@@ -79,7 +79,13 @@ impl Lot {
                 Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
                 Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
             },
-            Self::Qr(_qr) => Meaning::None,
+            Self::Qr(qr) => match qr {
+                Qr::Addr {
+                    draft: Some(_draft),
+                    ..
+                } => Meaning::Text1Draft,
+                _ => Meaning::None,
+            },
             Self::Error(_err) => Meaning::None,
         }
     }
@@ -118,7 +124,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
+                Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
                 Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),

node/README.md

@@ -234,7 +234,7 @@ We have the following scripts for building, testing and coverage:
 The following steps are needed to make a release:
 
 1. Wait until `pack-module` github action is completed
-2. Run `npm publish https://download.delta.chat/node/deltachat-node-v1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
+2. Run `npm publish https://download.delta.chat/node/deltachat-node-1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
 
 ## License
 

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.88.0"
+  "version": "1.91.0"
 }

python/README.rst

@@ -1,84 +1,77 @@
 =========================
-deltachat python bindings
+DeltaChat Python bindings
 =========================
 
-This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+This package provides `Python bindings`_ to the `deltachat-core library`_
+which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
+.. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
+.. _`Python bindings`: https://py.delta.chat/
 
 Installing pre-built packages (Linux-only)
-========================================================
+==========================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
 
 We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
-        virtualenv venv  # or: python -m venv
-        source venv/bin/activate
+        virtualenv env  # or: python -m venv
+        source env/bin/activate
 
 Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``venv`` directory and leaves
+modifies files in your ``env`` directory and leaves
 your system installation alone.
 
-For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
-github ``master`` branch::
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
+    pip install deltachat
 
 To verify it worked::
 
     python -c "import deltachat"
 
-.. note::
-
-    If you can help to automate the building of wheels for Mac or Windows,
-    that'd be much appreciated! please then get
-    `in contact with us <https://delta.chat/en/contribute>`_.
-
-
 Running tests
 =============
 
-After successful binding installation you can install a few more
-Python packages before running the tests::
+Recommended way to run tests is using `tox <https://tox.wiki>`_.
+After successful binding installation you can install tox
+and run the tests::
 
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
+    pip install tox
+    tox -e py3
 
 This will run all "offline" tests and skip all functional
 end-to-end tests that require accounts on real e-mail servers.
 
 .. _livetests:
 
-running "live" tests with temporary accounts
----------------------------------------------
+Running "live" tests with temporary accounts
+--------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
 
     export DCC_NEW_TMP_EMAIL=<URL you got from us>
 
 With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
+One hour is enough to invoke pytest and run all offline and online tests::
 
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
+    tox -e py3
 
 Each test run creates new accounts.
 
-
 .. _sourceinstall:
 
-Installing bindings from source (Updated: July 2020)
-=========================================================
+Installing bindings from source
+===============================
 
 Install Rust and Cargo first.
 The easiest is probably to use `rustup <https://rustup.rs/>`_.
@@ -97,74 +90,42 @@ E.g. on Debian-based systems `apt install python3 python3-pip
 python3-venv` should give you a usable python installation.
 
 Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
+virtual environment with dependencies using tox
+and activate it in your shell::
 
    cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
+   tox --devenv env
+   source env/bin/activate
 
 You should now be able to build the python bindings using the supplied script::
 
-   python install_python_bindings.py
+   python3 install_python_bindings.py
 
 The core compilation and bindings building might take a while,
 depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
-Code examples
-=============
-
-You may look at `examples <https://py.delta.chat/examples.html>`_.
-
-
-.. _`deltachat-core-rust github repository`: https://github.com/deltachat/deltachat-core-rust
-.. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
-
 
 Building manylinux based wheels
-====================================
+===============================
 
 Building portable manylinux wheels which come with libdeltachat.so
-can be done with docker-tooling.
+can be done with Docker_ or Podman_.
 
-using docker pull / premade images
-------------------------------------
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
 
-We publish a build environment under the ``deltachat/coredeps`` tag so
-that you can pull it from the ``hub.docker.com`` site's "deltachat"
-organization::
+If you want to build your own wheels, build container image first::
 
-    $ docker pull deltachat/coredeps
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
 
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
-
-
-Optionally build your own docker image
---------------------------------------
-
-If you want to build your own custom docker image you can do this::
-
-   $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
-
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
-up docker image called ``deltachat/coredeps``.  You can afterwards
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 
    $ docker images
 
+This docker image can be used to run tests and build Python wheels for all interpreters::
 
-Troubleshooting
----------------
-
-On more recent systems running the docker image may crash.  You can
-fix this by adding ``vsyscall=emulate`` to the Linux kernel boot
-arguments commandline.  E.g. on Debian you'd add this to
-``GRUB_CMDLINE_LINUX_DEFAULT`` in ``/etc/default/grub``.
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/pyproject.toml

@@ -1,7 +1,41 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2", "cffi>=1.0.0", "pkgconfig"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "deltachat"
+description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
+readme = "README.rst"
+requires-python = ">=3.7"
+authors = [
+    { name = "holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "Programming Language :: Python :: 3",
+    "Topic :: Communications :: Email",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "cffi>=1.0.0",
+    "imap-tools",
+    "pluggy",
+    "requests",
+]
+dynamic = [
+    "version"
+]
+
+[project.urls]
+"Home" = "https://github.com/deltachat/deltachat-core-rust/"
+"Bug Tracker" = "https://github.com/deltachat/deltachat-core-rust/issues"
+"Documentation" = "https://py.delta.chat/"
+
+[project.entry-points.pytest11]
+"deltachat.testplugin" = "deltachat.testplugin"
+
 [tool.setuptools_scm]
 root = ".."
 tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'

python/setup.py

@@ -1,40 +1,4 @@
-import os
-import re
-
-import setuptools
-
-
-def main():
-    with open("README.rst") as f:
-        long_description = f.read()
-    setuptools.setup(
-        name="deltachat",
-        description="Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat",
-        long_description=long_description,
-        author="holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors",
-        install_requires=["cffi>=1.0.0", "pluggy", "imap-tools", "requests"],
-        setup_requires=[
-            "setuptools_scm",  # required for compatibility with `python3 setup.py sdist`
-            "pkgconfig",
-        ],
-        packages=setuptools.find_packages("src"),
-        package_dir={"": "src"},
-        cffi_modules=["src/deltachat/_build.py:ffibuilder"],
-        entry_points={
-            "pytest11": [
-                "deltachat.testplugin = deltachat.testplugin",
-            ],
-        },
-        classifiers=[
-            "Development Status :: 4 - Beta",
-            "Intended Audience :: Developers",
-            "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
-            "Programming Language :: Python :: 3",
-            "Topic :: Communications :: Email",
-            "Topic :: Software Development :: Libraries",
-        ],
-    )
-
+from setuptools import setup
 
 if __name__ == "__main__":
-    main()
+    setup(cffi_modules=["src/deltachat/_build.py:ffibuilder"])

python/src/deltachat/__init__.py

@@ -60,29 +60,7 @@ def run_cmdline(argv=None, account_plugins=None):
 
     ac = Account(args.db)
 
-    if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
-        ac.add_account_plugin(log)
-
-    for plugin in account_plugins or []:
-        print("adding plugin", plugin)
-        ac.add_account_plugin(plugin)
-
-    if not ac.is_configured():
-        assert (
-            args.email and args.password
-        ), "you must specify --email and --password once to configure this database/account"
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
-
-    # start IO threads and configure if neccessary
-    ac.start_io()
+    ac.run_account(addr=args.email, password=args.password, account_plugins=account_plugins, show_ffi=args.show_ffi)
 
     print("{}: waiting for message".format(ac.get_config("addr")))
 

python/src/deltachat/account.py

@@ -20,7 +20,7 @@ from .cutil import (
     from_optional_dc_charpointer,
     iter_array,
 )
-from .events import EventThread
+from .events import EventThread, FFIEventLogger
 from .message import Message
 from .tracker import ConfigureTracker, ImexTracker
 
@@ -598,6 +598,36 @@ class Account(object):
     # meta API for start/stop and event based processing
     #
 
+    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
+        """get the account running, configure it if necessary. add plugins if provided.
+
+        :param addr: the email address of the account
+        :param password: the password of the account
+        :param account_plugins: a list of plugins to add
+        :param show_ffi: show low level ffi events
+        """
+        if show_ffi:
+            self.set_config("displayname", "bot")
+            log = FFIEventLogger(self)
+            self.add_account_plugin(log)
+
+        for plugin in account_plugins or []:
+            print("adding plugin", plugin)
+            self.add_account_plugin(plugin)
+
+        if not self.is_configured():
+            assert addr and password, "you must specify email and password once to configure this database/account"
+            self.set_config("addr", addr)
+            self.set_config("mail_pw", password)
+            self.set_config("mvbox_move", "0")
+            self.set_config("sentbox_watch", "0")
+            self.set_config("bot", "1")
+            configtracker = self.configure()
+            configtracker.wait_finish()
+
+        # start IO threads and configure if neccessary
+        self.start_io()
+
     def add_account_plugin(self, plugin, name=None):
         """add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.

python/tests/package_wheels.py

@@ -1,13 +0,0 @@
-import os
-import subprocess
-import sys
-
-if __name__ == "__main__":
-    assert len(sys.argv) == 2
-    wheelhousedir = sys.argv[1]
-    # pip wheel will build in an isolated tmp dir that does not have git
-    # history so setuptools_scm can not automatically determine a
-    # version there. So pass in the version through an env var.
-    version = subprocess.check_output(["python", "setup.py", "--version"]).strip().split(b"\n")[-1]
-    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version.decode("ascii")
-    subprocess.check_call(("pip wheel . -w %s" % wheelhousedir).split())

python/tests/test_1_online.py

@@ -1581,8 +1581,9 @@ def test_set_get_group_image(acfactory, data, lp):
 
     lp.sec("ac2: wait for receiving message from ac1")
     msg1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg1.is_system_message()  # Member added
     msg2 = ac2._evtracker.wait_next_incoming_message()
-    assert msg1.text == "hi" or msg2.text == "hi"
+    assert msg2.text == "hi"
     assert msg1.chat.id == msg2.chat.id
 
     lp.sec("ac2: see if chat now has got the profile image")
@@ -1596,6 +1597,8 @@ def test_set_get_group_image(acfactory, data, lp):
     lp.sec("ac2: delete profile image from chat")
     msg1.chat.remove_profile_image()
     msg_back = ac1._evtracker.wait_next_incoming_message()
+    assert msg_back.text == "Group image deleted by {}.".format(ac2.get_config("addr"))
+    assert msg_back.is_system_message()
     assert msg_back.chat == chat
     assert chat.get_profile_image() is None
 
@@ -1854,7 +1857,7 @@ def test_configure_error_msgs_invalid_server(acfactory):
     # Can't connect so it probably should say something about "internet"
     # again, should not repeat itself
     # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
-    # in configure/mod.rs returned false because the error message was changed
+    # in configure.rs returned false because the error message was changed
     # (i.e. did not contain "could not resolve" anymore)
     assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
     # Should mention that it can't connect:

python/tox.ini

@@ -9,9 +9,8 @@ envlist =
 [testenv]
 commands = 
     pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pip wheel . -w {toxworkdir}/wheelhouse --no-deps
 passenv = 
-    TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_NEW_TMP_EMAIL 

scripts/README.md

@@ -14,18 +14,15 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `doxygen/Dockerfile` specifies an image that contains
-  the doxygen tool which is used by `run-doxygen.sh` 
-  to generate C-docs which are then uploaded 
-  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-  (and the master branch is linked to https://c.delta.chat proper). 
+- `run-doxygen.sh` generates C-docs which are then uploaded to https://c.delta.chat/
 
+- `run_all.sh` builds Python wheels
 
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 
 from your local checkout/branch. You will need to be authorized to login to 
-the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type::
+the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type:
 
     scripts/manual_remote_tests.sh rust
     scripts/manual_remote_tests.sh python
@@ -33,19 +30,18 @@ the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
 
-# Outdated files (for later re-use)
+# coredeps Dockerfile
 
 `coredeps/Dockerfile` specifies an image that contains all 
-of Delta Chat's core dependencies. It used to run 
-python tests and build wheels (binary packages for Python)
+of Delta Chat's core dependencies. It is used to
+build python wheels (binary packages for Python).
 
 You can build the docker images yourself locally
-to avoid the relatively large download:: 
+to avoid the relatively large download:
  
     cd scripts  # where all CI things are 
-    docker build -t deltachat/coredeps docker-coredeps
-    docker build -t deltachat/doxygen docker-doxygen 
+    docker build -t deltachat/coredeps coredeps
 
-Additionally, you can install qemu and build arm64 docker image:
+Additionally, you can install qemu and build arm64 docker image on x86\_64 machine:
     apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64
+    docker build -t deltachat/coredeps-arm64 --build-arg BASEIMAGE=quay.io/pypa/manylinux2014_aarch64 coredeps

scripts/concourse/docs_wheels.yml

@@ -29,18 +29,19 @@ jobs:
         - name: c-docs
       image_resource:
         source:
-          repository: hrektts/doxygen
+          repository: alpine
         type: registry-image
       platform: linux
       run:
-        path: bash
+        path: sh
         args:
-        - -exc
+        - -ec
         - |
+          apk add --no-cache doxygen git
           cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
+          scripts/run-doxygen.sh
           cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
+          cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
 
   - task: upload-c-docs
     config:
@@ -302,3 +303,73 @@ jobs:
           devpi use https://m.devpi.net/dc/master
           devpi login ((devpi.login)) --password ((devpi.password))
           devpi upload py-wheels/*musllinux_1_1_x86_64*
+
+- name: python-musl-aarch64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: concourse/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/coredeps
+        UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload musl aarch64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*musllinux_1_1_aarch64*

scripts/docker-doxygen/Dockerfile

@@ -1,5 +0,0 @@
-FROM debian:stable
-
-# this is tagged as deltachat/doxygen 
-
-RUN apt-get update && apt-get install -y doxygen 

scripts/run-doxygen.sh

@@ -1,6 +1,5 @@
-#!/usr/bin/env bash
-
-set -ex
+#!/bin/sh
+set -e
 
 cd deltachat-ffi 
 PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

scripts/run_all.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Build the Delta Chat Core Rust library, Python wheels and docs 
 
@@ -8,21 +8,16 @@ set -e -x
 
 # compile core lib
 
-export PATH=/root/.cargo/bin:$PATH
 cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
 
 # Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
+export DCC_RS_DEV="$PWD"
 export DCC_RS_TARGET=release
 
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
+cd python
 
 TOXWORKDIR=.docker-tox
-pushd python
 # prepare a clean tox run
 rm -rf tests/__pycache__
 rm -rf src/deltachat/__pycache__
@@ -33,11 +28,13 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,pypy37,pypy38,pypy39,auditwheels
-popd
+
+# Try to build wheels for a range of interpreters, but don't fail if they are not available.
+# E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,pypy37,pypy38,pypy39,auditwheels --skip-missing-interpreters true
 
 
 echo -----------------------
 echo generating python docs
 echo -----------------------
-(cd python && tox --workdir "$TOXWORKDIR" -e doc)
+tox --workdir "$TOXWORKDIR" -e doc

src/configure.rs

@@ -12,9 +12,11 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use tokio::task;
 
 use crate::config::Config;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job;
+use crate::log::LogExt;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
@@ -101,35 +103,11 @@ impl Context {
         info!(self, "Configure ...");
 
         let mut param = LoginParam::load_candidate_params(self).await?;
+        let old_addr = self.get_config(Config::ConfiguredAddr).await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = param.provider {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    if !self.config_exists(def.key).await? {
-                        info!(self, "apply config_defaults {}={}", def.key, def.value);
-                        self.set_config(def.key, Some(def.value)).await?;
-                    } else {
-                        info!(
-                            self,
-                            "skip already set config_defaults {}={}", def.key, def.value
-                        );
-                    }
-                }
-            }
-
-            if !provider.after_login_hint.is_empty() {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(provider.after_login_hint.to_string());
-                if chat::add_device_msg(self, Some("core-provider-info"), Some(&mut msg))
-                    .await
-                    .is_err()
-                {
-                    warn!(self, "cannot add after_login_hint as core-provider-info");
-                }
-            }
-        }
+        on_configure_completed(self, param, old_addr).await?;
 
         success?;
         self.set_config(Config::NotifyAboutWrongPw, Some("1"))
@@ -138,6 +116,54 @@ impl Context {
     }
 }
 
+async fn on_configure_completed(
+    context: &Context,
+    param: LoginParam,
+    old_addr: Option<String>,
+) -> Result<()> {
+    if let Some(provider) = param.provider {
+        if let Some(config_defaults) = &provider.config_defaults {
+            for def in config_defaults.iter() {
+                if !context.config_exists(def.key).await? {
+                    info!(context, "apply config_defaults {}={}", def.key, def.value);
+                    context.set_config(def.key, Some(def.value)).await?;
+                } else {
+                    info!(
+                        context,
+                        "skip already set config_defaults {}={}", def.key, def.value
+                    );
+                }
+            }
+        }
+
+        if !provider.after_login_hint.is_empty() {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.text = Some(provider.after_login_hint.to_string());
+            if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg))
+                .await
+                .is_err()
+            {
+                warn!(context, "cannot add after_login_hint as core-provider-info");
+            }
+        }
+    }
+
+    if let Some(new_addr) = context.get_config(Config::ConfiguredAddr).await? {
+        if let Some(old_addr) = old_addr {
+            if !addr_cmp(&new_addr, &old_addr) {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text =
+                    Some(stock_str::aeap_explanation_and_link(context, old_addr, new_addr).await);
+                chat::add_device_msg(context, None, Some(&mut msg))
+                    .await
+                    .ok_or_log_msg(context, "Cannot add AEAP explanation");
+            }
+        }
+    }
+
+    Ok(())
+}
+
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 1);
 
@@ -642,6 +668,9 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
                 .to_lowercase()
                 .contains("temporary failure in name resolution")
             || e.msg.to_lowercase().contains("name or service not known")
+            || e.msg
+                .to_lowercase()
+                .contains("failed to lookup address information")
     }) {
         return stock_str::error_no_network(context).await;
     }

src/context.rs

@@ -594,7 +594,7 @@ impl Context {
 
         let list = if let Some(chat_id) = chat_id {
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -618,7 +618,7 @@ impl Context {
             // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
             // The limit is documented and UI may add a hint when getting 1000 results.
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id

src/download.rs

@@ -11,7 +11,7 @@ use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
-use crate::param::Params;
+use crate::param::{Param, Params};
 use crate::tools::time;
 use crate::{job_try, stock_str, EventType};
 use std::cmp::max;
@@ -69,6 +69,42 @@ impl Context {
             Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
         }
     }
+
+    // Merges the two messages to `placeholder_msg_id`;
+    // `full_msg_id` is no longer used afterwards.
+    pub(crate) async fn merge_messages(
+        &self,
+        full_msg_id: MsgId,
+        placeholder_msg_id: MsgId,
+    ) -> Result<()> {
+        let placeholder = Message::load_from_db(self, placeholder_msg_id).await?;
+        self.sql
+            .transaction(move |transaction| {
+                transaction
+                    .execute("DELETE FROM msgs WHERE id=?;", paramsv![placeholder_msg_id])?;
+                transaction.execute(
+                    "UPDATE msgs SET id=? WHERE id=?",
+                    paramsv![placeholder_msg_id, full_msg_id],
+                )?;
+                Ok(())
+            })
+            .await?;
+        let mut full = Message::load_from_db(self, placeholder_msg_id).await?;
+
+        for key in [
+            Param::WebxdcSummary,
+            Param::WebxdcSummaryTimestamp,
+            Param::WebxdcDocument,
+            Param::WebxdcDocumentTimestamp,
+        ] {
+            if let Some(value) = placeholder.param.get(key) {
+                full.param.set(key, value);
+            }
+        }
+        full.update_param(self).await?;
+
+        Ok(())
+    }
 }
 
 impl MsgId {
@@ -256,7 +292,7 @@ impl MimeMessage {
 mod tests {
     use num_traits::FromPrimitive;
 
-    use crate::chat::send_msg;
+    use crate::chat::{get_chat_msgs, send_msg};
     use crate::ephemeral::Timer;
     use crate::message::Viewtype;
     use crate::receive_imf::receive_imf_inner;
@@ -410,4 +446,119 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_status_update_expands_to_nothing() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat_id = alice.create_chat(&bob).await.id;
+
+        let file = alice.get_blobdir().join("minimal.xdc");
+        tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?;
+        let mut instance = Message::new(Viewtype::File);
+        instance.set_file(file.to_str().unwrap(), None);
+        let _sent1 = alice.send_msg(chat_id, &mut instance).await;
+
+        alice
+            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#, "d")
+            .await?;
+        alice.flush_status_updates().await?;
+        let sent2 = alice.pop_sent_msg().await;
+        let sent2_rfc742_mid = Message::load_from_db(&alice, sent2.sender_msg_id)
+            .await?
+            .rfc724_mid;
+
+        // not downloading the status update results in an placeholder
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            Some(sent2.payload().len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
+        // (usually status updates are too small for not being downloaded directly)
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            None,
+            false,
+        )
+        .await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdn_expands_to_nothing() -> Result<()> {
+        let bob = TestContext::new_bob().await;
+        let raw = b"Subject: Message opened\n\
+            Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <bar@example.org>\n\
+            To: Alice <alice@example.org>\n\
+            From: Bob <bob@example.org>\n\
+            Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+            boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: text/plain; charset=utf-8\n\
+            \n\
+            bla\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: message/disposition-notification\n\
+            \n\
+            Reporting-UA: Delta Chat 1.88.0\n\
+            Original-Recipient: rfc822;bob@example.org\n\
+            Final-Recipient: rfc822;bob@example.org\n\
+            Original-Message-ID: <foo@example.org>\n\
+            Disposition: manual-action/MDN-sent-automatically; displayed\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+            ";
+
+        // not downloading the mdn results in an placeholder
+        receive_imf_inner(
+            &bob,
+            "bar@example.org",
+            raw,
+            false,
+            Some(raw.len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the mdn afterwards expands to nothing and deletes the placeholder directly
+        // (usually mdn are too small for not being downloaded directly)
+        receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
 }

src/e2ee.rs

@@ -149,11 +149,13 @@ pub(crate) async fn get_autocrypt_peerstate(
 
     // Apply Autocrypt header
     if let Some(header) = autocrypt_header {
-        // The "from_nongossiped_fingerprint" part is for AEAP:
+        // The "from_verified_fingerprint" part is for AEAP:
         // If we know this fingerprint from another addr,
         // we may want to do a transition from this other addr
         // (and keep its peerstate)
-        peerstate = Peerstate::from_nongossiped_fingerprint_or_addr(
+        // For security reasons, for now, we only do a transition
+        // if the fingerprint is verified.
+        peerstate = Peerstate::from_verified_fingerprint_or_addr(
             context,
             &header.public_key.fingerprint(),
             from,

src/message.rs

@@ -19,7 +19,7 @@ use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::imap::markseen_on_imap_table;
-use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
+use crate::mimeparser::{parse_message_id, DeliveryReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::scheduler::InterruptInfo;
@@ -1532,7 +1532,7 @@ pub async fn handle_mdn(
 /// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
 pub(crate) async fn handle_ndn(
     context: &Context,
-    failed: &FailureReport,
+    failed: &DeliveryReport,
     error: Option<String>,
 ) -> Result<()> {
     if failed.rfc724_mid.is_empty() {
@@ -1588,7 +1588,7 @@ pub(crate) async fn handle_ndn(
 
 async fn ndn_maybe_add_info_msg(
     context: &Context,
-    failed: &FailureReport,
+    failed: &DeliveryReport,
     chat_id: ChatId,
     chat_type: Chattype,
 ) -> Result<()> {

src/mimeparser.rs

@@ -73,7 +73,7 @@ pub struct MimeMessage {
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
-    pub(crate) failure_report: Option<FailureReport>,
+    pub(crate) delivery_report: Option<DeliveryReport>,
 
     /// Standard USENET signature, if any.
     pub(crate) footer: Option<String>,
@@ -332,7 +332,7 @@ impl MimeMessage {
             webxdc_status_update: None,
             user_avatar: None,
             group_avatar: None,
-            failure_report: None,
+            delivery_report: None,
             footer: None,
             is_mime_modified: false,
             decoded_data: Vec::new(),
@@ -417,6 +417,8 @@ impl MimeMessage {
                 self.is_system_message = SystemMessage::ChatProtectionEnabled;
             } else if value == "protection-disabled" {
                 self.is_system_message = SystemMessage::ChatProtectionDisabled;
+            } else if value == "group-avatar-changed" {
+                self.is_system_message = SystemMessage::GroupImageChanged;
             }
         } else if self.get_header(HeaderDef::ChatGroupMemberRemoved).is_some() {
             self.is_system_message = SystemMessage::MemberRemovedFromGroup;
@@ -424,10 +426,6 @@ impl MimeMessage {
             self.is_system_message = SystemMessage::MemberAddedToGroup;
         } else if self.get_header(HeaderDef::ChatGroupNameChanged).is_some() {
             self.is_system_message = SystemMessage::GroupNameChanged;
-        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
-            if value == "group-avatar-changed" {
-                self.is_system_message = SystemMessage::GroupImageChanged;
-            }
         }
     }
 
@@ -535,7 +533,9 @@ impl MimeMessage {
         self.parse_system_message_headers(context);
         self.parse_avatar_headers(context).await;
         self.parse_videochat_headers();
-        self.squash_attachment_parts();
+        if self.delivery_report.is_none() {
+            self.squash_attachment_parts();
+        }
 
         if let Some(ref subject) = self.get_subject() {
             let mut prepend_subject = true;
@@ -867,7 +867,7 @@ impl MimeMessage {
                         // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
                         Some("delivery-status") | None => {
                             if let Some(report) = self.process_delivery_status(context, mail)? {
-                                self.failure_report = Some(report);
+                                self.delivery_report = Some(report);
                             }
 
                             // Add all parts (we need another part, preferably text/plain, to show as an error message)
@@ -1278,9 +1278,46 @@ impl MimeMessage {
         &self,
         context: &Context,
         report: &mailparse::ParsedMail<'_>,
-    ) -> Result<Option<FailureReport>> {
+    ) -> Result<Option<DeliveryReport>> {
+        // Assume failure.
+        let mut failure = true;
+
+        if let Some(status_part) = report.subparts.get(1) {
+            // RFC 3464 defines `message/delivery-status`
+            // RFC 6533 defines `message/global-delivery-status`
+            if status_part.ctype.mimetype != "message/delivery-status"
+                && status_part.ctype.mimetype != "message/global-delivery-status"
+            {
+                warn!(context, "Second part of Delivery Status Notification is not message/delivery-status or message/global-delivery-status, ignoring");
+                return Ok(None);
+            }
+
+            let status_body = status_part.get_body_raw()?;
+
+            // Skip per-message fields.
+            let (_, sz) = mailparse::parse_headers(&status_body)?;
+
+            // Parse first set of per-recipient fields
+            if let Some(status_body) = status_body.get(sz..) {
+                let (status_fields, _) = mailparse::parse_headers(status_body)?;
+                if let Some(action) = status_fields.get_first_value("action") {
+                    if action != "failed" {
+                        info!(context, "DSN with {:?} action", action);
+                        failure = false;
+                    }
+                } else {
+                    warn!(context, "DSN without action");
+                }
+            } else {
+                warn!(context, "DSN without per-recipient fields");
+            }
+        } else {
+            // No message/delivery-status part.
+            return Ok(None);
+        }
+
         // parse as mailheaders
-        if let Some(original_msg) = report.subparts.iter().find(|p| {
+        if let Some(original_msg) = report.subparts.get(2).filter(|p| {
             p.ctype.mimetype.contains("rfc822")
                 || p.ctype.mimetype == "message/global"
                 || p.ctype.mimetype == "message/global-headers"
@@ -1301,9 +1338,10 @@ impl MimeMessage {
                     None // We do not know which recipient failed
                 };
 
-                return Ok(Some(FailureReport {
+                return Ok(Some(DeliveryReport {
                     rfc724_mid: original_message_id,
                     failed_recipient: to.map(|s| s.addr),
+                    failure,
                 }));
             }
 
@@ -1384,7 +1422,7 @@ impl MimeMessage {
         } else {
             false
         };
-        if maybe_ndn && self.failure_report.is_none() {
+        if maybe_ndn && self.delivery_report.is_none() {
             static RE: Lazy<regex::Regex> =
                 Lazy::new(|| regex::Regex::new(r"Message-ID:(.*)").unwrap());
             for captures in self
@@ -1398,9 +1436,10 @@ impl MimeMessage {
                     if let Ok(Some(_)) =
                         message::rfc724_mid_exists(context, &original_message_id).await
                     {
-                        self.failure_report = Some(FailureReport {
+                        self.delivery_report = Some(DeliveryReport {
                             rfc724_mid: original_message_id,
                             failed_recipient: None,
+                            failure: true,
                         })
                     }
                 }
@@ -1438,13 +1477,15 @@ impl MimeMessage {
             }
         }
 
-        if let Some(failure_report) = &self.failure_report {
-            let error = parts
-                .iter()
-                .find(|p| p.typ == Viewtype::Text)
-                .map(|p| p.msg.clone());
-            if let Err(e) = message::handle_ndn(context, failure_report, error).await {
-                warn!(context, "Could not handle ndn: {}", e);
+        if let Some(delivery_report) = &self.delivery_report {
+            if delivery_report.failure {
+                let error = parts
+                    .iter()
+                    .find(|p| p.typ == Viewtype::Text)
+                    .map(|p| p.msg.clone());
+                if let Err(e) = message::handle_ndn(context, delivery_report, error).await {
+                    warn!(context, "Could not handle ndn: {}", e);
+                }
             }
         }
     }
@@ -1524,6 +1565,7 @@ async fn update_gossip_peerstates(
     Ok(gossiped_addr)
 }
 
+/// Message Disposition Notification (RFC 8098)
 #[derive(Debug)]
 pub(crate) struct Report {
     /// Original-Message-ID header
@@ -1535,10 +1577,12 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+/// Delivery Status Notification (RFC 3464, RFC 6533)
 #[derive(Debug)]
-pub(crate) struct FailureReport {
+pub(crate) struct DeliveryReport {
     pub rfc724_mid: String,
     pub failed_recipient: Option<String>,
+    pub failure: bool,
 }
 
 #[allow(clippy::indexing_slicing)]

src/peerstate.rs

@@ -165,7 +165,7 @@ impl Peerstate {
         Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
-    pub async fn from_nongossiped_fingerprint_or_addr(
+    pub async fn from_verified_fingerprint_or_addr(
         context: &Context,
         fingerprint: &Fingerprint,
         addr: &str,
@@ -174,9 +174,9 @@ impl Peerstate {
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
                      FROM acpeerstates  \
-                     WHERE public_key_fingerprint=? \
+                     WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
-                     ORDER BY public_key_fingerprint=? DESC, last_seen DESC LIMIT 1;";
+                     ORDER BY verified_key_fingerprint=? DESC, last_seen DESC LIMIT 1;";
         let fp = fingerprint.hex();
         Self::from_stmt(context, query, paramsv![fp, addr, fp]).await
     }
@@ -516,22 +516,22 @@ impl Peerstate {
             .with_context(|| format!("contact with peerstate.addr {:?} not found", &self.addr))?;
 
         let chats = Chatlist::try_load(context, 0, None, Some(contact_id)).await?;
+        let msg = match &change {
+            PeerstateChange::FingerprintChange => {
+                stock_str::contact_setup_changed(context, self.addr.clone()).await
+            }
+            PeerstateChange::Aeap(new_addr) => {
+                let old_contact = Contact::load_from_db(context, contact_id).await?;
+                stock_str::aeap_addr_changed(
+                    context,
+                    old_contact.get_display_name(),
+                    &self.addr,
+                    new_addr,
+                )
+                .await
+            }
+        };
         for (chat_id, msg_id) in chats.iter() {
-            let msg = match &change {
-                PeerstateChange::FingerprintChange => {
-                    stock_str::contact_setup_changed(context, self.addr.clone()).await
-                }
-                PeerstateChange::Aeap(new_addr) => {
-                    let old_contact = Contact::load_from_db(context, contact_id).await?;
-                    stock_str::aeap_addr_changed(
-                        context,
-                        old_contact.get_display_name(),
-                        &self.addr,
-                        new_addr,
-                    )
-                    .await
-                }
-            };
             let timestamp_sort = if let Some(msg_id) = msg_id {
                 let lastmsg = Message::load_from_db(context, *msg_id).await?;
                 lastmsg.timestamp_sort
@@ -545,21 +545,23 @@ impl Peerstate {
                     .await?
                     .unwrap_or(0)
             };
-            chat::add_info_msg_with_cmd(
-                context,
-                *chat_id,
-                &msg,
-                SystemMessage::Unknown,
-                timestamp_sort,
-                Some(timestamp),
-                None,
-                None,
-            )
-            .await?;
 
             if let PeerstateChange::Aeap(new_addr) = &change {
                 let chat = Chat::load_from_db(context, *chat_id).await?;
-                if chat.typ == Chattype::Group || chat.typ == Chattype::Broadcast {
+
+                if chat.typ == Chattype::Group && !chat.is_protected() {
+                    // Don't add an info_msg to the group, in order not to make the user think
+                    // that the address was automatically replaced in the group.
+                    continue;
+                }
+
+                // For security reasons, for now, we only do the AEAP transition if the fingerprint
+                // is verified (that's what from_verified_fingerprint_or_addr() does).
+                // In order to not have inconsistent group membership state, we then only do the
+                // transition in verified groups and in broadcast lists.
+                if (chat.typ == Chattype::Group && chat.is_protected())
+                    || chat.typ == Chattype::Broadcast
+                {
                     chat::remove_from_chat_contacts_table(context, *chat_id, contact_id).await?;
 
                     let (new_contact_id, _) =
@@ -572,6 +574,18 @@ impl Peerstate {
                     context.emit_event(EventType::ChatModified(*chat_id));
                 }
             }
+
+            chat::add_info_msg_with_cmd(
+                context,
+                *chat_id,
+                &msg,
+                SystemMessage::Unknown,
+                timestamp_sort,
+                Some(timestamp),
+                None,
+                None,
+            )
+            .await?;
         }
 
         Ok(())
@@ -621,7 +635,7 @@ pub async fn maybe_do_aeap_transition(
                 && mime_parser.from_is_signed
                 && info.message_time > peerstate.last_seen
             {
-                // Add an info messages to all chats with this contact
+                // Add info messages to chats with this (verified) contact
                 //
                 peerstate
                     .handle_setup_change(

src/qr.rs

@@ -61,6 +61,7 @@ pub enum Qr {
     },
     Addr {
         contact_id: ContactId,
+        draft: Option<String>,
     },
     Url {
         url: String,
@@ -451,15 +452,52 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
 async fn decode_mailto(context: &Context, qr: &str) -> Result<Qr> {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
-    let addr = if let Some(query_index) = payload.find('?') {
-        &payload[..query_index]
+    let (addr, query) = if let Some(query_index) = payload.find('?') {
+        (&payload[..query_index], &payload[query_index + 1..])
     } else {
-        payload
+        (payload, "")
+    };
+
+    let param: BTreeMap<&str, &str> = query
+        .split('&')
+        .filter_map(|s| {
+            if let [key, value] = s.splitn(2, '=').collect::<Vec<_>>()[..] {
+                Some((key, value))
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    let subject = if let Some(subject) = param.get("subject") {
+        subject.to_string()
+    } else {
+        "".to_string()
+    };
+    let draft = if let Some(body) = param.get("body") {
+        if subject.is_empty() {
+            body.to_string()
+        } else {
+            subject + "\n" + body
+        }
+    } else {
+        subject
+    };
+    let draft = draft.replace('+', "%20"); // sometimes spaces are encoded as `+`
+    let draft = match percent_decode_str(&draft).decode_utf8() {
+        Ok(decoded_draft) => decoded_draft.to_string(),
+        Err(_err) => draft,
     };
 
     let addr = normalize_address(addr)?;
     let name = "".to_string();
-    Qr::from_address(context, name, addr).await
+    Qr::from_address(
+        context,
+        name,
+        addr,
+        if draft.is_empty() { None } else { Some(draft) },
+    )
+    .await
 }
 
 /// Extract address for the smtp scheme.
@@ -477,7 +515,7 @@ async fn decode_smtp(context: &Context, qr: &str) -> Result<Qr> {
 
     let addr = normalize_address(addr)?;
     let name = "".to_string();
-    Qr::from_address(context, name, addr).await
+    Qr::from_address(context, name, addr, None).await
 }
 
 /// Extract address for the matmsg scheme.
@@ -502,7 +540,7 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Result<Qr> {
 
     let addr = normalize_address(addr)?;
     let name = "".to_string();
-    Qr::from_address(context, name, addr).await
+    Qr::from_address(context, name, addr, None).await
 }
 
 static VCARD_NAME_RE: Lazy<regex::Regex> =
@@ -531,14 +569,19 @@ async fn decode_vcard(context: &Context, qr: &str) -> Result<Qr> {
         bail!("Bad e-mail address");
     };
 
-    Qr::from_address(context, name, addr).await
+    Qr::from_address(context, name, addr, None).await
 }
 
 impl Qr {
-    pub async fn from_address(context: &Context, name: String, addr: String) -> Result<Self> {
+    pub async fn from_address(
+        context: &Context,
+        name: String,
+        addr: String,
+        draft: Option<String>,
+    ) -> Result<Self> {
         let (contact_id, _) =
             Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan).await?;
-        Ok(Qr::Addr { contact_id })
+        Ok(Qr::Addr { contact_id, draft })
     }
 }
 
@@ -619,12 +662,13 @@ mod tests {
             "BEGIN:VCARD\nVERSION:3.0\nN:Last;First\nEMAIL;TYPE=INTERNET:stress@test.local\nEND:VCARD"
         ).await?;
 
-        if let Qr::Addr { contact_id } = qr {
+        if let Qr::Addr { contact_id, draft } = qr {
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
             assert_eq!(contact.get_addr(), "stress@test.local");
             assert_eq!(contact.get_name(), "First Last");
             assert_eq!(contact.get_authname(), "");
             assert_eq!(contact.get_display_name(), "First Last");
+            assert!(draft.is_none());
         } else {
             bail!("Wrong QR code type");
         }
@@ -642,9 +686,10 @@ mod tests {
         )
         .await?;
 
-        if let Qr::Addr { contact_id } = qr {
+        if let Qr::Addr { contact_id, draft } = qr {
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
             assert_eq!(contact.get_addr(), "stress@test.local");
+            assert!(draft.is_none());
         } else {
             bail!("Wrong QR code type");
         }
@@ -658,20 +703,22 @@ mod tests {
 
         let qr = check_qr(
             &ctx.ctx,
-            "mailto:stress@test.local?subject=hello&body=world",
+            "mailto:stress@test.local?subject=hello&body=beautiful+world",
         )
         .await?;
-        if let Qr::Addr { contact_id } = qr {
+        if let Qr::Addr { contact_id, draft } = qr {
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
             assert_eq!(contact.get_addr(), "stress@test.local");
+            assert_eq!(draft.unwrap(), "hello\nbeautiful world");
         } else {
             bail!("Wrong QR code type");
         }
 
         let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org").await?;
-        if let Qr::Addr { contact_id } = res {
+        if let Qr::Addr { contact_id, draft } = res {
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
             assert_eq!(contact.get_addr(), "no-questionmark@example.org");
+            assert!(draft.is_none());
         } else {
             bail!("Wrong QR code type");
         }
@@ -686,11 +733,12 @@ mod tests {
     async fn test_decode_smtp() -> Result<()> {
         let ctx = TestContext::new().await;
 
-        if let Qr::Addr { contact_id } =
+        if let Qr::Addr { contact_id, draft } =
             check_qr(&ctx.ctx, "SMTP:stress@test.local:subjecthello:bodyworld").await?
         {
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
             assert_eq!(contact.get_addr(), "stress@test.local");
+            assert!(draft.is_none());
         } else {
             bail!("Wrong QR code type");
         }

src/ratelimit.rs

@@ -51,7 +51,7 @@ impl Ratelimit {
 
     /// Returns true if it is allowed to send a message.
     fn can_send_at(&self, now: SystemTime) -> bool {
-        self.current_value_at(now) <= self.quota
+        self.current_value_at(now) + 1.0 <= self.quota
     }
 
     /// Returns true if can send another message now.
@@ -62,7 +62,7 @@ impl Ratelimit {
     }
 
     fn send_at(&mut self, now: SystemTime) {
-        self.current_value = self.current_value_at(now) + 1.0;
+        self.current_value = f64::min(self.quota, self.current_value_at(now) + 1.0);
         self.last_update = now;
     }
 
@@ -77,10 +77,10 @@ impl Ratelimit {
 
     fn until_can_send_at(&self, now: SystemTime) -> Duration {
         let current_value = self.current_value_at(now);
-        if current_value <= self.quota {
+        if current_value + 1.0 <= self.quota {
             Duration::ZERO
         } else {
-            let requirement = current_value - self.quota;
+            let requirement = current_value + 1.0 - self.quota;
             let rate = self.quota / self.window.as_secs_f64();
             Duration::from_secs_f64(requirement / rate)
         }
@@ -109,8 +109,6 @@ mod tests {
         ratelimit.send_at(now);
         assert!(ratelimit.can_send_at(now));
         ratelimit.send_at(now);
-        assert!(ratelimit.can_send_at(now));
-        ratelimit.send_at(now);
 
         // Can't send more messages now.
         assert!(!ratelimit.can_send_at(now));
@@ -125,11 +123,8 @@ mod tests {
         // Send one more message anyway, over quota.
         ratelimit.send_at(now);
 
-        // Waiting 20 seconds is not enough.
-        let now = now + Duration::from_secs(20);
-        assert!(!ratelimit.can_send_at(now));
-
-        // Can send another message after 40 seconds.
+        // Always can send another message after 20 seconds,
+        // leaky bucket never overflows.
         let now = now + Duration::from_secs(20);
         assert!(ratelimit.can_send_at(now));
 

src/receive_imf.rs

@@ -130,15 +130,14 @@ pub(crate) async fn receive_imf_inner(
                     context,
                     "Message already partly in DB, replacing by full message."
                 );
-                old_msg_id.delete_from_db(context).await?;
-                true
+                Some(old_msg_id)
             } else {
                 // the message was probably moved around.
                 info!(context, "Message already in DB, doing nothing.");
                 return Ok(None);
             }
         } else {
-            false
+            None
         };
 
     // the function returns the number of created messages in the database
@@ -189,8 +188,9 @@ pub(crate) async fn receive_imf_inner(
         sent_timestamp,
         rcvd_timestamp,
         from_id,
-        seen || replace_partial_download,
+        seen || replace_partial_download.is_some(),
         is_partial_download,
+        replace_partial_download,
         fetching_existing_messages,
         prevent_rename,
     )
@@ -322,7 +322,7 @@ pub(crate) async fn receive_imf_inner(
         }
     }
 
-    if replace_partial_download {
+    if replace_partial_download.is_some() {
         context.emit_msgs_changed(chat_id, MsgId::new(0));
     } else if !chat_id.is_trash() {
         let fresh = received_msg.state == MessageState::InFresh;
@@ -401,6 +401,7 @@ async fn add_parts(
     from_id: ContactId,
     seen: bool,
     is_partial_download: Option<u32>,
+    replace_msg_id: Option<MsgId>,
     fetching_existing_messages: bool,
     prevent_rename: bool,
 ) -> Result<ReceivedMsg> {
@@ -496,9 +497,9 @@ async fn add_parts(
             ChatIdBlocked::lookup_by_contact(context, from_id).await?
         };
 
-        if chat_id.is_none() && mime_parser.failure_report.is_some() {
+        if chat_id.is_none() && mime_parser.delivery_report.is_some() {
             chat_id = Some(DC_CHAT_ID_TRASH);
-            info!(context, "Message belongs to an NDN (TRASH)",);
+            info!(context, "Message is a DSN (TRASH)",);
         }
 
         if chat_id.is_none() {
@@ -1145,6 +1146,17 @@ INSERT INTO msgs
     }
     drop(conn);
 
+    if let Some(replace_msg_id) = replace_msg_id {
+        if let Some(created_msg_id) = created_db_entries.pop() {
+            context
+                .merge_messages(created_msg_id, replace_msg_id)
+                .await?;
+            created_db_entries.push(replace_msg_id);
+        } else {
+            replace_msg_id.delete_from_db(context).await?;
+        }
+    }
+
     chat_id.unarchive_if_not_muted(context).await?;
 
     info!(
@@ -2647,7 +2659,7 @@ mod tests {
             "shenauithz@testrun.org",
             "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
             include_bytes!("../test-data/message/tiscali_ndn.eml"),
-            Some("Delivery to at least one recipient failed."),
+            Some("Delivery status notification –       This is an automatically generated Delivery Status Notification.      \n\nDelivery to the following recipients was aborted after 2 second(s):\n\n  * shenauithz@testrun.org"),
         )
         .await;
     }
@@ -2724,6 +2736,32 @@ mod tests {
         .await;
     }
 
+    /// Tests that text part is not squashed into OpenPGP attachment.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_parse_ndn_with_attachment() {
+        test_parse_ndn(
+            "alice@example.org",
+            "bob@example.net",
+            "Mr.I6Da6dXcTel.TroC5J3uSDH@example.org",
+            include_bytes!("../test-data/message/ndn_with_attachment.eml"),
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host relay01.example.org.\n\nI'm sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It's attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<bob@example.net>: host mx2.example.net[80.241.60.215] said: 552 5.2.2\n    <bob@example.net>: Recipient address rejected: Mailbox quota exceeded (in\n    reply to RCPT TO command)\n\n<bob2@example.net>: host mx1.example.net[80.241.60.212] said: 552 5.2.2\n    <bob2@example.net>: Recipient address rejected: Mailbox quota\n    exceeded (in reply to RCPT TO command)")
+        )
+        .await;
+    }
+
+    /// Test that DSN is not treated as NDN if Action: is not "failed"
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_parse_dsn_relayed() {
+        test_parse_ndn(
+            "anon_1@posteo.de",
+            "anon_2@gmx.at",
+            "8b7b1a9d0c8cc588c7bcac47f5687634@posteo.de",
+            include_bytes!("../test-data/message/dsn_relayed.eml"),
+            None,
+        )
+        .await;
+    }
+
     // ndn = Non Delivery Notification
     async fn test_parse_ndn(
         self_addr: &str,
@@ -2773,7 +2811,14 @@ mod tests {
         receive_imf(&t, raw_ndn, false).await.unwrap();
         let msg = Message::load_from_db(&t, msg_id).await.unwrap();
 
-        assert_eq!(msg.state, MessageState::OutFailed);
+        assert_eq!(
+            msg.state,
+            if error_msg.is_some() {
+                MessageState::OutFailed
+            } else {
+                MessageState::OutDelivered
+            }
+        );
 
         assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
     }

src/scheduler.rs

@@ -324,29 +324,25 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
         let mut timeout = None;
         loop {
-            match send_smtp_messages(&ctx, &mut connection).await {
-                Err(err) => {
-                    warn!(ctx, "send_smtp_messages failed: {:#}", err);
-                    timeout = Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
-                }
-                Ok(ratelimited) => {
-                    if ratelimited {
-                        let duration_until_can_send = ctx.ratelimit.read().await.until_can_send();
-                        info!(
-                            ctx,
-                            "smtp got rate limited, waiting for {} until can send again",
-                            duration_to_str(duration_until_can_send)
-                        );
-                        tokio::time::timeout(duration_until_can_send, async {
-                            idle_interrupt_receiver.recv().await.unwrap_or_default()
-                        })
-                        .await
-                        .unwrap_or_default();
-                        continue;
-                    } else {
-                        timeout = None;
-                    }
+            if let Err(err) = send_smtp_messages(&ctx, &mut connection).await {
+                warn!(ctx, "send_smtp_messages failed: {:#}", err);
+                timeout = Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
+            } else {
+                let duration_until_can_send = ctx.ratelimit.read().await.until_can_send();
+                if !duration_until_can_send.is_zero() {
+                    info!(
+                        ctx,
+                        "smtp got rate limited, waiting for {} until can send again",
+                        duration_to_str(duration_until_can_send)
+                    );
+                    tokio::time::timeout(duration_until_can_send, async {
+                        idle_interrupt_receiver.recv().await.unwrap_or_default()
+                    })
+                    .await
+                    .unwrap_or_default();
+                    continue;
                 }
+                timeout = None;
             }
 
             // Fake Idle

src/smtp.rs

@@ -477,21 +477,17 @@ pub(crate) async fn send_msg_to_smtp(
 }
 
 /// Attempts to send queued MDNs.
-///
-/// Returns true if there are more MDNs to send, but rate limiter does not
-/// allow to send them. Returns false if there are no more MDNs to send.
-/// If sending an MDN fails, returns an error.
-async fn send_mdns(context: &Context, connection: &mut Smtp) -> Result<bool> {
+async fn send_mdns(context: &Context, connection: &mut Smtp) -> Result<()> {
     loop {
         if !context.ratelimit.read().await.can_send() {
             info!(context, "Ratelimiter does not allow sending MDNs now");
-            return Ok(true);
+            return Ok(());
         }
 
         let more_mdns = send_mdn(context, connection).await?;
         if !more_mdns {
             // No more MDNs to send.
-            return Ok(false);
+            return Ok(());
         }
     }
 }
@@ -500,10 +496,8 @@ async fn send_mdns(context: &Context, connection: &mut Smtp) -> Result<bool> {
 ///
 /// Logs and ignores SMTP errors to ensure that a single SMTP message constantly failing to be sent
 /// does not block other messages in the queue from being sent.
-///
-/// Returns true if sending was ratelimited, false otherwise. Errors are propagated to the caller.
-pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) -> Result<bool> {
-    let mut ratelimited = if context.ratelimit.read().await.can_send() {
+pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) -> Result<()> {
+    let ratelimited = if context.ratelimit.read().await.can_send() {
         // add status updates and sync messages to end of sending queue
         context.flush_status_updates().await?;
         context.send_sync_msg().await?;
@@ -538,12 +532,11 @@ pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp)
     // do not attempt to send MDNs if ratelimited happend before on status-updates/sync:
     // instead, let the caller recall this function so that more important status-updates/sync are sent out.
     if !ratelimited {
-        ratelimited = send_mdns(context, connection)
+        send_mdns(context, connection)
             .await
             .context("failed to send MDNs")?;
     }
-
-    Ok(ratelimited)
+    Ok(())
 }
 
 /// Tries to send MDN for message `msg_id` to `contact_id`.

src/stock_str.rs

@@ -335,6 +335,11 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "%1$s changed their address from %2$s to %3$s"))]
     AeapAddrChanged = 122,
+
+    #[strum(props(
+        fallback = "You changed your email address from %1$s to %2$s.\n\nIf you now send a message to a verified group, contacts there will automatically replace the old with your new address.\n\nIt's highly advised to set up your old email provider to forward all emails to your new email address. Otherwise you might miss messages of contacts who did not get your new address yet."
+    ))]
+    AeapExplanationAndLink = 123,
 }
 
 impl StockMessage {
@@ -1104,6 +1109,17 @@ pub(crate) async fn aeap_addr_changed(
         .replace3(new_addr)
 }
 
+pub(crate) async fn aeap_explanation_and_link(
+    context: &Context,
+    old_addr: impl AsRef<str>,
+    new_addr: impl AsRef<str>,
+) -> String {
+    translated(context, StockMessage::AeapExplanationAndLink)
+        .await
+        .replace1(old_addr)
+        .replace2(new_addr)
+}
+
 impl Context {
     /// Set the stock string for the [StockMessage].
     ///

src/tests/aeap.rs

@@ -184,9 +184,11 @@ async fn check_aeap_transition(
     let already_new_contact = Contact::create(&bob, "Alice", "fiona@example.net")
         .await
         .unwrap();
-    chat::add_contact_to_chat(&bob, groups[0], already_new_contact)
-        .await
-        .unwrap();
+    if verified {
+        chat::add_contact_to_chat(&bob, groups[2], already_new_contact)
+            .await
+            .unwrap();
+    }
 
     // groups 0 and 2 stay unpromoted (i.e. local
     // on Bob's device, Alice doesn't know about them)
@@ -219,7 +221,7 @@ async fn check_aeap_transition(
 
     tcm.section("Check that the AEAP transition worked");
     check_that_transition_worked(
-        &groups,
+        &groups[2..],
         &alice,
         "alice@example.org",
         ALICE_NEW_ADDR,
@@ -227,6 +229,7 @@ async fn check_aeap_transition(
         &bob,
     )
     .await;
+    check_no_transition_done(&groups[0..2], "alice@example.org", &bob).await;
 
     // Assert that the autocrypt header is also applied to the peerstate
     // if the address changed
@@ -246,7 +249,7 @@ async fn check_aeap_transition(
     assert_eq!(recvd.text.unwrap(), "Hello from my old addr!");
 
     check_that_transition_worked(
-        &groups,
+        &groups[2..],
         &alice,
         // Note that "alice@example.org" and ALICE_NEW_ADDR are switched now:
         ALICE_NEW_ADDR,
@@ -274,11 +277,19 @@ async fn check_that_transition_worked(
         let members = chat::get_chat_contacts(bob, *group).await.unwrap();
         // In all the groups, exactly Bob and Alice's new number are members.
         // (and Alice's new number isn't in there twice)
-        assert_eq!(members.len(), 2);
+        assert_eq!(
+            members.len(),
+            2,
+            "Group {} has members {:?}, but should have members {:?} and {:?}",
+            group,
+            &members,
+            new_contact,
+            ContactId::SELF
+        );
         assert!(members.contains(&new_contact));
         assert!(members.contains(&ContactId::SELF));
 
-        let info_msg = get_last_info_msg(bob, *group).await;
+        let info_msg = get_last_info_msg(bob, *group).await.unwrap();
         let expected_text =
             stock_str::aeap_addr_changed(bob, name, old_alice_addr, new_alice_addr).await;
         assert_eq!(info_msg.text.unwrap(), expected_text);
@@ -291,6 +302,36 @@ async fn check_that_transition_worked(
     }
 }
 
+async fn check_no_transition_done(groups: &[ChatId], old_alice_addr: &str, bob: &TestContext) {
+    let old_contact = Contact::lookup_id_by_addr(bob, old_alice_addr, contact::Origin::Unknown)
+        .await
+        .unwrap()
+        .unwrap();
+
+    for group in groups {
+        let members = chat::get_chat_contacts(bob, *group).await.unwrap();
+        // In all the groups, exactly Bob and Alice's _old_ number are members.
+        assert_eq!(
+            members.len(),
+            2,
+            "Group {} has members {:?}, but should have members {:?} and {:?}",
+            group,
+            &members,
+            old_contact,
+            ContactId::SELF
+        );
+        assert!(members.contains(&old_contact));
+        assert!(members.contains(&ContactId::SELF));
+
+        let last_info_msg = get_last_info_msg(bob, *group).await;
+        assert!(
+            last_info_msg.is_none(),
+            "{:?} shouldn't be there (or it's an unrelated info msg)",
+            last_info_msg
+        );
+    }
+}
+
 async fn mark_as_verified(this: &TestContext, other: &TestContext) {
     let other_addr = other.get_primary_self_addr().await.unwrap();
     let mut peerstate = peerstate::Peerstate::from_addr(this, &other_addr)
@@ -305,16 +346,16 @@ async fn mark_as_verified(this: &TestContext, other: &TestContext) {
     peerstate.save_to_db(&this.sql, false).await.unwrap();
 }
 
-async fn get_last_info_msg(t: &TestContext, chat_id: ChatId) -> Message {
+async fn get_last_info_msg(t: &TestContext, chat_id: ChatId) -> Option<Message> {
     let msgs = chat::get_chat_msgs(&t.ctx, chat_id, constants::DC_GCM_INFO_ONLY)
         .await
         .unwrap();
-    let msg_id = if let chat::ChatItem::Message { msg_id } = msgs.last().unwrap() {
+    let msg_id = if let chat::ChatItem::Message { msg_id } = msgs.last()? {
         msg_id
     } else {
-        panic!("Wrong item type");
+        return None;
     };
-    Message::load_from_db(&t.ctx, *msg_id).await.unwrap()
+    Some(Message::load_from_db(&t.ctx, *msg_id).await.unwrap())
 }
 
 /// Test that an attacker - here Fiona - can't replay a message sent by Alice

src/webxdc.rs

@@ -14,6 +14,7 @@ use tokio::io::AsyncReadExt;
 use crate::chat::Chat;
 use crate::contact::ContactId;
 use crate::context::Context;
+use crate::download::DownloadState;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
@@ -330,10 +331,12 @@ impl Context {
 
         let status_update_serial = StatusUpdateSerial(u32::try_from(rowid)?);
 
-        self.emit_event(EventType::WebxdcStatusUpdate {
-            msg_id: instance.id,
-            status_update_serial,
-        });
+        if instance.viewtype == Viewtype::Webxdc {
+            self.emit_event(EventType::WebxdcStatusUpdate {
+                msg_id: instance.id,
+                status_update_serial,
+            });
+        }
 
         Ok(status_update_serial)
     }
@@ -475,6 +478,8 @@ impl Context {
         } else if let Some(parent) = msg.parent(self).await? {
             if parent.viewtype == Viewtype::Webxdc {
                 (msg.timestamp_sort, parent, true)
+            } else if parent.download_state() != DownloadState::Done {
+                (msg.timestamp_sort, parent, false)
             } else {
                 bail!("receive_status_update: message is not the child of a webxdc message.")
             }
@@ -730,7 +735,7 @@ mod tests {
     use crate::chatlist::Chatlist;
     use crate::config::Config;
     use crate::contact::Contact;
-    use crate::receive_imf::receive_imf;
+    use crate::receive_imf::{receive_imf, receive_imf_inner};
     use crate::test_utils::TestContext;
 
     use super::*;
@@ -1038,6 +1043,71 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
+        // Alice sends a larger instance and an update
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+        bob.set_config(Config::DownloadLimit, Some("40000")).await?;
+        let mut alice_instance = create_webxdc_instance(
+            &alice,
+            "chess.xdc",
+            include_bytes!("../test-data/webxdc/chess.xdc"),
+        )
+        .await?;
+        let sent1 = alice.send_msg(chat.id, &mut alice_instance).await;
+        let alice_instance = Message::load_from_db(&alice, sent1.sender_msg_id).await?;
+        alice
+            .send_webxdc_status_update(
+                alice_instance.id,
+                r#"{"payload": 7, "summary":"sum", "document":"doc"}"#,
+                "bla",
+            )
+            .await?;
+        alice.flush_status_updates().await?;
+        let sent2 = alice.pop_sent_msg().await;
+
+        // Bob does not download instance but already receives update
+        receive_imf_inner(
+            &bob,
+            &alice_instance.rfc724_mid,
+            sent1.payload().as_bytes(),
+            false,
+            Some(70790),
+            false,
+        )
+        .await?;
+        let bob_instance = bob.get_last_msg().await;
+        bob_instance.chat_id.accept(&bob).await?;
+        bob.recv_msg(&sent2).await;
+        assert_eq!(bob_instance.download_state, DownloadState::Available);
+
+        // Bob downloads instance, updates should be assigned correctly
+        receive_imf_inner(
+            &bob,
+            &alice_instance.rfc724_mid,
+            sent1.payload().as_bytes(),
+            false,
+            None,
+            false,
+        )
+        .await?;
+        let bob_instance = bob.get_last_msg().await;
+        assert_eq!(bob_instance.viewtype, Viewtype::Webxdc);
+        assert_eq!(bob_instance.download_state, DownloadState::Done);
+        assert_eq!(
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"#
+        );
+        let info = bob_instance.get_webxdc_info(&bob).await?;
+        assert_eq!(info.document, "doc");
+        assert_eq!(info.summary, "sum");
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_webxdc_instance() -> Result<()> {
         let t = TestContext::new_alice().await;

test-data/message/dsn_relayed.eml

@@ -0,0 +1,109 @@
+Return-Path: <>
+Delivered-To: anon_1@posteo.de
+Received: from proxy02.posteo.name ([127.0.0.1])
+	by dovecot16.posteo.name (Dovecot) with LMTP id 4LJTJKBpxGClSAAAchYRkQ
+	for <anon_1@posteo.de>; Sat, 12 Jun 2021 10:42:09 +0200
+Received: from proxy02.posteo.de ([127.0.0.1])
+	by proxy02.posteo.name (Dovecot) with LMTP id 0NENMHNXxGDI4AIAGFAyLg
+	; Sat, 12 Jun 2021 10:42:09 +0200
+Received: from mailin02.posteo.de (unknown [10.0.0.62])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 4G2B686dbVz11xc
+	for <anon_1@posteo.de>; Sat, 12 Jun 2021 10:42:08 +0200 (CEST)
+Received: from mx01.posteo.de (mailin02.posteo.de [127.0.0.1])
+	by mailin02.posteo.de (Postfix) with ESMTPS id AC2472152F
+	for <anon_1@posteo.de>; Sat, 12 Jun 2021 10:42:08 +0200 (CEST)
+X-Virus-Scanned: amavisd-new at posteo.de
+X-Spam-Flag: NO
+X-Spam-Score: -1
+X-Spam-Level: 
+X-Spam-Status: No, score=-1 tagged_above=-1000 required=7
+	tests=[ALL_TRUSTED=-1] autolearn=disabled
+X-Posteo-Antispam-Signature: v=1; e=base64; a=aes-256-gcm; d=7/8PYiypR3F6lmk8rQGNxZgmuPRJI9wU2IwnCWX1fg/nFdbPrDu9pCFSVsnrK1SjAWJJ9HtJVYECbeMxMhq9tOMxZf1nSN2cM/XXzeH6ELaaQfOWfQbBff3ZIe+rix/CF1uWX164
+Authentication-Results: posteo.de; dmarc=none (p=none dis=none) header.from=mout02.posteo.de
+X-Posteo-TLS-Received-Status: TLSv1.3
+Received: from mout02.posteo.de (mout02.posteo.de [185.67.36.66])
+	by mx01.posteo.de (Postfix) with ESMTPS id 4G2B676wGBz10Wt
+	for <anon_1@posteo.at>; Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+Received: by mout02.posteo.de (Postfix)
+	id A9F481A0089; Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+Date: Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+From: MAILER-DAEMON@mout02.posteo.de (Mail Delivery System)
+Subject: Successful Mail Delivery Report
+To: anon_1@posteo.at
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="56E6D1A007F.1623487327/mout02.posteo.de"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20210612084207.A9F481A0089@mout02.posteo.de>
+
+This is a MIME-encapsulated message.
+
+--56E6D1A007F.1623487327/mout02.posteo.de
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host mout02.posteo.de.
+
+Your message was successfully delivered to the destination(s)
+listed below. If the message was delivered to mailbox you will
+receive no further notifications. Otherwise you may still receive
+notifications of mail delivery errors from other systems.
+
+                   The mail system
+
+<anon_2@gmx.at>: delivery via mx00.emig.gmx.net[212.227.15.9]:25: 250
+    Requested mail action okay, completed: id=1M9ohD-1lvXys2NFd-005r3O
+
+--56E6D1A007F.1623487327/mout02.posteo.de
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mout02.posteo.de
+X-Postfix-Queue-ID: 56E6D1A007F
+X-Postfix-Sender: rfc822; anon_1@posteo.at
+Arrival-Date: Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+
+Final-Recipient: rfc822; anon_2@gmx.at
+Original-Recipient: rfc822;anon_2@gmx.at
+Action: relayed
+Status: 2.0.0
+Remote-MTA: dns; mx00.emig.gmx.net
+Diagnostic-Code: smtp; 250 Requested mail action okay, completed:
+    id=1M9ohD-1lvXys2NFd-005r3O
+
+--56E6D1A007F.1623487327/mout02.posteo.de
+Content-Description: Message Headers
+Content-Type: text/rfc822-headers
+
+Return-Path: <anon_1@posteo.at>
+Received: from mout02.posteo.de (unknown [10.0.0.66])
+	by mout02.posteo.de (Postfix) with ESMTPS id 56E6D1A007F
+	for <anon_2@gmx.at>; Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+Received: from submission-encrypt01.posteo.de (unknown [10.0.0.75])
+	by mout02.posteo.de (Postfix) with ESMTPS id 1C39E2400FD
+	for <anon_2@gmx.at>; Sat, 12 Jun 2021 10:42:07 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.at; s=2017;
+	t=1623487327; bh=+ZIKEoFCh8N5xYBj6tMbfqiyHmay76uM4H4bfme6VyU=;
+	h=Date:From:To:Subject:From;
+	b=QK6HwDU2YEzzTgHN2PRT2lPaf5uwC7ZJ1Y0QMSUrEyvJxwPj6+z6OoEqRDcgQcGVo
+	 biAO2aKyBX+YCFwM5a6CaJotv8DaL+hn/XLk3RKqxGKTu5cBLQXJc0gjfRMel7LnBg
+	 i0UxTeOqoTw2anWTonH2GnseUPtVAhi23UICVD6gC6DchuNYF/YloMltns5HMGthQh
+	 z279J05txneSKgpbU/R3fN2v5ACEve7X6GoxM0hDZRNmAur0HAxAREc9xIaHwQ3zXM
+	 dEGFyO53s+UzLlOFnY4vhGVI3AiyOZUProq6vX40g9e4TkrIJMGd1pyKG4NdajauuY
+	 KTIwbUiR5Y2Xw==
+Received: from customer (localhost [127.0.0.1])
+	by submission (posteo.de) with ESMTPSA id 4G2B665xBPz6tmH
+	for <anon_2@gmx.at>; Sat, 12 Jun 2021 10:42:06 +0200 (CEST)
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="=_d0190a7dc3b70a1dcf12785779aad292"
+Date: Sat, 12 Jun 2021 08:42:06 +0000
+From: Anon_1 <anon_1@posteo.at>
+To: Anon_2 <anon_2@gmx.at>
+Subject: Hallo
+Message-ID: <8b7b1a9d0c8cc588c7bcac47f5687634@posteo.de>
+Posteo-User: anon_1@posteo.de
+Posteo-Dkim: ok
+
+--56E6D1A007F.1623487327/mout02.posteo.de--

test-data/message/ndn_with_attachment.eml

@@ -0,0 +1,123 @@
+Return-Path: <>
+Delivered-To: <alice@example.org>
+Date: Sat, 25 Jun 2022 21:35:33 -0400 (CDT)
+From: MAILER-DAEMON@example.org (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@example.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="C06EAE01B0.1656207333/relay01.example.org"
+Content-Transfer-Encoding: 7bit
+Message-Id: <20220626013533.5F0E5E0180@relay01.example.org>
+
+This is a MIME-encapsulated message.
+
+--C06EAE01B0.1656207333/relay01.example.org
+Content-Description: Notification
+Content-Type: text/plain; charset=us-ascii
+
+This is the mail system at host relay01.example.org.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<bob@example.net>: host mx2.example.net[80.241.60.215] said: 552 5.2.2
+    <bob@example.net>: Recipient address rejected: Mailbox quota exceeded (in
+    reply to RCPT TO command)
+
+<bob2@example.net>: host mx1.example.net[80.241.60.212] said: 552 5.2.2
+    <bob2@example.net>: Recipient address rejected: Mailbox quota
+    exceeded (in reply to RCPT TO command)
+
+--C06EAE01B0.1656207333/relay01.example.org
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; relay01.example.org
+X-Postfix-Queue-ID: C06EAE01B0
+X-Postfix-Sender: rfc822; alice@example.org
+Arrival-Date: Sat, 25 Jun 2022 21:35:23 -0400 (CDT)
+
+Final-Recipient: rfc822; bob@example.org
+Original-Recipient: rfc822;bob@example.org
+Action: failed
+Status: 5.2.2
+Remote-MTA: dns; mx2.example.net
+Diagnostic-Code: smtp; 552 5.2.2 <bob@example.org>: Recipient address
+    rejected: Mailbox quota exceeded
+
+Final-Recipient: rfc822; bob2@example.net
+Original-Recipient: rfc822;bob2@example.net
+Action: failed
+Status: 5.2.2
+Remote-MTA: dns; mx1.example.net
+Diagnostic-Code: smtp; 552 5.2.2 <bob2@example.net>: Recipient address
+    rejected: Mailbox quota exceeded
+
+--C06EAE01B0.1656207333/relay01.example.org
+Content-Description: Undelivered Message
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+
+Return-Path: <alice@example.org>
+Received: from smtp-gw01.enet.cu (unknown [172.29.8.31])
+	by relay01.example.org (Postfix) with ESMTP id C06EAE01B0;
+	Sat, 25 Jun 2022 21:35:23 -0400 (CDT)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.org; s=nauta;
+	t=1656207323; bh=c0t8muX/2spIAQ/MRxSXsZ/ljSWP8uTxPcM5fqYC6qg=;
+	h=Subject:From:To:Date:In-Reply-To:References;
+	b=fggUmxnR3RjNfxO4S460fvxfenu3eUPvsGjldIGQeApxT508qGctJZ0tzYiFelOAu
+	 4zL8+VhlX0KdnrvRXo2/00O9U2kM8gpRLE9tIR0Rn6FyWM4nU5rcvAJp3oiUWxYzPt
+	 NMlzFeUsBvwFCJCEmZllSHaAyt/HTvWe68sGQsu8=
+Received: from amavis4.example.org (unknown [172.29.8.235])
+	by smtp-gw01.enet.cu (Postfix) with ESMTP id ECFE9180004D;
+	Sat, 25 Jun 2022 21:35:18 -0400 (CDT)
+Received: from smtp.example.org ([172.29.8.248])
+	by amavis4.example.org  with ESMTP id 25Q1ZNDW016368-25Q1ZNDX016368;
+	Sat, 25 Jun 2022 21:35:23 -0400
+Received: from [127.0.0.1] (unknown [10.59.196.114])
+	by smtp.example.org (Postfix) with ESMTPSA id E07E79F110;
+	Sat, 25 Jun 2022 21:35:21 -0400 (CDT)
+Subject: ...
+From: Alice <alice@example.org>
+To: <bob@example.net>, <bob2@example.net>
+Date: Sun, 26 Jun 2022 01:35:20 +0000
+Message-ID: <Mr.I6Da6dXcTel.TroC5J3uSDH@example.org>
+In-Reply-To: <Mr.I6Da6dXcTel.-qoK1IWpqby@example.org>
+References: <Mr.I6Da6dXcTel.TroC5J3uSDH@example.org>
+Chat-Version: 1.0
+MIME-Version: 1.0
+Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
+	boundary="OOGjcgxCkv3w8d1thY6jg0b8O0NFJE"
+
+
+--OOGjcgxCkv3w8d1thY6jg0b8O0NFJE
+Content-Type: application/pgp-encrypted
+Content-Description: PGP/MIME version identification
+
+Version: 1
+
+
+--OOGjcgxCkv3w8d1thY6jg0b8O0NFJE
+Content-Type: application/octet-stream; name="encrypted.asc"
+Content-Description: OpenPGP encrypted message
+Content-Disposition: inline; filename="encrypted.asc";
+
+-----BEGIN PGP MESSAGE-----
+
+REMOVED
+-----END PGP MESSAGE-----
+
+
+--OOGjcgxCkv3w8d1thY6jg0b8O0NFJE--
+
+
+--C06EAE01B0.1656207333/relay01.example.org--