Introduce config caching

Don't count the account creation in the receive emails benchmark

Use Criterion's async support

See https://bheisler.github.io/criterion.rs/book/user_guide/benchmarking_async.html

.github/workflows/ci.yml

@@ -150,4 +150,4 @@ jobs:
         DCC_RS_TARGET: debug
         DCC_RS_DEV: ${{ github.workspace }}
       working-directory: python
-      run: tox -e lint,doc,py3
+      run: tox -e lint,mypy,doc,py3

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.65.0
+
+### Changes
+- python: add mypy support and some type hints #2809
+
+### Fixes
+- do not disable ephemeral timer when downloading a message partially #2811
+- apply existing ephemeral timer also to partially downloaded messages;
+  after full download, the ephemeral timer starts over #2811
+- replace user-visible error on verification failure with warning;
+  the error is logged to the corresponding chat anyway #2808
+
+
 ## 1.64.0
 
 ### Fixes
@@ -50,10 +63,10 @@
 ## 1.61.0
 
 ### API Changes
-- download-on-demand added: `dc_msg_get_download_status()`, `dc_download_full_msg()`
+- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
   and `download_limit` config option #2631 #2696
 - `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs: `dc_set_ui_config()` and `dc_get_ui_config()` #2672
+- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
 - new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
   to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
 - emit warnings and errors from account manager with account-id 0 #2712

Cargo.lock

@@ -847,11 +847,13 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
 dependencies = [
+ "async-std",
  "atty",
  "cast",
  "clap",
  "criterion-plot",
  "csv",
+ "futures",
  "itertools",
  "lazy_static",
  "num-traits",
@@ -1072,7 +1074,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.64.0"
+version = "1.65.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1152,7 +1154,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.64.0"
+version = "1.65.0"
 dependencies = [
  "anyhow",
  "async-std",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.64.0"
+version = "1.65.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -79,7 +79,7 @@ humansize = "1"
 [dev-dependencies]
 ansi_term = "0.12.0"
 async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
+criterion = { version = "0.3.4", features = ["async_std"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_assertions = "1.0"
@@ -116,6 +116,10 @@ harness = false
 name = "search_msgs"
 harness = false
 
+[[bench]]
+name = "receive_emails"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []

benches/create_account.rs

@@ -8,7 +8,7 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts").into();
 
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+    let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

benches/receive_emails.rs

@@ -0,0 +1,117 @@
+use std::convert::TryInto;
+
+use async_std::{path::PathBuf, task::block_on};
+use criterion::{
+    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
+    BenchmarkId, Criterion,
+};
+use deltachat::{
+    config::Config,
+    context::Context,
+    dc_receive_imf::dc_receive_imf,
+    imex::{imex, ImexMode},
+};
+use tempfile::tempdir;
+
+async fn recv_emails(context: Context, emails: &[&[u8]]) -> Context {
+    for (i, bytes) in emails.iter().enumerate() {
+        dc_receive_imf(
+            &context,
+            bytes,
+            "INBOX",
+            black_box(i.try_into().unwrap()),
+            false,
+        )
+        .await
+        .unwrap();
+    }
+    context
+}
+
+async fn recv_all_emails(mut context: Context, needs_move_enabled: bool) -> Context {
+    context.disable_needs_move = !needs_move_enabled;
+
+    for i in 0..100 {
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        dc_receive_imf(
+            &context,
+            imf_raw.as_bytes(),
+            "INBOX",
+            black_box(i.try_into().unwrap()),
+            false,
+        )
+        .await
+        .unwrap();
+    }
+    context
+}
+
+async fn create_context() -> Context {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    let backup: PathBuf = std::env::current_dir()
+        .unwrap()
+        .join("delta-chat-backup.tar")
+        .into();
+    if backup.exists().await {
+        println!("Importing backup");
+        imex(&context, ImexMode::ImportBackup, &backup)
+            .await
+            .unwrap();
+    }
+
+    let addr = "alice@example.com";
+    context.set_config(Config::Addr, Some(addr)).await.unwrap();
+    context
+        .set_config(Config::ConfiguredAddr, Some(addr))
+        .await
+        .unwrap();
+    context
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    context
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("from_elem");
+    for needs_move_enabled in [false, true] {
+        group.bench_with_input(
+            BenchmarkId::new("Receive many messages", needs_move_enabled),
+            &needs_move_enabled,
+            |b, needs_move_enabled| {
+                b.to_async(AsyncStdExecutor).iter_batched(
+                    || block_on(create_context()),
+                    |context| recv_all_emails(black_box(context), *needs_move_enabled),
+                    BatchSize::LargeInput,
+                );
+            },
+        );
+    }
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.64.0"
+version = "1.65.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

python/mypy.ini

@@ -0,0 +1,19 @@
+[mypy]
+
+[mypy-deltachat.capi.*]
+ignore_missing_imports = True
+
+[mypy-pluggy.*]
+ignore_missing_imports = True
+
+[mypy-cffi.*]
+ignore_missing_imports = True
+
+[mypy-imapclient.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-_pytest.*]
+ignore_missing_imports = True

python/pyproject.toml

@@ -1,5 +1,6 @@
 [build-system]
 requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
 root = ".."

python/setup.py

@@ -12,6 +12,7 @@ def main():
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
         install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name, val in vars(const).items():
+        for name in dir(const):
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[val] = name
+                _DC_EVENTNAME_MAP[getattr(const, name)] = name
     return _DC_EVENTNAME_MAP[integer]
 
 

python/src/deltachat/account.py

@@ -15,6 +15,7 @@ from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
+from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
@@ -28,7 +29,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, os_name=None, logging=True) -> None:
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -58,11 +59,11 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self):
+    def disable_logging(self) -> None:
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self):
+    def enable_logging(self) -> None:
         """ re-enable logging. """
         self._logging = True
 
@@ -73,7 +74,7 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name):
+    def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
@@ -105,19 +106,19 @@ class Account(object):
             cursor += len(entry) + 1
         log("")
 
-    def set_stock_translation(self, id, string):
+    def set_stock_translation(self, id: int, string: str) -> None:
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        string = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        bytestring = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name, value):
+    def set_config(self, name: str, value: Optional[str]) -> None:
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -125,16 +126,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        name = name.encode("utf8")
-        if name == b"addr" and self.is_configured():
+        namebytes = name.encode("utf8")
+        if namebytes == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            value = value.encode("utf8")
+            valuebytes = value.encode("utf8")
         else:
-            value = ffi.NULL
-        lib.dc_set_config(self._dc_context, name, value)
+            valuebytes = ffi.NULL
+        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
-    def get_config(self, name):
+    def get_config(self, name: str):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -143,12 +144,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        name = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, name)
+        namebytes = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, namebytes)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
+    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -160,7 +161,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs):
+    def update_config(self, kwargs: Dict[str, Any]) -> None:
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -170,7 +171,7 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self):
+    def is_configured(self) -> bool:
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -178,7 +179,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def set_avatar(self, img_path):
+    def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -190,12 +191,12 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self):
+    def check_is_configured(self) -> None:
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir):
+    def get_latest_backupfile(self, backupdir) -> Optional[str]:
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
@@ -203,7 +204,7 @@ class Account(object):
             return None
         return from_dc_charpointer(res)
 
-    def get_blobdir(self):
+    def get_blobdir(self) -> Optional[str]:
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -211,15 +212,15 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self):
+    def get_self_contact(self) -> Contact:
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
+        """create a (new) Contact or return an existing one.
 
         Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
@@ -236,13 +237,13 @@ class Account(object):
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
-    def get_contact(self, obj):
+    def get_contact(self, obj) -> Optional[Contact]:
         if isinstance(obj, Contact):
             return obj
         (_, addr) = self.get_contact_addr_and_name(obj)
         return self.get_contact_by_addr(addr)
 
-    def get_contact_addr_and_name(self, obj, name=None):
+    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -260,7 +261,7 @@ class Account(object):
             name = displayname
         return (name, addr)
 
-    def delete_contact(self, contact):
+    def delete_contact(self, contact: Contact) -> bool:
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -271,22 +272,23 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
+    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
+        return None
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
+    def get_contact_by_id(self, contact_id: int) -> Contact:
+        """ return Contact instance or raise an exception.
         :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self):
+    def get_blocked_contacts(self) -> List[Contact]:
         """ return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
@@ -297,8 +299,13 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_contacts(self, query=None, with_self=False, only_verified=False):
-        """ get a (filtered) list of contacts.
+    def get_contacts(
+        self,
+        query: Optional[str] = None,
+        with_self: bool = False,
+        only_verified: bool = False,
+    ) -> List[Contact]:
+        """get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -318,7 +325,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self):
+    def get_fresh_messages(self) -> Generator[Message, None, None]:
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -326,12 +333,17 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
+    def create_chat(self, obj) -> Chat:
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(self, name, contacts=None, verified=False):
-        """ create a new group chat object.
+    def create_group_chat(
+        self,
+        name: str,
+        contacts: Optional[List[Contact]] = None,
+        verified: bool = False,
+    ) -> Chat:
+        """create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -347,7 +359,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self):
+    def get_chats(self) -> List[Chat]:
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -364,17 +376,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self):
+    def get_device_chat(self) -> Chat:
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id):
+    def get_message_by_id(self, msg_id: int) -> Message:
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id):
+    def get_chat_by_id(self, chat_id: int) -> Chat:
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -386,19 +398,18 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages):
+    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            msg = getattr(msg, "id", msg)
-            arr.append(msg)
+            arr.append(getattr(msg, "id", msg))
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages, chat):
+    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -408,7 +419,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages):
+    def delete_messages(self, messages: List[Message]) -> None:
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -477,7 +488,7 @@ class Account(object):
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self):
+    def get_setup_contact_qr(self) -> Optional[str]:
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -527,7 +538,9 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+    def set_location(
+        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
+    ) -> None:
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -621,7 +634,7 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure=False):
+    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
@@ -634,11 +647,11 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self):
+    def wait_shutdown(self) -> None:
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self):
+    def stop_io(self) -> None:
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
@@ -646,7 +659,7 @@ class Account(object):
         self.log("dc_stop_io (stop core IO scheduler)")
         lib.dc_stop_io(self._dc_context)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:

python/src/deltachat/chat.py

@@ -9,6 +9,7 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
+from typing import Optional
 
 
 class Chat(object):
@@ -17,20 +18,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id):
+    def __init__(self, account, id) -> None:
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         return not (self == other)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -40,7 +41,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self):
+    def delete(self) -> None:
         """Delete this chat and all its messages.
 
         Note:
@@ -50,24 +51,24 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self):
+    def block(self) -> None:
         """Block this chat."""
         lib.dc_block_chat(self.account._dc_context, self.id)
 
-    def accept(self):
+    def accept(self) -> None:
         """Accept this contact request chat."""
         lib.dc_accept_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
+    def is_group(self) -> bool:
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_muted(self):
+    def is_muted(self) -> bool:
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
@@ -90,7 +91,7 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self):
+    def can_send(self) -> bool:
         """Check if messages can be sent to a give chat.
         This is not true eg. for the contact requests or for the device-talk
 
@@ -98,30 +99,30 @@ class Chat(object):
         """
         return lib.dc_chat_can_send(self._dc_chat)
 
-    def is_protected(self):
+    def is_protected(self) -> bool:
         """ return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
         return lib.dc_chat_is_protected(self._dc_chat)
 
-    def get_name(self):
+    def get_name(self) -> Optional[str]:
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name):
+    def set_name(self, name: str) -> bool:
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: None
+        :returns: True on success, False otherwise
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
+        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
-    def mute(self, duration=None):
+    def mute(self, duration: Optional[int] = None) -> None:
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -135,7 +136,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self):
+    def unmute(self) -> None:
         """ unmutes the chat
 
         :returns: None
@@ -144,7 +145,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self):
+    def get_mute_duration(self) -> int:
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
@@ -152,37 +153,37 @@ class Chat(object):
         """
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
-    def get_ephemeral_timer(self):
+    def get_ephemeral_timer(self) -> int:
         """ get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
-    def set_ephemeral_timer(self, timer):
+    def set_ephemeral_timer(self, timer: int) -> bool:
         """ set ephemeral timer.
 
         :param: timer value in seconds
 
-        :returns: None
+        :returns: True on success, False otherwise
         """
-        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
 
-    def get_type(self):
+    def get_type(self) -> int:
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self):
+    def get_encryption_info(self) -> Optional[str]:
         """Return encryption info for this chat.
 
         :returns: a string with encryption preferences of all chat members"""
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
-    def get_join_qr(self):
+    def get_join_qr(self) -> Optional[str]:
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -194,7 +195,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg):
+    def send_msg(self, msg: Message) -> Message:
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance

python/src/deltachat/const.py

@@ -1,7 +1,13 @@
+from typing import Any, List
+
 from .capi import lib
 
 
-for name in dir(lib):
+def __getattr__(name: str) -> Any:
     if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+        return getattr(lib, name)
+    return globals()[name]
+
+
+def __dir__() -> List[str]:
+    return sorted(name for name in dir(lib) if name.startswith("DC_"))

python/src/deltachat/cutil.py

@@ -1,6 +1,9 @@
 from .capi import lib
 from .capi import ffi
 from datetime import datetime, timezone
+from typing import Optional, TypeVar, Generator, Callable
+
+T = TypeVar('T')
 
 
 def as_dc_charpointer(obj):
@@ -11,21 +14,22 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor):
+def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj):
+def from_dc_charpointer(obj) -> Optional[str]:
     if obj != ffi.NULL:
         return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    return None
 
 
 class DCLot:
-    def __init__(self, dc_lot):
+    def __init__(self, dc_lot) -> None:
         self._dc_lot = dc_lot
 
-    def id(self):
+    def id(self) -> int:
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):

python/src/deltachat/direct_imap.py

@@ -11,7 +11,7 @@ from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
 import imaplib
 import deltachat
-from deltachat import const
+from deltachat import const, Account
 
 
 SEEN = b'\\Seen'
@@ -62,7 +62,7 @@ def dc_account_after_shutdown(account):
 
 
 class DirectImap:
-    def __init__(self, account):
+    def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False

python/src/deltachat/events.py

@@ -13,7 +13,7 @@ from .cutil import from_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name, data1, data2):
+    def __init__(self, name: str, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,13 +29,13 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
         self.account.log(str(ffi_event))
 
     @account_hookimpl
@@ -69,7 +69,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -96,7 +96,7 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex):
+    def get_info_contains(self, regex: str) -> FFIEvent:
         rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
@@ -176,6 +176,7 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
+        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -189,7 +190,7 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
@@ -202,17 +203,17 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self):
+    def mark_shutdown(self) -> None:
         self._marked_for_shutdown = True
 
-    def wait(self, timeout=None):
+    def wait(self, timeout=None) -> None:
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
         self.join(timeout=timeout)
 
-    def run(self):
+    def run(self) -> None:
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -250,7 +251,7 @@ class EventThread(threading.Thread):
                 if self.account._dc_context is not None:
                     raise
 
-    def _map_ffi_event(self, ffi_event):
+    def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, account, addr) -> None:
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,

python/src/deltachat/testplugin.py

@@ -9,6 +9,7 @@ import fnmatch
 import time
 import weakref
 import tempfile
+from typing import List, Dict, Callable
 
 import pytest
 import requests
@@ -126,7 +127,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn):
+    def __init__(self, fn) -> None:
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -137,19 +138,21 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index):
+    def get(self, index: int):
         return self.configlist[index]
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    def __init__(self, url):
+    configlist: List[Dict[str, str]]
+
+    def __init__(self, url: str) -> None:
         self.configlist = []
         self.url = url
 
-    def get(self, index):
+    def get(self, index: int):
         try:
             return self.configlist[index]
         except IndexError:
@@ -162,7 +165,7 @@ class SessionLiveConfigFromURL:
             self.configlist.append(config)
             return config
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
@@ -179,7 +182,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self):
+        def __init__(self) -> None:
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -210,7 +213,10 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        def __init__(self):
+        _finalizers: List[Callable[[], None]]
+        _accounts: List[Account]
+
+        def __init__(self) -> None:
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
@@ -423,7 +429,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                         pass
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        def get_accepted_chat(self, ac1, ac2):
+        def get_accepted_chat(self, ac1: Account, ac2: Account):
             ac2.create_chat(ac1)
             return ac1.create_chat(ac2)
 
@@ -451,7 +457,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
 
 class BotProcess:
-    def __init__(self, popen, bot_cfg):
+    stdout_queue: queue.Queue
+
+    def __init__(self, popen, bot_cfg) -> None:
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -459,10 +467,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
+        t.setDaemon(True)
         t.start()
 
-    def _run_stdout_thread(self):
+    def _run_stdout_thread(self) -> None:
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -474,10 +482,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self):
+    def kill(self) -> None:
         self.popen.kill()
 
-    def wait(self, timeout=30):
+    def wait(self, timeout=30) -> None:
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -509,14 +517,14 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg):
+        def sec(self, msg: str) -> None:
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg):
+        def step(self, msg: str) -> None:
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg):
+        def indent(self, msg: str) -> None:
             print("  " + msg)
 
     return Printer()

python/tox.ini

@@ -1,7 +1,9 @@
 [tox]
+isolated_build = true
 envlist =
     py3
     lint
+    mypy
     auditwheels
 
 [testenv]
@@ -42,6 +44,15 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
+[testenv:mypy]
+deps =
+    mypy
+    typing
+    types-setuptools
+    types-requests
+commands =
+    mypy --no-incremental src/
+
 [testenv:doc]
 changedir=doc
 deps =

src/context.rs

@@ -30,6 +30,7 @@ use crate::sql::Sql;
 #[derive(Clone, Debug)]
 pub struct Context {
     pub(crate) inner: Arc<InnerContext>,
+    pub disable_needs_move: bool, // TODO just added for some extra profiling info
 }
 
 impl Deref for Context {
@@ -157,6 +158,7 @@ impl Context {
 
         let ctx = Context {
             inner: Arc::new(inner),
+            disable_needs_move: false,
         };
         ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 

src/dc_receive_imf.rs

@@ -193,7 +193,7 @@ pub(crate) async fn dc_receive_imf_inner(
     }
 
     // Add parts
-    let chat_id = add_parts(
+    let (chat_id, msgs) = add_parts(
         context,
         &mut mime_parser,
         imf_raw,
@@ -323,7 +323,11 @@ pub(crate) async fn dc_receive_imf_inner(
                 )
                 .await?;
             }
-        } else if insert_msg_id
+        } else if msgs
+            .first()
+            .unwrap() // TODO unwrap() (should be safe though)
+            .clone() // TODO unnecessary clone
+            .into_lazy()
             .needs_move(context, server_folder.as_ref())
             .await
             .unwrap_or_default()
@@ -439,7 +443,7 @@ async fn add_parts(
     create_event_to_send: &mut Option<CreateEvent>,
     fetching_existing_messages: bool,
     prevent_rename: bool,
-) -> Result<ChatId> {
+) -> Result<(ChatId, Vec<Message>)> {
     let mut chat_id = None;
     let mut chat_id_blocked = Blocked::Not;
     let mut incoming_origin = incoming_origin;
@@ -513,7 +517,7 @@ async fn add_parts(
                 }
                 Err(err) => {
                     warn!(context, "Error in Secure-Join message handling: {}", err);
-                    return Ok(DC_CHAT_ID_TRASH);
+                    return Ok((DC_CHAT_ID_TRASH, Vec::new()));
                 }
             }
         } else {
@@ -740,7 +744,7 @@ async fn add_parts(
                 }
                 Err(err) => {
                     warn!(context, "Error in Secure-Join watching: {}", err);
-                    return Ok(DC_CHAT_ID_TRASH);
+                    return Ok((DC_CHAT_ID_TRASH, Vec::new()));
                 }
             }
         } else if mime_parser.sync_items.is_some() && self_sent {
@@ -864,9 +868,10 @@ async fn add_parts(
         DC_CHAT_ID_TRASH
     });
 
-    // Extract ephemeral timer from the message.
-    let mut ephemeral_timer = if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer)
-    {
+    // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded.
+    let mut ephemeral_timer = if is_partial_download.is_some() {
+        chat_id.get_ephemeral_timer(context).await?
+    } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) {
         match value.parse::<EphemeralTimer>() {
             Ok(timer) => timer,
             Err(err) => {
@@ -1004,7 +1009,7 @@ async fn add_parts(
                                 sort_timestamp,
                             )
                             .await?;
-                            return Ok(chat_id); // do not return an error as this would result in retrying the message
+                            return Ok((chat_id, Vec::new())); // do not return an error as this would result in retrying the message
                         }
                     }
                     set_better_msg(
@@ -1075,6 +1080,7 @@ async fn add_parts(
     let mut ids = Vec::with_capacity(parts.len());
 
     let conn = context.sql.get_conn().await?;
+    let mut msgs = Vec::new();
 
     for part in &mut parts {
         let mut txt_raw = "".to_string();
@@ -1132,6 +1138,53 @@ INSERT INTO msgs
         // also change `MsgId::trash()` and `delete_expired_messages()`
         let trash = chat_id.is_trash();
 
+        let msg = Message {
+            id: MsgId::new(0),
+            rfc724_mid: rfc724_mid.to_string(),
+            server_uid: server_uid,
+            chat_id,
+            from_id: if trash { 0 } else { from_id },
+            to_id: if trash { 0 } else { to_id },
+            timestamp_sort: sort_timestamp,
+            timestamp_sent: sent_timestamp,
+            timestamp_rcvd: rcvd_timestamp,
+            viewtype: part.typ,
+            state,
+            is_dc_message,
+            text: if trash {
+                None
+            } else {
+                Some(part.msg.to_string())
+            },
+            subject: if trash {
+                "".to_string()
+            } else {
+                subject.to_string()
+            },
+            param: if trash {
+                Params::new()
+            } else {
+                part.param.clone()
+            },
+            in_reply_to: Some(mime_in_reply_to.to_string()), // TODO be careful with Some("") and None
+            mime_modified,
+            error: part.error.clone(),
+            ephemeral_timer,
+            ephemeral_timestamp,
+            download_state: if is_partial_download.is_some() {
+                DownloadState::Available
+            } else {
+                DownloadState::Done
+            },
+            hidden: false,
+            chat_blocked: chat_id_blocked, // TODO not sure if correct
+            location_id: 0,
+            server_folder: Some(server_folder.to_string()),
+        };
+        // TODO mabye reuse some of the fields from above in the query below to avoid duplicate code
+
+        msgs.push(msg);
+
         stmt.execute(paramsv![
             rfc724_mid,
             server_folder,
@@ -1222,7 +1275,7 @@ INSERT INTO msgs
         }
     }
 
-    Ok(chat_id)
+    Ok((chat_id, msgs))
 }
 
 /// Saves attached locations to the database.

src/download.rs

@@ -227,6 +227,7 @@ mod tests {
     use crate::chat::send_msg;
     use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf_inner;
+    use crate::ephemeral::Timer;
     use crate::test_utils::TestContext;
     use num_traits::FromPrimitive;
 
@@ -343,4 +344,40 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_partial_download_and_ephemeral() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = t
+            .create_chat_with_contact("bob", "bob@example.org")
+            .await
+            .id;
+        chat_id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        // download message from bob partially, this must not change the ephemeral timer
+        dc_receive_imf_inner(
+            &t,
+            b"From: Bob <bob@example.org>\n\
+                    To: Alice <alice@example.com>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: subject\n\
+                    Message-ID: <first@example.org>\n\
+                    Date: Sun, 14 Nov 2021 00:10:00 +0000\
+                    Content-Type: text/plain",
+            "INBOX",
+            1,
+            false,
+            Some(100000),
+            false,
+        )
+        .await?;
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&t).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
 }

src/imap.rs

@@ -361,6 +361,7 @@ impl Imap {
 
                 let lock = context.wrong_pw_warning_mutex.lock().await;
                 if self.login_failed_once
+                    && err.to_string().to_lowercase().contains("authentication")
                     && context.get_config_bool(Config::NotifyAboutWrongPw).await?
                 {
                     if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
@@ -1597,6 +1598,7 @@ async fn precheck_imf(
 
             if delete_server_after != Some(0) {
                 if msg_id
+                    .to_lazy()
                     .needs_move(context, server_folder)
                     .await
                     .unwrap_or_default()

src/job.rs

@@ -561,11 +561,16 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let server_folder = &job_try!(msg
+        let server_folder = job_try!(msg
             .server_folder
+            .as_ref()
             .context("Can't move message out of folder if we don't know the current folder"));
 
-        let move_res = msg.id.needs_move(context, server_folder).await;
+        let move_res = msg
+            .clone() // TODO avoid clone()?
+            .into_lazy()
+            .needs_move(context, &server_folder)
+            .await;
         let dest_folder = match move_res {
             Err(e) => {
                 warn!(context, "could not load dest folder: {}", e);
@@ -589,7 +594,7 @@ impl Job {
 
         if let Some(dest_folder) = dest_folder {
             match imap
-                .mv(context, server_folder, msg.server_uid, &dest_folder)
+                .mv(context, &server_folder, msg.server_uid, &dest_folder)
                 .await
             {
                 ImapActionResult::RetryLater => Status::RetryLater,

src/message.rs

@@ -6,6 +6,7 @@ use std::convert::TryInto;
 use anyhow::{ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
+use once_cell::sync::OnceCell;
 use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
@@ -53,6 +54,13 @@ impl MsgId {
         MsgId(0)
     }
 
+    pub fn to_lazy(&self) -> LazyMsg {
+        LazyMsg {
+            id: *self,
+            message: OnceCell::new(),
+        }
+    }
+
     /// Whether the message ID signifies a special message.
     ///
     /// This kind of message ID can not be used for real messages.
@@ -83,65 +91,6 @@ impl MsgId {
         Ok(result)
     }
 
-    /// Returns Some if the message needs to be moved from `folder`.
-    /// If yes, returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder`,
-    /// depending on where the message should be moved
-    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<Option<Config>> {
-        use Config::*;
-        if context.is_mvbox(folder).await? {
-            return Ok(None);
-        }
-
-        let msg = Message::load_from_db(context, self).await?;
-
-        if context.is_spam_folder(folder).await? {
-            let msg_unblocked = msg.chat_id != DC_CHAT_ID_TRASH && msg.chat_blocked == Blocked::Not;
-
-            return if msg_unblocked {
-                if self.needs_move_to_mvbox(context, &msg).await? {
-                    Ok(Some(ConfiguredMvboxFolder))
-                } else {
-                    Ok(Some(ConfiguredInboxFolder))
-                }
-            } else {
-                // Blocked or contact request message in the spam folder, leave it there
-                Ok(None)
-            };
-        }
-
-        if self.needs_move_to_mvbox(context, &msg).await? {
-            Ok(Some(ConfiguredMvboxFolder))
-        } else if msg.state.is_outgoing()
-                && msg.is_dc_message == MessengerMessage::Yes
-                && !msg.is_setupmessage()
-                && msg.to_id != DC_CONTACT_ID_SELF // Leave self-chat-messages in the inbox, not sure about this
-                && context.is_inbox(folder).await?
-                && context.get_config_bool(SentboxMove).await?
-                && context.get_config(ConfiguredSentboxFolder).await?.is_some()
-        {
-            Ok(Some(ConfiguredSentboxFolder))
-        } else {
-            Ok(None)
-        }
-    }
-
-    async fn needs_move_to_mvbox(self, context: &Context, msg: &Message) -> Result<bool> {
-        if !context.get_config_bool(Config::MvboxMove).await? {
-            return Ok(false);
-        }
-
-        if msg.is_setupmessage() {
-            // do not move setup messages;
-            // there may be a non-delta device that wants to handle it
-            return Ok(false);
-        }
-
-        match msg.is_dc_message {
-            MessengerMessage::No => Ok(false),
-            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
-        }
-    }
-
     /// Put message into trash chat and delete message text.
     ///
     /// It means the message is deleted locally, but not on the server.
@@ -282,6 +231,90 @@ impl Default for MessengerMessage {
     }
 }
 
+#[derive(Debug)]
+pub struct LazyMsg {
+    id: MsgId,
+    message: OnceCell<Message>,
+}
+
+impl LazyMsg {
+    // TODO should get() require &mut self? Or lock a mutex during computation?
+    async fn get(&self, context: &Context) -> Result<&Message> {
+        if let Some(m) = self.message.get() {
+            Ok(m)
+        } else {
+            let m = Message::load_from_db(context, self.id).await?;
+            self.message.set(m).ok();
+            self.message.get().context("message.set() failed???")
+        }
+    }
+
+    /// Returns Some if the message needs to be moved from `folder`.
+    /// If yes, returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder`,
+    /// depending on where the message should be moved
+    pub async fn needs_move(&self, context: &Context, folder: &str) -> Result<Option<Config>> {
+        use Config::*;
+        if context.disable_needs_move {
+            return Ok(None);
+        }
+
+        if context.is_mvbox(folder).await? {
+            return Ok(None);
+        }
+
+        let msg = self.get(context).await?;
+
+        if context.is_spam_folder(folder).await? {
+            let msg_unblocked = msg.chat_id != DC_CHAT_ID_TRASH && msg.chat_blocked == Blocked::Not;
+
+            return if msg_unblocked {
+                if self.needs_move_to_mvbox(context).await? {
+                    Ok(Some(ConfiguredMvboxFolder))
+                } else {
+                    Ok(Some(ConfiguredInboxFolder))
+                }
+            } else {
+                // Blocked or contact request message in the spam folder, leave it there
+                Ok(None)
+            };
+        }
+
+        if self.needs_move_to_mvbox(context).await? {
+            Ok(Some(ConfiguredMvboxFolder))
+        } else if msg.state.is_outgoing()
+                && msg.is_dc_message == MessengerMessage::Yes
+                && !msg.is_setupmessage()
+                && msg.to_id != DC_CONTACT_ID_SELF // Leave self-chat-messages in the inbox, not sure about this
+                && context.is_inbox(folder).await?
+                && context.get_config_bool(SentboxMove).await?
+                && context.get_config(ConfiguredSentboxFolder).await?.is_some()
+        {
+            Ok(Some(ConfiguredSentboxFolder))
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn needs_move_to_mvbox(&self, context: &Context) -> Result<bool> {
+        if !context.get_config_bool(Config::MvboxMove).await? {
+            return Ok(false);
+        }
+
+        let msg = self.get(context).await?;
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
+    }
+}
+
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -426,6 +459,13 @@ impl Message {
         Ok(msg)
     }
 
+    pub fn into_lazy(self) -> LazyMsg {
+        LazyMsg {
+            id: self.id,
+            message: self.into(),
+        }
+    }
+
     pub fn get_filemime(&self) -> Option<String> {
         if let Some(m) = self.param.get(Param::MimeType) {
             return Some(m.to_string());
@@ -1990,11 +2030,12 @@ mod tests {
         let exists = rfc724_mid_exists(&t, "abc@example.com").await.unwrap();
         let (folder_1, _, msg_id) = exists.unwrap();
         assert_eq!(folder, folder_1);
-        let actual = if let Some(config) = msg_id.needs_move(&t.ctx, folder).await.unwrap() {
-            t.ctx.get_config(config).await.unwrap()
-        } else {
-            None
-        };
+        let actual =
+            if let Some(config) = msg_id.to_lazy().needs_move(&t.ctx, folder).await.unwrap() {
+                t.ctx.get_config(config).await.unwrap()
+            } else {
+                None
+            };
         let expected = if expected_destination == folder {
             None
         } else {

src/securejoin.rs

@@ -873,7 +873,7 @@ async fn could_not_establish_secure_connection(
     )
     .await;
     chat::add_info_msg(context, chat_id, &msg, time()).await?;
-    error!(
+    warn!(
         context,
         "StockMessage::ContactNotVerified posted to 1:1 chat ({})", details
     );

src/sql.rs

@@ -3,7 +3,7 @@
 use async_std::path::Path;
 use async_std::sync::RwLock;
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::convert::TryFrom;
 use std::time::Duration;
 
@@ -39,12 +39,14 @@ mod migrations;
 #[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
+    config_cache: RwLock<HashMap<String, String>>, // TODO should be <String, Option<String>> in order to save that an option is None?
 }
 
 impl Default for Sql {
     fn default() -> Self {
         Self {
             pool: RwLock::new(None),
+            config_cache: RwLock::new(HashMap::new()),
         }
     }
 }
@@ -387,6 +389,8 @@ impl Sql {
     /// will already have been logged.
     pub async fn set_raw_config(&self, key: impl AsRef<str>, value: Option<&str>) -> Result<()> {
         let key = key.as_ref();
+
+        // TODO could be on another thread
         if let Some(value) = value {
             let exists = self
                 .exists(
@@ -413,20 +417,36 @@ impl Sql {
                 .await?;
         }
 
+        let mut lock = self.config_cache.write().await;
+        if let Some(v) = value {
+            lock.insert(key.to_string(), v.to_string());
+        } else {
+            lock.remove(key);
+        }
+        drop(lock);
+
         Ok(())
     }
 
     /// Get configuration options from the database.
     pub async fn get_raw_config(&self, key: impl AsRef<str>) -> Result<Option<String>> {
-        let value = self
-            .query_get_value(
-                "SELECT value FROM config WHERE keyname=?;",
-                paramsv![key.as_ref()],
-            )
-            .await
-            .context(format!("failed to fetch raw config: {}", key.as_ref()))?;
+        let lock = self.config_cache.read().await;
+        let cached = lock.get(key.as_ref()).cloned();
+        drop(lock);
 
-        Ok(value)
+        if let Some(c) = cached {
+            Ok(Some(c))
+        } else {
+            let value = self
+                .query_get_value(
+                    "SELECT value FROM config WHERE keyname=?;",
+                    paramsv![key.as_ref()],
+                )
+                .await
+                .context(format!("failed to fetch raw config: {}", key.as_ref()))?;
+
+            Ok(value)
+        }
     }
 
     pub async fn set_raw_config_int(&self, key: impl AsRef<str>, value: i32) -> Result<()> {

src/sql/migrations.rs

@@ -526,6 +526,10 @@ impl Sql {
         })
         .await?;
 
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), format!("{}", version));
+        drop(lock);
+
         Ok(())
     }
 }