stop timings

* Fix #2335 (delete job flooding)

The problem was:

- You are offline, and an ephemeral message is due to delete.
- load_imap_deletion_job() is called and returns a deletion job for it.
- The job fails because, well, we are offline.
- The job is saved into the database.
- load_imap_deletion_job() is called again, and so on.

* Add test

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
     executor: doxygen
     steps:
       - checkout
-      - run: bash ci_scripts/run-doxygen.sh
+      - run: bash scripts/run-doxygen.sh
       - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
@@ -27,7 +27,7 @@ jobs:
       - checkout
       # the following commands on success produces
       # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
+      - run: bash scripts/remote_python_packaging.sh
       - persist_to_workspace:
           root: workspace
           paths:
@@ -42,7 +42,7 @@ jobs:
       - attach_workspace:
           at: workspace
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
 workflows:
   version: 2.1

.github/workflows/remote_tests.yml

@@ -18,4 +18,4 @@ jobs:
         shell: bash
         env:
           SSH_KEY: ${{ secrets.SSH_KEY }}
-      - run: ci_scripts/remote_tests_python.sh
+      - run: scripts/remote_tests_python.sh

.gitignore

@@ -26,3 +26,5 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
+
+coverage/

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog 
 
+## 1.53.0
+
+- fix sqlx performance regression #2355 2356
+
+- add a `ci_scripts/coverage.sh` #2333 #2334
+
+- refactorings and tests #2348 #2349 #2350
+
+- improve python bindings #2332 #2326
+
+
 ## 1.52.0
 
 - database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340

Cargo.lock

@@ -1160,7 +1160,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.52.0"
+version = "1.53.0"
 dependencies = [
  "ansi_term 0.12.1",
  "anyhow",
@@ -1229,7 +1229,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.52.0"
+version = "1.53.0"
 dependencies = [
  "anyhow",
  "async-std",
@@ -3464,8 +3464,8 @@ dependencies = [
 
 [[package]]
 name = "sqlx"
-version = "0.5.1"
-source = "git+https://github.com/launchbadge/sqlx?branch=master#9e8e3346970cd382a9baca1bba6462b7df4b4b63"
+version = "0.5.2"
+source = "git+https://github.com/deltachat/sqlx?branch=master#7a3c367a94902edc92ed58914301052a0a9ac466"
 dependencies = [
  "sqlx-core",
  "sqlx-macros",
@@ -3473,8 +3473,8 @@ dependencies = [
 
 [[package]]
 name = "sqlx-core"
-version = "0.5.1"
-source = "git+https://github.com/launchbadge/sqlx?branch=master#9e8e3346970cd382a9baca1bba6462b7df4b4b63"
+version = "0.5.2"
+source = "git+https://github.com/deltachat/sqlx?branch=master#7a3c367a94902edc92ed58914301052a0a9ac466"
 dependencies = [
  "ahash 0.7.2",
  "atoi",
@@ -3511,8 +3511,8 @@ dependencies = [
 
 [[package]]
 name = "sqlx-macros"
-version = "0.5.1"
-source = "git+https://github.com/launchbadge/sqlx?branch=master#9e8e3346970cd382a9baca1bba6462b7df4b4b63"
+version = "0.5.2"
+source = "git+https://github.com/deltachat/sqlx?branch=master#7a3c367a94902edc92ed58914301052a0a9ac466"
 dependencies = [
  "dotenv",
  "either",
@@ -3529,8 +3529,8 @@ dependencies = [
 
 [[package]]
 name = "sqlx-rt"
-version = "0.3.0"
-source = "git+https://github.com/launchbadge/sqlx?branch=master#9e8e3346970cd382a9baca1bba6462b7df4b4b63"
+version = "0.5.2"
+source = "git+https://github.com/deltachat/sqlx?branch=master#7a3c367a94902edc92ed58914301052a0a9ac466"
 dependencies = [
  "async-native-tls",
  "async-std",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.52.0"
+version = "1.53.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -60,7 +60,7 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.9.3"
 sha2 = "0.9.0"
 smallvec = "1.0.0"
-sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std-native-tls", "sqlite"] }
+sqlx = { git = "https://github.com/deltachat/sqlx", branch = "master", features = ["runtime-async-std-native-tls", "sqlite"] }
 # keep in sync with sqlx
 libsqlite3-sys = { version = "0.22.0", default-features = false, features = [ "pkg-config", "vcpkg", "bundled" ] }
 stop-token = { version = "0.1.1", features = ["unstable"] }
@@ -103,6 +103,14 @@ required-features = ["repl"]
 name = "create_account"
 harness = false
 
+[[bench]]
+name = "contacts"
+harness = false
+
+[[bench]]
+name = "search_msgs"
+harness = false
+
 [features]
 default = []
 internals = []

README.md

@@ -2,7 +2,9 @@
 
 > Deltachat-core written in Rust 
 
-[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
+[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![Remote tests](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -111,11 +113,6 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
-[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
-[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
-[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
-[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
 ## Language bindings and frontend projects
 
 Language bindings are available for:

benches/contacts.rs

@@ -0,0 +1,39 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use tempfile::tempdir;
+
+async fn address_book_benchmark(n: u32, read_count: u32) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    let book = (0..n)
+        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .collect::<Vec<String>>()
+        .join("");
+
+    Contact::add_address_book(&context, book).await.unwrap();
+
+    let query: Option<&str> = None;
+    for _ in 0..read_count {
+        Contact::get_all(&context, 0, query).await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 500 contacts", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+    });
+
+    c.bench_function("create 100 contacts and read it 1000 times", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -0,0 +1,29 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::context::Context;
+use std::path::Path;
+
+async fn search_benchmark(path: impl AsRef<Path>) {
+    let dbfile = path.as_ref();
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    for _ in 0..10u32 {
+        context.search_msgs(None, "hello").await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        c.bench_function("search hello", |b| {
+            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+        });
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.52.0"
+version = "1.53.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -1272,6 +1272,12 @@ uint32_t dc_get_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id);
  * search results may just hilite the corresponding messages and present a
  * prev/next button.
  *
+ * For global search, result is limited to 1000 messages,
+ * this allows incremental search done fast.
+ * So, when getting exactly 1000 results, the result may be truncated;
+ * the UIs may display sth. as "1000+ messages found" in this case.
+ * Chat search (if a chat_id is set) is not limited.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id ID of the chat to search messages in.

examples/repl/cmdline.rs

@@ -525,9 +525,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = std::time::SystemTime::now()
-                .duration_since(time_start)
-                .unwrap_or_default();
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -604,7 +602,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
+            let time_start = std::time::SystemTime::now();
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
+
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -659,7 +660,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "{} messages.",
                 sel_chat.get_id().get_msg_cnt(&context).await?
             );
+
+            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
+            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
+
+            println!(
+                "{:?} to create this list, {:?} to mark all messages as noticed.",
+                time_needed, time_noticed_needed
+            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
@@ -898,10 +907,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None
             };
 
+            let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
+            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");

python/README.rst

@@ -142,7 +142,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -151,9 +151,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps scripts/docker_coredeps
 
-This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/src/deltachat/account.py

@@ -89,6 +89,22 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        log("=============== " + self.get_config("displayname") + " ===============")
+        cursor = 0
+        for name, val in self.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 

python/src/deltachat/direct_imap.py

@@ -47,8 +47,9 @@ def dc_account_extra_configure(account):
 
     except Exception as e:
         # Uncaught exceptions here would lead to a timeout without any note written to the log
-        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
-        account.log("===================", e, "===================")
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
 
 
 @deltachat.global_hookimpl
@@ -172,21 +173,6 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()

python/src/deltachat/message.py

@@ -86,6 +86,23 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
+    @props.with_doc
+    def html(self):
+        """html text of this messages (might be empty if not an html message). """
+        return from_dc_charpointer(
+            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
+
+    def has_html(self):
+        """return True if this message has an html part, False otherwise."""
+        return lib.dc_msg_has_html(self._dc_msg)
+
+    def set_html(self, html_text):
+        """set the html part of this message.
+
+        It is possible to have text and html part at the same time.
+        """
+        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
+
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -236,6 +253,20 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    @props.with_doc
+    def override_sender_name(self):
+        """the name that should be shown over the message instead of the contact display name.
+
+        Usually used to impersonate someone else.
+        """
+        return from_dc_charpointer(
+            lib.dc_msg_get_override_sender_name(self._dc_msg))
+
+    def set_override_sender_name(self, name):
+        """set different sender name for a message. """
+        lib.dc_msg_set_override_sender_name(
+            self._dc_msg, as_dc_charpointer(name))
+
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 

python/src/deltachat/testplugin.py

@@ -414,13 +414,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
-                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_accepted_chat(self, ac1, ac2):

python/tests/test_account.py

@@ -816,6 +816,48 @@ class TestOnlineAccount:
         assert open(msg.filename).read() == content
         assert msg.filename.endswith(basename)
 
+    def test_html_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        html_text = "<p>hello HTML world</p>"
+
+        lp.sec("ac1: prepare and send text message to ac2")
+        msg1 = chat.send_text("message0")
+        assert not msg1.has_html()
+        assert msg1.html == ""
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"
+        assert not msg2.has_html()
+        assert msg2.html == ""
+
+        lp.sec("ac1: prepare and send HTML+text message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message1")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+        assert msg1.has_html()
+        assert html_text in msg1.html
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
+        lp.sec("ac1: prepare and send HTML-only message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert "<p>" not in msg2.text
+        assert "hello HTML world" in msg2.text
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
@@ -991,6 +1033,38 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_message_override_sender_name(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        overridden_name = "someone else"
+
+        ac1.set_config("displayname", "ac1")
+
+        lp.sec("sending text message with overridden name from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_override_sender_name(overridden_name)
+        msg1.set_text("message1")
+        msg1 = chat.send_msg(msg1)
+        assert msg1.override_sender_name == overridden_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert msg2.override_sender_name == overridden_name
+
+        lp.sec("sending normal text message from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message2")
+        msg1 = chat.send_msg(msg1)
+        assert not msg1.override_sender_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message2"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert not msg2.override_sender_name
+
     @pytest.mark.parametrize("mvbox_move", [True, False])
     def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
         # Please only change this test if you are very sure that it will still catch the issues it catches now.
@@ -1498,6 +1572,12 @@ class TestOnlineAccount:
         original_image_path = data.get_path("d.png")
         chat1.send_image(original_image_path)
 
+        # Add another 100KB file that ensures that the progress is smooth enough
+        path = tmpdir.join("attachment.txt")
+        with open(path, "w") as file:
+            file.truncate(100000)
+        chat1.send_file(path.strpath)
+
         def assert_account_is_proper(ac):
             contacts = ac.get_contacts(query="some1")
             assert len(contacts) == 1
@@ -1505,7 +1585,7 @@ class TestOnlineAccount:
             assert contact2.addr == "some1@example.org"
             chat2 = contact2.create_chat()
             messages = chat2.get_messages()
-            assert len(messages) == 2
+            assert len(messages) == 3
             assert messages[0].text == "msg1"
             assert messages[1].filemime == "image/png"
             assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size

scripts/README.md

@@ -30,8 +30,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    ci_scripts/manual_remote_tests.sh rust
-    ci_scripts/manual_remote_tests.sh python
+    scripts/manual_remote_tests.sh rust
+    scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -45,6 +45,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd ci_scripts  # where all CI things are 
+    cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 

scripts/ci_upload.sh

@@ -42,7 +42,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ci_scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
 rsync -avz \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   $WHEELHOUSEDIR \

scripts/coverage.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+
+if ! which grcov 2>/dev/null 1>&2; then
+    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
+    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
+    exit 1
+fi
+
+# Allow `-Z` flags without using nightly Rust.
+export RUSTC_BOOTSTRAP=1
+
+# We are using `-Zprofile` instead of source-based coverage [1]
+# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
+#
+# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
+# [2] https://github.com/mozilla/grcov/issues/595
+
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTDOCFLAGS="-Cpanic=abort"
+cargo clean
+cargo build
+cargo test
+
+grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/manual_remote_tests.sh

@@ -6,4 +6,4 @@ export CIRCLE_BUILD_NUM=$USER
 export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
 export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
 
-time bash ci_scripts/$CIRCLE_JOB.sh 
+time bash scripts/$CIRCLE_JOB.sh 

scripts/remote_python_packaging.sh

@@ -36,7 +36,7 @@ ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     # run everything else inside docker 
     docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 _HERE
 

scripts/remote_tests_python.sh

@@ -42,5 +42,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash ci_scripts/run-python-test.sh 
+    bash scripts/run-python-test.sh 
 _HERE

scripts/remote_tests_rust.sh

@@ -25,6 +25,6 @@ ssh $SSHTARGET <<_HERE
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 
-    bash ci_scripts/run-rust-test.sh
+    bash scripts/run-rust-test.sh
 _HERE
 

scripts/run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   ./run-integration-tests.sh -e py35
+#   scripts/run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   ./run-integration-tests.sh -e py35 -- -x
+#   scripts/run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}

src/chat.rs

@@ -958,7 +958,6 @@ impl Chat {
         timestamp: i64,
     ) -> Result<MsgId, Error> {
         let mut new_references = "".into();
-        let mut msg_id = 0;
         let mut to_id = 0;
         let mut location_id = 0;
 
@@ -1066,10 +1065,10 @@ impl Chat {
 
         // add independent location to database
 
-        if msg.param.exists(Param::SetLatitude)
-            && context
+        if msg.param.exists(Param::SetLatitude) {
+            if let Ok(row_id) = context
                 .sql
-                .execute(
+                .insert(
                     sqlx::query(
                         "INSERT INTO locations \
                      (timestamp,from_id,chat_id, latitude,longitude,independent)\
@@ -1082,18 +1081,9 @@ impl Chat {
                     .bind(msg.param.get_float(Param::SetLongitude).unwrap_or_default()),
                 )
                 .await
-                .is_ok()
-        {
-            location_id = context
-                .sql
-                .get_rowid2(
-                    "locations",
-                    "timestamp",
-                    timestamp,
-                    "from_id",
-                    DC_CONTACT_ID_SELF as i64,
-                )
-                .await?;
+            {
+                location_id = row_id;
+            }
         }
 
         let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
@@ -1123,9 +1113,9 @@ impl Chat {
 
         // add message to the database
 
-        if context
+        let msg_id = context
             .sql
-            .execute(
+            .insert(
                 sqlx::query(
                     "INSERT INTO msgs (
                         rfc724_mid,
@@ -1168,19 +1158,7 @@ impl Chat {
                 .bind(ephemeral_timer)
                 .bind(ephemeral_timestamp),
             )
-            .await
-            .is_ok()
-        {
-            msg_id = context
-                .sql
-                .get_rowid("msgs", "rfc724_mid", new_rfc724_mid)
-                .await?;
-        } else {
-            error!(
-                context,
-                "Cannot send message, cannot insert to database ({}).", self.id,
-            );
-        }
+            .await?;
         schedule_ephemeral_task(context).await;
 
         Ok(MsgId::new(u32::try_from(msg_id)?))
@@ -2173,9 +2151,9 @@ pub async fn create_group_chat(
     let draft_txt = stock_str::new_group_draft(context, &chat_name).await;
     let grpid = dc_create_id();
 
-    context
+    let row_id = context
         .sql
-        .execute(
+        .insert(
             sqlx::query(
                 "INSERT INTO chats
             (type, name, grpid, param, created_timestamp)
@@ -2188,8 +2166,6 @@ pub async fn create_group_chat(
         )
         .await?;
 
-    let row_id = context.sql.get_rowid("chats", "grpid", grpid).await?;
-
     let chat_id = ChatId::new(u32::try_from(row_id)?);
     if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
         let mut draft_msg = Message::new(Viewtype::Text);
@@ -2945,9 +2921,9 @@ pub async fn add_device_msg_with_importance(
             }
         }
 
-        context
+        let row_id = context
             .sql
-            .execute(
+            .insert(
                 sqlx::query(
                     "INSERT INTO msgs (
             chat_id,
@@ -2979,10 +2955,6 @@ pub async fn add_device_msg_with_importance(
             )
             .await?;
 
-        let row_id = context
-            .sql
-            .get_rowid("msgs", "rfc724_mid", &rfc724_mid)
-            .await?;
         msg_id = MsgId::new(u32::try_from(row_id)?);
     }
 
@@ -3056,7 +3028,8 @@ pub(crate) async fn add_info_msg_with_cmd(
         param.set_cmd(cmd)
     }
 
-    context.sql.execute(
+    let row_id =
+    context.sql.insert(
         sqlx::query("INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid,ephemeral_timer, param) VALUES (?,?,?, ?,?,?, ?,?,?, ?);")
             .bind(chat_id)
             .bind(DC_CONTACT_ID_INFO as i32)
@@ -3070,11 +3043,6 @@ pub(crate) async fn add_info_msg_with_cmd(
             .bind(param.to_string())
     ).await?;
 
-    let row_id = context
-        .sql
-        .get_rowid("msgs", "rfc724_mid", &rfc724_mid)
-        .await
-        .unwrap_or_default();
     let msg_id = MsgId::new(u32::try_from(row_id)?);
     context.emit_event(EventType::MsgsChanged { chat_id, msg_id });
     Ok(msg_id)
@@ -3887,7 +3855,6 @@ mod tests {
             .unwrap();
         let alice_chat = Chat::load_from_db(&alice, alice_chat_id).await.unwrap();
 
-        println!("----- add_contact_to_chat");
         add_contact_to_chat(&alice, alice_chat_id, contact_id).await;
         assert_eq!(
             get_chat_contacts(&alice, alice_chat_id)
@@ -3896,11 +3863,9 @@ mod tests {
                 .len(),
             2
         );
-        println!("----- send_text_msg");
         send_text_msg(&alice, alice_chat_id, "hi!".to_string())
             .await
             .ok();
-        println!("----- get_chat_msgs");
         assert_eq!(
             get_chat_msgs(&alice, alice_chat_id, 0, None)
                 .await
@@ -3909,7 +3874,6 @@ mod tests {
             1
         );
 
-        println!("----- pop_sent_msg");
         // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com).
         let msg = alice.pop_sent_msg().await.payload();
         assert_eq!(msg.match_indices("Gr.").count(), 2);
@@ -3922,11 +3886,9 @@ mod tests {
             .unwrap();
         let msg = bob.get_last_msg().await;
 
-        println!("load from dbb");
         let bob_chat = Chat::load_from_db(&bob, msg.chat_id).await.unwrap();
         assert_eq!(bob_chat.grpid, alice_chat.grpid);
 
-        println!("chat loaded");
         // Bob answers - simulate a normal MUA by not setting `Chat-*`-headers;
         // moreover, Bob's SMTP-server also replaces the `Message-ID:`-header
         send_text_msg(&bob, bob_chat.id, "ho!".to_string())
@@ -3937,12 +3899,10 @@ mod tests {
         let msg = msg.replace("Chat-", "XXXX-");
         assert_eq!(msg.match_indices("Chat-").count(), 0);
 
-        println!("last receive start");
         // Alice receives this message - she can still detect the group by the `References:`-header
         dc_receive_imf(&alice, msg.as_bytes(), "INBOX", 2, false)
             .await
             .unwrap();
-        println!("----- last receie if");
         let msg = alice.get_last_msg().await;
         assert_eq!(msg.chat_id, alice_chat_id);
         assert_eq!(msg.text, Some("ho!".to_string()));

src/configure/server_params.rs

@@ -25,7 +25,7 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
@@ -42,7 +42,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
         if self.hostname.is_empty() {
             self.hostname = param_domain.to_string();
@@ -62,7 +62,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+    fn expand_ports(mut self) -> Vec<ServerParams> {
         // Try to infer port from socket security.
         if self.port == 0 {
             self.port = match self.socket {
@@ -160,5 +160,37 @@ mod tests {
                 username: "foobar".to_string(),
             }],
         );
+
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Automatic,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string()
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string()
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -310,16 +310,6 @@ impl Default for Viewtype {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -328,3 +318,100 @@ pub enum KeyType {
     Public = 0,
     Private = 1,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_chattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Chattype::Undefined, Chattype::default());
+        assert_eq!(Chattype::Undefined, Chattype::from_i32(0).unwrap());
+        assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
+        assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
+        assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+    }
+
+    #[test]
+    fn test_keygentype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyGenType::Default, KeyGenType::default());
+        assert_eq!(KeyGenType::Default, KeyGenType::from_i32(0).unwrap());
+        assert_eq!(KeyGenType::Rsa2048, KeyGenType::from_i32(1).unwrap());
+        assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_keytype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap());
+        assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_showemails_values() {
+        // values may be written to disk and must not change
+        assert_eq!(ShowEmails::Off, ShowEmails::default());
+        assert_eq!(ShowEmails::Off, ShowEmails::from_i32(0).unwrap());
+        assert_eq!(
+            ShowEmails::AcceptedContacts,
+            ShowEmails::from_i32(1).unwrap()
+        );
+        assert_eq!(ShowEmails::All, ShowEmails::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_blocked_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Blocked::Not, Blocked::default());
+        assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
+        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_mediaquality_values() {
+        // values may be written to disk and must not change
+        assert_eq!(MediaQuality::Balanced, MediaQuality::default());
+        assert_eq!(MediaQuality::Balanced, MediaQuality::from_i32(0).unwrap());
+        assert_eq!(MediaQuality::Worse, MediaQuality::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_videochattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(VideochatType::Unknown, VideochatType::default());
+        assert_eq!(VideochatType::Unknown, VideochatType::from_i32(0).unwrap());
+        assert_eq!(
+            VideochatType::BasicWebrtc,
+            VideochatType::from_i32(1).unwrap()
+        );
+        assert_eq!(VideochatType::Jitsi, VideochatType::from_i32(2).unwrap());
+    }
+}

src/contact.rs

@@ -530,9 +530,9 @@ impl Contact {
             let update_name = manual;
             let update_authname = !manual;
 
-            if context
+            if let Ok(new_row_id) = context
                 .sql
-                .execute(
+                .insert(
                     sqlx::query(
                         "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
                     )
@@ -550,9 +550,8 @@ impl Contact {
                     }),
                 )
                 .await
-                .is_ok()
             {
-                row_id = context.sql.get_rowid("contacts", "addr", &addr).await?;
+                row_id = new_row_id;
                 sth_modified = Modifier::Created;
                 info!(context, "added contact id={} addr={}", row_id, &addr);
             } else {

src/context.rs

@@ -471,7 +471,6 @@ impl Context {
             return Ok(Vec::new());
         }
         let str_like_in_text = format!("%{}%", real_query);
-        let str_like_beg = format!("{}%", real_query);
 
         let list = if let Some(chat_id) = chat_id {
             self.sql
@@ -484,12 +483,11 @@ impl Context {
                  WHERE m.chat_id=?
                    AND m.hidden=0
                    AND ct.blocked=0
-                   AND (txt LIKE ? OR ct.name LIKE ?)
+                   AND txt LIKE ?
                  ORDER BY m.timestamp,m.id;",
                     )
                     .bind(chat_id)
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
+                    .bind(str_like_in_text),
                 )
                 .await?
                 .map(|row| {
@@ -500,6 +498,16 @@ impl Context {
                 .collect::<sqlx::Result<Vec<MsgId>>>()
                 .await?
         } else {
+            // For performance reasons results are sorted only by `id`, that is in the order of
+            // message reception.
+            //
+            // Unlike chat view, sorting by `timestamp` is not necessary but slows down the query by
+            // ~25% according to benchmarks.
+            //
+            // To speed up incremental search, where queries for few characters usually return lots
+            // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
+            // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
+            // The limit is documented and UI may add a hint when getting 1000 results.
             self.sql
                 .fetch(
                     sqlx::query(
@@ -513,11 +521,10 @@ impl Context {
                    AND m.hidden=0
                    AND c.blocked=0
                    AND ct.blocked=0
-                   AND (m.txt LIKE ? OR ct.name LIKE ?)
-                 ORDER BY m.timestamp DESC,m.id DESC;",
+                   AND m.txt LIKE ?
+                 ORDER BY m.id DESC LIMIT 1000",
                     )
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
+                    .bind(str_like_in_text),
                 )
                 .await?
                 .map(|row| {
@@ -605,9 +612,14 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::chat::{get_chat_contacts, get_chat_msgs, set_muted, Chat, MuteDuration};
+    use crate::chat::{
+        create_by_contact_id, get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat,
+        MuteDuration,
+    };
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::message::Message;
     use crate::test_utils::TestContext;
     use std::time::Duration;
     use strum::IntoEnumIterator;
@@ -879,4 +891,93 @@ mod tests {
             }
         }
     }
+
+    #[async_std::test]
+    async fn test_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let self_talk = create_by_contact_id(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Global search finds nothing.
+        let res = alice.search_msgs(None, "foo").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds nothing.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert!(res.is_empty());
+
+        // Add messages to chat with Bob.
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.set_text(Some("foobar".to_string()));
+        send_msg(&alice, chat.id, &mut msg1).await?;
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_text(Some("barbaz".to_string()));
+        send_msg(&alice, chat.id, &mut msg2).await?;
+
+        // Global search with a part of text finds the message.
+        let res = alice.search_msgs(None, "ob").await?;
+        assert_eq!(res.len(), 1);
+
+        // Global search for "bar" matches both "foobar" and "barbaz".
+        let res = alice.search_msgs(None, "bar").await?;
+        assert_eq!(res.len(), 2);
+
+        // Message added later is returned first.
+        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.get(1), Some(&msg1.id));
+
+        // Global search with longer text does not find any message.
+        let res = alice.search_msgs(None, "foobarbaz").await?;
+        assert!(res.is_empty());
+
+        // Search for random string finds nothing.
+        let res = alice.search_msgs(None, "abc").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds the message.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1);
+
+        // Search in Saved Messages does not find the message.
+        let res = alice.search_msgs(Some(self_talk), "foo").await?;
+        assert!(res.is_empty());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_limit_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Add 999 messages
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foobar".to_string()));
+        for _ in 0..999 {
+            send_msg(&alice, chat.id, &mut msg).await?;
+        }
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 999);
+
+        // Add one more message, no limit yet
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // Add one more message, that one is truncated then
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // In-chat should not be not limited
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1001);
+
+        Ok(())
+    }
 }

src/dc_receive_imf.rs

@@ -71,11 +71,7 @@ pub(crate) async fn dc_receive_imf_inner(
     info!(
         context,
         "Receiving message {}/{}, seen={}...",
-        if !server_folder.as_ref().is_empty() {
-            server_folder.as_ref()
-        } else {
-            "?"
-        },
+        server_folder.as_ref(),
         server_uid,
         seen
     );
@@ -978,9 +974,9 @@ async fn add_parts(
         // also change `MsgId::trash()` and `delete_expired_messages()`
         let trash = chat_id.is_trash();
 
-        context
+        let row_id = context
             .sql
-            .execute(
+            .insert(
                 sqlx::query(
                     r#"
 INSERT INTO msgs
@@ -1040,12 +1036,7 @@ INSERT INTO msgs
                 .bind(ephemeral_timestamp),
             )
             .await?;
-        let msg_id = MsgId::new(u32::try_from(
-            context
-                .sql
-                .get_rowid("msgs", "rfc724_mid", &rfc724_mid)
-                .await?,
-        )?);
+        let msg_id = MsgId::new(u32::try_from(row_id)?);
 
         created_db_entries.push((*chat_id, msg_id));
         *insert_msg_id = msg_id;
@@ -1769,7 +1760,8 @@ async fn create_multiuser_record(
     create_blocked: Blocked,
     create_protected: ProtectionStatus,
 ) -> Result<ChatId> {
-    context.sql.execute(
+    let row_id =
+    context.sql.insert(
         sqlx::query(
             "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected) VALUES(?, ?, ?, ?, ?, ?);")
             .bind(chattype)
@@ -1780,11 +1772,6 @@ async fn create_multiuser_record(
             .bind(create_protected)
     ).await?;
 
-    let row_id = context
-        .sql
-        .get_rowid("chats", "grpid", grpid.as_ref())
-        .await?;
-
     let chat_id = ChatId::new(u32::try_from(row_id)?);
     info!(
         context,

src/dc_tools.rs

@@ -879,16 +879,6 @@ mod tests {
         }
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
-        if dc_file_exist!(context, "$BLOBDIR/foobar").await
-            || dc_file_exist!(context, "$BLOBDIR/dada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar").await;
-            dc_delete_file(context, "$BLOBDIR/dada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
-        }
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());

src/dehtml.rs

@@ -314,6 +314,7 @@ mod tests {
                 "[ Foo ](https://example.com)",
             ),
             ("<b> bar </b>", "* bar *"),
+            ("<i>foo</i>", "_foo_"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
             // Despite missing ', this should be shown:
@@ -391,6 +392,13 @@ mod tests {
         assert_eq!(txt.trim(), "lots of text");
     }
 
+    #[test]
+    fn test_pre_tag() {
+        let input = "<html><pre>\ntwo\nlines\n</pre></html>";
+        let txt = dehtml(input).unwrap();
+        assert_eq!(txt.trim(), "two\nlines");
+    }
+
     #[async_std::test]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");

src/ephemeral.rs

@@ -66,7 +66,6 @@ use async_std::task;
 use serde::{Deserialize, Serialize};
 use sqlx::Row;
 
-use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
@@ -77,6 +76,10 @@ use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
 use crate::stock_str;
+use crate::{
+    chat::{lookup_by_contact_id, send_msg, ChatId},
+    job,
+};
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -490,10 +493,12 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
+         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
             )
             .bind(threshold_timestamp)
-            .bind(now),
+            .bind(now)
+            .bind(job::Action::DeleteMsgOnImap),
         )
         .await?;
 
@@ -537,6 +542,7 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use crate::param::Params;
     use async_std::task::sleep;
 
     use super::*;
@@ -758,7 +764,31 @@ mod tests {
 
         sleep(Duration::from_millis(1100)).await;
 
+        // Check checks that the msg was deleted locally
         check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+
+        // Check that the msg will be deleted on the server
+        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
+        t.sql
+            .execute(sqlx::query("UPDATE msgs SET server_uid=1 WHERE id=?").bind(msg.sender_msg_id))
+            .await
+            .unwrap();
+        let job = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(
+            job,
+            Some(job::Job::new(
+                job::Action::DeleteMsgOnImap,
+                msg.sender_msg_id.to_u32(),
+                Params::new(),
+                0,
+            ))
+        );
+        // Let's assume that executing the job fails on first try and the job is saved to the db
+        job.unwrap().save(&t).await.unwrap();
+
+        // Make sure that we don't get yet another job when loading from db
+        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(job2, None);
     }
 
     async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {

src/format_flowed.rs

@@ -146,6 +146,17 @@ mod tests {
 
         let text = "> Not a quote";
         assert_eq!(format_flowed(text), " > Not a quote");
+
+        // Test space stuffing of wrapped lines
+        let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    >                               \n\
+                    > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "\x20> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        clients.\r\n\
+                        \x20>\r\n\
+                        \x20> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
     }
 
     #[test]

src/imap/scan_folders.rs

@@ -61,14 +61,8 @@ impl Imap {
                 spam_folder = Some(folder.name().to_string());
             }
 
-            if watched_folders.contains(&foldername.to_string()) {
-                info!(
-                    context,
-                    "Not scanning folder {} as it is watched anyway", foldername
-                );
-            } else {
-                info!(context, "Scanning folder: {}", foldername);
-
+            // Don't scan folders that are watched anyway
+            if !watched_folders.contains(&foldername.to_string()) {
                 if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
                     warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
                 }

src/job.rs

@@ -130,7 +130,7 @@ impl From<Action> for Thread {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Job {
     pub job_id: u32,
     pub action: Action,
@@ -1030,7 +1030,7 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
+pub(crate) async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         info!(context, "verbose (issue 2335): loading imap deletion job");
         Some(Job::new(
@@ -1118,7 +1118,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
             if let Err(err) = res {
                 warn!(
                     context,
-                    "{} removes job {} as it failed with error {:?}", &connection, job, err
+                    "{} removes job {} as it failed with error {:#}", &connection, job, err
                 );
             } else {
                 info!(

src/location.rs

@@ -571,9 +571,9 @@ pub async fn save(
             .exists(sqlx::query(stmt_test).bind(timestamp).bind(contact_id))
             .await?;
         if independent || !exists {
-            context
+            let row_id = context
                 .sql
-                .execute(
+                .insert(
                     sqlx::query(stmt_insert)
                         .bind(timestamp)
                         .bind(contact_id)
@@ -587,16 +587,7 @@ pub async fn save(
 
             if timestamp > newest_timestamp {
                 newest_timestamp = timestamp;
-                newest_location_id = context
-                    .sql
-                    .get_rowid2(
-                        "locations",
-                        "timestamp",
-                        timestamp,
-                        "from_id",
-                        contact_id as i64,
-                    )
-                    .await?;
+                newest_location_id = row_id;
             }
         }
     }

src/login_param.rs

@@ -30,7 +30,7 @@ impl Default for CertificateChecks {
 }
 
 /// Login parameters for a single server, either IMAP or SMTP
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub struct ServerLoginParam {
     pub server: String,
     pub user: String,
@@ -43,7 +43,7 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub struct LoginParam {
     pub addr: String,
     pub imap: ServerLoginParam,
@@ -304,6 +304,8 @@ pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
 mod tests {
     use super::*;
 
+    use crate::test_utils::TestContext;
+
     #[test]
     fn test_certificate_checks_display() {
         use std::string::ToString;
@@ -313,4 +315,37 @@ mod tests {
             CertificateChecks::AcceptInvalidCertificates.to_string()
         );
     }
+
+    #[async_std::test]
+    async fn test_save_load_login_param() -> anyhow::Result<()> {
+        let t = TestContext::new().await;
+
+        let param = LoginParam {
+            addr: "alice@example.com".to_string(),
+            imap: ServerLoginParam {
+                server: "imap.example.com".to_string(),
+                user: "alice".to_string(),
+                password: "foo".to_string(),
+                port: 123,
+                security: Socket::Starttls,
+                certificate_checks: CertificateChecks::Strict,
+            },
+            smtp: ServerLoginParam {
+                server: "smtp.example.com".to_string(),
+                user: "alice@example.com".to_string(),
+                password: "bar".to_string(),
+                port: 456,
+                security: Socket::Ssl,
+                certificate_checks: CertificateChecks::AcceptInvalidCertificates,
+            },
+            server_flags: 0,
+            provider: get_provider_by_id("example.com"),
+        };
+
+        param.save_to_database(&t, "foobar_").await?;
+        let loaded = LoginParam::from_database(&t, "foobar_").await?;
+
+        assert_eq!(param, loaded);
+        Ok(())
+    }
 }

src/message.rs

@@ -317,7 +317,8 @@ impl Message {
     pub async fn load_from_db(context: &Context, id: MsgId) -> Result<Message, Error> {
         ensure!(
             !id.is_special(),
-            "Can not load special message IDs from DB."
+            "Can not load special message ID {} from DB.",
+            id
         );
         let row = context
             .sql

src/mimefactory.rs

@@ -1,6 +1,6 @@
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, format_err, Error};
+use anyhow::{bail, ensure, format_err, Result};
 use async_std::prelude::*;
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
@@ -90,7 +90,7 @@ impl<'a> MimeFactory<'a> {
         context: &Context,
         msg: &'a Message,
         attach_selfavatar: bool,
-    ) -> Result<MimeFactory<'a>, Error> {
+    ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
         let from_addr = context
@@ -178,7 +178,7 @@ impl<'a> MimeFactory<'a> {
         context: &Context,
         msg: &'a Message,
         additional_msg_ids: Vec<String>,
-    ) -> Result<MimeFactory<'a>, Error> {
+    ) -> Result<MimeFactory<'a>> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
@@ -222,7 +222,7 @@ impl<'a> MimeFactory<'a> {
     async fn peerstates_for_recipients(
         &self,
         context: &Context,
-    ) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
+    ) -> Result<Vec<(Option<Peerstate>, &str)>> {
         let self_addr = context
             .get_config(Config::ConfiguredAddr)
             .await?
@@ -302,7 +302,7 @@ impl<'a> MimeFactory<'a> {
         }
     }
 
-    async fn should_do_gossip(&self, context: &Context) -> Result<bool, Error> {
+    async fn should_do_gossip(&self, context: &Context) -> Result<bool> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
@@ -401,7 +401,7 @@ impl<'a> MimeFactory<'a> {
             .collect()
     }
 
-    pub async fn render(mut self, context: &Context) -> Result<RenderedEmail, Error> {
+    pub async fn render(mut self, context: &Context) -> Result<RenderedEmail> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
         // - Secure-Join*
@@ -673,7 +673,7 @@ impl<'a> MimeFactory<'a> {
         Some(part)
     }
 
-    async fn get_location_kml_part(&mut self, context: &Context) -> Result<PartBuilder, Error> {
+    async fn get_location_kml_part(&mut self, context: &Context) -> Result<PartBuilder> {
         let (kml_content, last_added_location_id) =
             location::get_kml(context, self.msg.chat_id).await?;
         let part = PartBuilder::new()
@@ -701,7 +701,7 @@ impl<'a> MimeFactory<'a> {
         protected_headers: &mut Vec<Header>,
         unprotected_headers: &mut Vec<Header>,
         grpimage: &Option<String>,
-    ) -> Result<(PartBuilder, Vec<PartBuilder>), Error> {
+    ) -> Result<(PartBuilder, Vec<PartBuilder>)> {
         let chat = match &self.loaded {
             Loaded::Message { chat } => chat,
             Loaded::Mdn { .. } => bail!("Attempt to render MDN as a message"),
@@ -977,7 +977,7 @@ impl<'a> MimeFactory<'a> {
 
         // add attachment part
         if chat::msgtype_has_file(self.msg.viewtype) {
-            if !is_file_size_okay(context, self.msg).await {
+            if !is_file_size_okay(context, self.msg).await? {
                 bail!(
                     "Message exceeds the recommended {} MB.",
                     RECOMMENDED_FILE_SIZE / 1_000_000,
@@ -1022,7 +1022,7 @@ impl<'a> MimeFactory<'a> {
     }
 
     /// Render an MDN
-    async fn render_mdn(&mut self, context: &Context) -> Result<PartBuilder, Error> {
+    async fn render_mdn(&mut self, context: &Context) -> Result<PartBuilder> {
         // RFC 6522, this also requires the `report-type` parameter which is equal
         // to the MIME subtype of the second body part of the multipart/report
         //
@@ -1120,7 +1120,7 @@ async fn build_body_file(
     context: &Context,
     msg: &Message,
     base_name: &str,
-) -> Result<(PartBuilder, String), Error> {
+) -> Result<(PartBuilder, String)> {
     let blob = msg
         .param
         .get_blob(Param::File, context, true)
@@ -1194,7 +1194,7 @@ async fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String), Error> {
+fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String)> {
     let blob = BlobObject::from_path(context, path)?;
     let filename_to_send = match blob.suffix() {
         Some(suffix) => format!("avatar.{}", suffix),
@@ -1226,13 +1226,13 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool
         .any(|(_, cur)| cur.to_lowercase() == addr_lc)
 }
 
-async fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
-    match msg.param.get_path(Param::File, context).unwrap_or(None) {
+async fn is_file_size_okay(context: &Context, msg: &Message) -> Result<bool> {
+    match msg.param.get_path(Param::File, context)? {
         Some(path) => {
             let bytes = dc_get_filebytes(context, &path).await;
-            bytes <= UPPER_LIMIT_FILE_SIZE
+            Ok(bytes <= UPPER_LIMIT_FILE_SIZE)
         }
-        None => false,
+        None => Ok(false),
     }
 }
 

src/provider/mod.rs

@@ -51,7 +51,7 @@ pub enum Oauth2Authorizer {
     Gmail = 2,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Server {
     pub protocol: Protocol,
     pub socket: Socket,
@@ -60,13 +60,13 @@ pub struct Server {
     pub username_pattern: UsernamePattern,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct ConfigDefault {
     pub key: Config,
     pub value: &'static str,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct Provider {
     /// Unique ID, corresponding to provider database filename.
     pub id: &'static str,

src/sql/mod.rs

@@ -86,6 +86,7 @@ impl Sql {
             .read_only(false)
             .busy_timeout(Duration::from_secs(100))
             .create_if_missing(true)
+            .shared_cache(true)
             .synchronous(SqliteSynchronous::Normal);
 
         PoolOptions::<Sqlite>::new()
@@ -112,6 +113,7 @@ PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
             .journal_mode(SqliteJournalMode::Wal)
             .filename(dbfile.as_ref())
             .read_only(readonly)
+            .shared_cache(true)
             .busy_timeout(Duration::from_secs(100))
             .synchronous(SqliteSynchronous::Normal);
 
@@ -122,6 +124,7 @@ PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
                     let q = r#"
 PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
 PRAGMA query_only=1; -- Protect against writes even in read-write mode
+PRAGMA read_uncommitted=1; -- This helps avoid "table locked" errors in shared cache mode
 "#;
 
                     conn.execute_many(sqlx::query(q))
@@ -222,6 +225,18 @@ PRAGMA query_only=1; -- Protect against writes even in read-write mode
         Ok(rows.rows_affected())
     }
 
+    /// Executes the given query, returning the last inserted row ID.
+    pub async fn insert<'q, E>(&self, query: Query<'q, Sqlite, E>) -> Result<i64>
+    where
+        E: 'q + IntoArguments<'q, Sqlite>,
+    {
+        let lock = self.writer.read().await;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
+
+        let rows = pool.execute(query).await?;
+        Ok(rows.last_insert_rowid())
+    }
+
     /// Execute many queries.
     pub async fn execute_many<'q, E>(&self, query: Query<'q, Sqlite, E>) -> Result<()>
     where
@@ -484,52 +499,6 @@ PRAGMA query_only=1; -- Protect against writes even in read-write mode
             .await
             .map(|s| s.and_then(|r| r.parse().ok()))
     }
-
-    /// Alternative to sqlite3_last_insert_rowid() which MUST NOT be used due to race conditions, see comment above.
-    /// the ORDER BY ensures, this function always returns the most recent id,
-    /// eg. if a Message-ID is split into different messages.
-    pub async fn get_rowid(
-        &self,
-        table: impl AsRef<str>,
-        field: impl AsRef<str>,
-        value: impl AsRef<str>,
-    ) -> Result<i64> {
-        // alternative to sqlite3_last_insert_rowid() which MUST NOT be used due to race conditions, see comment above.
-        // the ORDER BY ensures, this function always returns the most recent id,
-        // eg. if a Message-ID is split into different messages.
-        let query = format!(
-            "SELECT id FROM {} WHERE {}=? ORDER BY id DESC",
-            table.as_ref(),
-            field.as_ref(),
-        );
-
-        self.query_get_value(sqlx::query(&query).bind(value.as_ref()))
-            .await
-            .map(|id| id.unwrap_or_default())
-    }
-
-    /// Fetches the rowid by restricting the rows through two different key, value settings.
-    pub async fn get_rowid2(
-        &self,
-        table: impl AsRef<str>,
-        field: impl AsRef<str>,
-        value: i64,
-        field2: impl AsRef<str>,
-        value2: i64,
-    ) -> Result<i64> {
-        let query = format!(
-            "SELECT id FROM {} WHERE {}={} AND {}={} ORDER BY id DESC",
-            table.as_ref(),
-            field.as_ref(),
-            value,
-            field2.as_ref(),
-            value2,
-        );
-
-        self.query_get_value(sqlx::query(&query))
-            .await
-            .map(|id| id.unwrap_or_default())
-    }
 }
 
 pub async fn housekeeping(context: &Context) -> Result<()> {

src/test_utils.rs

@@ -660,7 +660,7 @@ fn receive_event(event: Event) {
     }
 }
 
-/// Logs and individual message to stdout.
+/// Logs an individual message to stdout.
 ///
 /// This includes a bunch of the message meta-data as well.
 async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {