stop timings

* Fix #2335 (delete job flooding)

The problem was:

- You are offline, and an ephemeral message is due to delete.
- load_imap_deletion_job() is called and returns a deletion job for it.
- The job fails because, well, we are offline.
- The job is saved into the database.
- load_imap_deletion_job() is called again, and so on.

* Add test

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
     executor: doxygen
     steps:
       - checkout
-      - run: bash ci_scripts/run-doxygen.sh
+      - run: bash scripts/run-doxygen.sh
       - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
@@ -27,7 +27,7 @@ jobs:
       - checkout
       # the following commands on success produces
       # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
+      - run: bash scripts/remote_python_packaging.sh
       - persist_to_workspace:
           root: workspace
           paths:
@@ -42,7 +42,7 @@ jobs:
       - attach_workspace:
           at: workspace
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
 workflows:
   version: 2.1

.github/workflows/remote_tests.yml

@@ -18,4 +18,4 @@ jobs:
         shell: bash
         env:
           SSH_KEY: ${{ secrets.SSH_KEY }}
-      - run: ci_scripts/remote_tests_python.sh
+      - run: scripts/remote_tests_python.sh

Cargo.toml

@@ -103,6 +103,14 @@ required-features = ["repl"]
 name = "create_account"
 harness = false
 
+[[bench]]
+name = "contacts"
+harness = false
+
+[[bench]]
+name = "search_msgs"
+harness = false
+
 [features]
 default = []
 internals = []

README.md

@@ -2,7 +2,9 @@
 
 > Deltachat-core written in Rust 
 
-[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
+[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![Remote tests](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -111,11 +113,6 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
-[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
-[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
-[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
-[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
 ## Language bindings and frontend projects
 
 Language bindings are available for:

benches/contacts.rs

@@ -0,0 +1,39 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use tempfile::tempdir;
+
+async fn address_book_benchmark(n: u32, read_count: u32) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    let book = (0..n)
+        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .collect::<Vec<String>>()
+        .join("");
+
+    Contact::add_address_book(&context, book).await.unwrap();
+
+    let query: Option<&str> = None;
+    for _ in 0..read_count {
+        Contact::get_all(&context, 0, query).await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 500 contacts", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+    });
+
+    c.bench_function("create 100 contacts and read it 1000 times", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -0,0 +1,29 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::context::Context;
+use std::path::Path;
+
+async fn search_benchmark(path: impl AsRef<Path>) {
+    let dbfile = path.as_ref();
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    for _ in 0..10u32 {
+        context.search_msgs(None, "hello").await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        c.bench_function("search hello", |b| {
+            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+        });
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/deltachat.h

@@ -1272,6 +1272,12 @@ uint32_t dc_get_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id);
  * search results may just hilite the corresponding messages and present a
  * prev/next button.
  *
+ * For global search, result is limited to 1000 messages,
+ * this allows incremental search done fast.
+ * So, when getting exactly 1000 results, the result may be truncated;
+ * the UIs may display sth. as "1000+ messages found" in this case.
+ * Chat search (if a chat_id is set) is not limited.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id ID of the chat to search messages in.

examples/repl/cmdline.rs

@@ -525,9 +525,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = std::time::SystemTime::now()
-                .duration_since(time_start)
-                .unwrap_or_default();
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -604,7 +602,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
+            let time_start = std::time::SystemTime::now();
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
+
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -659,7 +660,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "{} messages.",
                 sel_chat.get_id().get_msg_cnt(&context).await?
             );
+
+            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
+            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
+
+            println!(
+                "{:?} to create this list, {:?} to mark all messages as noticed.",
+                time_needed, time_noticed_needed
+            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
@@ -898,10 +907,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None
             };
 
+            let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
+            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");

python/README.rst

@@ -142,7 +142,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -151,9 +151,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps scripts/docker_coredeps
 
-This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/src/deltachat/account.py

@@ -89,6 +89,22 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        log("=============== " + self.get_config("displayname") + " ===============")
+        cursor = 0
+        for name, val in self.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 

python/src/deltachat/direct_imap.py

@@ -47,8 +47,9 @@ def dc_account_extra_configure(account):
 
     except Exception as e:
         # Uncaught exceptions here would lead to a timeout without any note written to the log
-        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
-        account.log("===================", e, "===================")
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
 
 
 @deltachat.global_hookimpl
@@ -172,21 +173,6 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()

python/src/deltachat/testplugin.py

@@ -414,13 +414,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
-                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_accepted_chat(self, ac1, ac2):

python/tests/test_account.py

@@ -1572,6 +1572,12 @@ class TestOnlineAccount:
         original_image_path = data.get_path("d.png")
         chat1.send_image(original_image_path)
 
+        # Add another 100KB file that ensures that the progress is smooth enough
+        path = tmpdir.join("attachment.txt")
+        with open(path, "w") as file:
+            file.truncate(100000)
+        chat1.send_file(path.strpath)
+
         def assert_account_is_proper(ac):
             contacts = ac.get_contacts(query="some1")
             assert len(contacts) == 1
@@ -1579,7 +1585,7 @@ class TestOnlineAccount:
             assert contact2.addr == "some1@example.org"
             chat2 = contact2.create_chat()
             messages = chat2.get_messages()
-            assert len(messages) == 2
+            assert len(messages) == 3
             assert messages[0].text == "msg1"
             assert messages[1].filemime == "image/png"
             assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size

scripts/README.md

@@ -30,8 +30,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    ci_scripts/manual_remote_tests.sh rust
-    ci_scripts/manual_remote_tests.sh python
+    scripts/manual_remote_tests.sh rust
+    scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -45,6 +45,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd ci_scripts  # where all CI things are 
+    cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 

scripts/ci_upload.sh

@@ -42,7 +42,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ci_scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
 rsync -avz \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   $WHEELHOUSEDIR \

scripts/manual_remote_tests.sh

@@ -6,4 +6,4 @@ export CIRCLE_BUILD_NUM=$USER
 export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
 export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
 
-time bash ci_scripts/$CIRCLE_JOB.sh 
+time bash scripts/$CIRCLE_JOB.sh 

scripts/remote_python_packaging.sh

@@ -36,7 +36,7 @@ ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     # run everything else inside docker 
     docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 _HERE
 

scripts/remote_tests_python.sh

@@ -42,5 +42,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash ci_scripts/run-python-test.sh 
+    bash scripts/run-python-test.sh 
 _HERE

scripts/remote_tests_rust.sh

@@ -25,6 +25,6 @@ ssh $SSHTARGET <<_HERE
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 
-    bash ci_scripts/run-rust-test.sh
+    bash scripts/run-rust-test.sh
 _HERE
 

scripts/run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   ./run-integration-tests.sh -e py35
+#   scripts/run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   ./run-integration-tests.sh -e py35 -- -x
+#   scripts/run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}

src/chat.rs

@@ -3855,7 +3855,6 @@ mod tests {
             .unwrap();
         let alice_chat = Chat::load_from_db(&alice, alice_chat_id).await.unwrap();
 
-        println!("----- add_contact_to_chat");
         add_contact_to_chat(&alice, alice_chat_id, contact_id).await;
         assert_eq!(
             get_chat_contacts(&alice, alice_chat_id)
@@ -3864,11 +3863,9 @@ mod tests {
                 .len(),
             2
         );
-        println!("----- send_text_msg");
         send_text_msg(&alice, alice_chat_id, "hi!".to_string())
             .await
             .ok();
-        println!("----- get_chat_msgs");
         assert_eq!(
             get_chat_msgs(&alice, alice_chat_id, 0, None)
                 .await
@@ -3877,7 +3874,6 @@ mod tests {
             1
         );
 
-        println!("----- pop_sent_msg");
         // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com).
         let msg = alice.pop_sent_msg().await.payload();
         assert_eq!(msg.match_indices("Gr.").count(), 2);
@@ -3890,11 +3886,9 @@ mod tests {
             .unwrap();
         let msg = bob.get_last_msg().await;
 
-        println!("load from dbb");
         let bob_chat = Chat::load_from_db(&bob, msg.chat_id).await.unwrap();
         assert_eq!(bob_chat.grpid, alice_chat.grpid);
 
-        println!("chat loaded");
         // Bob answers - simulate a normal MUA by not setting `Chat-*`-headers;
         // moreover, Bob's SMTP-server also replaces the `Message-ID:`-header
         send_text_msg(&bob, bob_chat.id, "ho!".to_string())
@@ -3905,12 +3899,10 @@ mod tests {
         let msg = msg.replace("Chat-", "XXXX-");
         assert_eq!(msg.match_indices("Chat-").count(), 0);
 
-        println!("last receive start");
         // Alice receives this message - she can still detect the group by the `References:`-header
         dc_receive_imf(&alice, msg.as_bytes(), "INBOX", 2, false)
             .await
             .unwrap();
-        println!("----- last receie if");
         let msg = alice.get_last_msg().await;
         assert_eq!(msg.chat_id, alice_chat_id);
         assert_eq!(msg.text, Some("ho!".to_string()));

src/configure/server_params.rs

@@ -25,7 +25,7 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
@@ -42,7 +42,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
         if self.hostname.is_empty() {
             self.hostname = param_domain.to_string();
@@ -62,7 +62,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+    fn expand_ports(mut self) -> Vec<ServerParams> {
         // Try to infer port from socket security.
         if self.port == 0 {
             self.port = match self.socket {
@@ -160,5 +160,37 @@ mod tests {
                 username: "foobar".to_string(),
             }],
         );
+
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Automatic,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string()
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string()
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -310,16 +310,6 @@ impl Default for Viewtype {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -328,3 +318,100 @@ pub enum KeyType {
     Public = 0,
     Private = 1,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_chattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Chattype::Undefined, Chattype::default());
+        assert_eq!(Chattype::Undefined, Chattype::from_i32(0).unwrap());
+        assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
+        assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
+        assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+    }
+
+    #[test]
+    fn test_keygentype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyGenType::Default, KeyGenType::default());
+        assert_eq!(KeyGenType::Default, KeyGenType::from_i32(0).unwrap());
+        assert_eq!(KeyGenType::Rsa2048, KeyGenType::from_i32(1).unwrap());
+        assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_keytype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap());
+        assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_showemails_values() {
+        // values may be written to disk and must not change
+        assert_eq!(ShowEmails::Off, ShowEmails::default());
+        assert_eq!(ShowEmails::Off, ShowEmails::from_i32(0).unwrap());
+        assert_eq!(
+            ShowEmails::AcceptedContacts,
+            ShowEmails::from_i32(1).unwrap()
+        );
+        assert_eq!(ShowEmails::All, ShowEmails::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_blocked_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Blocked::Not, Blocked::default());
+        assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
+        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_mediaquality_values() {
+        // values may be written to disk and must not change
+        assert_eq!(MediaQuality::Balanced, MediaQuality::default());
+        assert_eq!(MediaQuality::Balanced, MediaQuality::from_i32(0).unwrap());
+        assert_eq!(MediaQuality::Worse, MediaQuality::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_videochattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(VideochatType::Unknown, VideochatType::default());
+        assert_eq!(VideochatType::Unknown, VideochatType::from_i32(0).unwrap());
+        assert_eq!(
+            VideochatType::BasicWebrtc,
+            VideochatType::from_i32(1).unwrap()
+        );
+        assert_eq!(VideochatType::Jitsi, VideochatType::from_i32(2).unwrap());
+    }
+}

src/context.rs

@@ -471,7 +471,6 @@ impl Context {
             return Ok(Vec::new());
         }
         let str_like_in_text = format!("%{}%", real_query);
-        let str_like_beg = format!("{}%", real_query);
 
         let list = if let Some(chat_id) = chat_id {
             self.sql
@@ -484,12 +483,11 @@ impl Context {
                  WHERE m.chat_id=?
                    AND m.hidden=0
                    AND ct.blocked=0
-                   AND (txt LIKE ? OR ct.name LIKE ?)
+                   AND txt LIKE ?
                  ORDER BY m.timestamp,m.id;",
                     )
                     .bind(chat_id)
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
+                    .bind(str_like_in_text),
                 )
                 .await?
                 .map(|row| {
@@ -500,6 +498,16 @@ impl Context {
                 .collect::<sqlx::Result<Vec<MsgId>>>()
                 .await?
         } else {
+            // For performance reasons results are sorted only by `id`, that is in the order of
+            // message reception.
+            //
+            // Unlike chat view, sorting by `timestamp` is not necessary but slows down the query by
+            // ~25% according to benchmarks.
+            //
+            // To speed up incremental search, where queries for few characters usually return lots
+            // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
+            // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
+            // The limit is documented and UI may add a hint when getting 1000 results.
             self.sql
                 .fetch(
                     sqlx::query(
@@ -513,11 +521,10 @@ impl Context {
                    AND m.hidden=0
                    AND c.blocked=0
                    AND ct.blocked=0
-                   AND (m.txt LIKE ? OR ct.name LIKE ?)
-                 ORDER BY m.timestamp DESC,m.id DESC;",
+                   AND m.txt LIKE ?
+                 ORDER BY m.id DESC LIMIT 1000",
                     )
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
+                    .bind(str_like_in_text),
                 )
                 .await?
                 .map(|row| {
@@ -605,9 +612,14 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::chat::{get_chat_contacts, get_chat_msgs, set_muted, Chat, MuteDuration};
+    use crate::chat::{
+        create_by_contact_id, get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat,
+        MuteDuration,
+    };
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::message::Message;
     use crate::test_utils::TestContext;
     use std::time::Duration;
     use strum::IntoEnumIterator;
@@ -879,4 +891,93 @@ mod tests {
             }
         }
     }
+
+    #[async_std::test]
+    async fn test_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let self_talk = create_by_contact_id(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Global search finds nothing.
+        let res = alice.search_msgs(None, "foo").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds nothing.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert!(res.is_empty());
+
+        // Add messages to chat with Bob.
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.set_text(Some("foobar".to_string()));
+        send_msg(&alice, chat.id, &mut msg1).await?;
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_text(Some("barbaz".to_string()));
+        send_msg(&alice, chat.id, &mut msg2).await?;
+
+        // Global search with a part of text finds the message.
+        let res = alice.search_msgs(None, "ob").await?;
+        assert_eq!(res.len(), 1);
+
+        // Global search for "bar" matches both "foobar" and "barbaz".
+        let res = alice.search_msgs(None, "bar").await?;
+        assert_eq!(res.len(), 2);
+
+        // Message added later is returned first.
+        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.get(1), Some(&msg1.id));
+
+        // Global search with longer text does not find any message.
+        let res = alice.search_msgs(None, "foobarbaz").await?;
+        assert!(res.is_empty());
+
+        // Search for random string finds nothing.
+        let res = alice.search_msgs(None, "abc").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds the message.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1);
+
+        // Search in Saved Messages does not find the message.
+        let res = alice.search_msgs(Some(self_talk), "foo").await?;
+        assert!(res.is_empty());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_limit_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Add 999 messages
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foobar".to_string()));
+        for _ in 0..999 {
+            send_msg(&alice, chat.id, &mut msg).await?;
+        }
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 999);
+
+        // Add one more message, no limit yet
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // Add one more message, that one is truncated then
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // In-chat should not be not limited
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1001);
+
+        Ok(())
+    }
 }

src/dc_receive_imf.rs

@@ -71,11 +71,7 @@ pub(crate) async fn dc_receive_imf_inner(
     info!(
         context,
         "Receiving message {}/{}, seen={}...",
-        if !server_folder.as_ref().is_empty() {
-            server_folder.as_ref()
-        } else {
-            "?"
-        },
+        server_folder.as_ref(),
         server_uid,
         seen
     );

src/dc_tools.rs

@@ -879,16 +879,6 @@ mod tests {
         }
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
-        if dc_file_exist!(context, "$BLOBDIR/foobar").await
-            || dc_file_exist!(context, "$BLOBDIR/dada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar").await;
-            dc_delete_file(context, "$BLOBDIR/dada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
-        }
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());

src/ephemeral.rs

@@ -66,7 +66,6 @@ use async_std::task;
 use serde::{Deserialize, Serialize};
 use sqlx::Row;
 
-use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
@@ -77,6 +76,10 @@ use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
 use crate::stock_str;
+use crate::{
+    chat::{lookup_by_contact_id, send_msg, ChatId},
+    job,
+};
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -490,10 +493,12 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
+         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
             )
             .bind(threshold_timestamp)
-            .bind(now),
+            .bind(now)
+            .bind(job::Action::DeleteMsgOnImap),
         )
         .await?;
 
@@ -537,6 +542,7 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use crate::param::Params;
     use async_std::task::sleep;
 
     use super::*;
@@ -758,7 +764,31 @@ mod tests {
 
         sleep(Duration::from_millis(1100)).await;
 
+        // Check checks that the msg was deleted locally
         check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+
+        // Check that the msg will be deleted on the server
+        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
+        t.sql
+            .execute(sqlx::query("UPDATE msgs SET server_uid=1 WHERE id=?").bind(msg.sender_msg_id))
+            .await
+            .unwrap();
+        let job = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(
+            job,
+            Some(job::Job::new(
+                job::Action::DeleteMsgOnImap,
+                msg.sender_msg_id.to_u32(),
+                Params::new(),
+                0,
+            ))
+        );
+        // Let's assume that executing the job fails on first try and the job is saved to the db
+        job.unwrap().save(&t).await.unwrap();
+
+        // Make sure that we don't get yet another job when loading from db
+        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(job2, None);
     }
 
     async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {

src/imap/scan_folders.rs

@@ -61,14 +61,8 @@ impl Imap {
                 spam_folder = Some(folder.name().to_string());
             }
 
-            if watched_folders.contains(&foldername.to_string()) {
-                info!(
-                    context,
-                    "Not scanning folder {} as it is watched anyway", foldername
-                );
-            } else {
-                info!(context, "Scanning folder: {}", foldername);
-
+            // Don't scan folders that are watched anyway
+            if !watched_folders.contains(&foldername.to_string()) {
                 if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
                     warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
                 }

src/job.rs

@@ -130,7 +130,7 @@ impl From<Action> for Thread {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Job {
     pub job_id: u32,
     pub action: Action,
@@ -1030,7 +1030,7 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
+pub(crate) async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         info!(context, "verbose (issue 2335): loading imap deletion job");
         Some(Job::new(
@@ -1118,7 +1118,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
             if let Err(err) = res {
                 warn!(
                     context,
-                    "{} removes job {} as it failed with error {:?}", &connection, job, err
+                    "{} removes job {} as it failed with error {:#}", &connection, job, err
                 );
             } else {
                 info!(

src/test_utils.rs

@@ -660,7 +660,7 @@ fn receive_event(event: Event) {
     }
 }
 
-/// Logs and individual message to stdout.
+/// Logs an individual message to stdout.
 ///
 /// This includes a bunch of the message meta-data as well.
 async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {