try add a little testing of imex events

CHANGELOG.md

@@ -1,73 +1,10 @@
 # Changelog 
 
-## 1.47.0
+## Unreleased
 
-- breaking change: `dc_update_device_chats()` removed;
-  this is now done automatically during configure
-  unless the new config-option `bot` is set #1957
-
-- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
-  and remove `dc_marknoticed_all_chats()` #1942 #1981
-
-- breaking change: remove unused starring options #1965
-
-- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
-  `dc_chat_is_protected()`; also single-chats may be protected now, this may
-  happen over the wire even if the UI do not offer an option for that #1968
-
-- breaking change: split quotes off message text,
-  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
-
-- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
-  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
-
-- require quorum to enable encryption #1946
-
-- speed up and clean up account creation #1912 #1927 #1960 #1961
-
-- configure now collects recent contacts and fetches last messages
-  unless disabled by `fetch_existing` config-option #1913 #2003
-
-- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
-  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
-
-- add `dc_set_chat_protection()`; the `protect` parameter in
-  `dc_create_group_chat()` will be removed in an upcoming release;
-  up to then, UIs using the "verified group" paradigm
-  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
-
-- remove unneeded `DC_STR_COUNT` #1991
-
-- mark all failed messages as failed when receiving an NDN #1993
-
-- fix import temporary directory usage #1929
-
-- fix forcing encryption for reset peers #1998
-
-- fix: do not allow to save drafts in non-writeable chats #1997
-
-- fix: do not show HTML if there is no content and there is an attachment #1988
-
-- fix recovering offline/lost connections, fixes background receive bug #1983
-
-- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
-
-- fix whitespace for summaries #1938
-
-- fix: improve sentbox name guessing #1941
-
-- fix: avoid manual poll impl for accounts events #1944
-
-- fix encoding newlines in param as a preparation for storing quotes #1945
-
-- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
-
-- fix ci #1928 #1931 #1932 #1933 #1934 #1943
-
-- update provider-database #1940 #2005 #2006
-
-- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
+- breaking change: `dc_update_device_chats()` was removed. This is now done automatically during configure.
 
+- Added a `bot` config. Currently, it only prevents filling the device chats automatically.
 
 ## 1.46.0
 
@@ -814,3 +751,4 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

Cargo.lock

@@ -1003,7 +1003,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.47.0"
+version = "1.46.0"
 dependencies = [
  "ansi_term 0.12.1",
  "anyhow",
@@ -1079,7 +1079,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.47.0"
+version = "1.46.0"
 dependencies = [
  "anyhow",
  "async-std",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.47.0"
+version = "1.46.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.47.0"
+version = "1.46.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

python/src/deltachat/account.py

@@ -263,17 +263,6 @@ class Account(object):
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self):
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 

python/src/deltachat/contact.py

@@ -52,17 +52,9 @@ class Contact(object):
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        """ Block or unblock a contact. """
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/tracker.py

@@ -20,6 +20,14 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
+    def wait_progress(self, target_progress, progress_timeout=60):
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, int) and ev >= target_progress:
+                return ev
+            if ev == 1000 or ev == 0:
+                return None
+
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/test_account.py

@@ -6,6 +6,7 @@ import queue
 import time
 from deltachat import const, Account
 from deltachat.message import Message
+from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
 from datetime import datetime, timedelta
 
@@ -129,20 +130,6 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
-    def test_get_blocked(self, acfactory):
-        ac1 = acfactory.get_configured_offline_account()
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some2@example.org", name="some2")
-        ac1.create_contact("some3@example.org", name="some3")
-        assert ac1.get_blocked_contacts() == []
-        contact1.block()
-        assert ac1.get_blocked_contacts() == [contact1]
-        contact2.block()
-        blocked = ac1.get_blocked_contacts()
-        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
-        contact2.unblock()
-        assert ac1.get_blocked_contacts() == [contact1]
-
     def test_create_self_contact(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact(ac1.get_config("addr"))
@@ -1259,8 +1246,16 @@ class TestOnlineAccount:
         backupdir = tmpdir.mkdir("backup")
 
         lp.sec("export all to {}".format(backupdir))
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
+        with ac1.temp_plugin(ImexTracker()) as imex_tracker:
+            path = ac1.export_all(backupdir.strpath)
+            assert os.path.exists(path)
+
+        # check progress events for export
+        imex_tracker.wait_progress(250)
+        imex_tracker.wait_progress(500)
+        imex_tracker.wait_progress(750)
+        imex_tracker.wait_progress(1000)
+        # return mex_tracker.wait_finish()
         t = time.time()
 
         lp.sec("get fresh empty account")
@@ -1271,7 +1266,15 @@ class TestOnlineAccount:
         assert path2 == path
 
         lp.sec("import backup and check it's proper")
-        ac2.import_all(path)
+        with ac2.temp_plugin(ImexTracker()) as imex_tracker:
+            ac2.import_all(path)
+
+        # check progress events for import
+        imex_tracker.wait_progress(250)
+        imex_tracker.wait_progress(500)
+        imex_tracker.wait_progress(750)
+        imex_tracker.wait_progress(1000)
+
         contacts = ac2.get_contacts(query="some1")
         assert len(contacts) == 1
         contact2 = contacts[0]

src/configure/mod.rs

@@ -163,8 +163,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
-    //let ctx2 = ctx.clone();
-    //let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
@@ -217,32 +217,28 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 500);
 
-    let mut servers = param_autoconfig.unwrap_or_default();
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::IMAP)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::IMAP,
-            hostname: param.imap.server.clone(),
-            port: param.imap.port,
-            socket: param.imap.security,
-            username: param.imap.user.clone(),
-        })
-    }
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::SMTP)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::SMTP,
-            hostname: param.smtp.server.clone(),
-            port: param.smtp.port,
-            socket: param.smtp.security,
-            username: param.smtp.user.clone(),
-        })
-    }
-    let servers = expand_param_vector(servers, &param.addr, &param_domain);
+    let servers = expand_param_vector(
+        param_autoconfig.unwrap_or_else(|| {
+            vec![
+                ServerParams {
+                    protocol: Protocol::IMAP,
+                    hostname: param.imap.server.clone(),
+                    port: param.imap.port,
+                    socket: param.imap.security,
+                    username: param.imap.user.clone(),
+                },
+                ServerParams {
+                    protocol: Protocol::SMTP,
+                    hostname: param.smtp.server.clone(),
+                    port: param.smtp.port,
+                    socket: param.smtp.security,
+                    username: param.smtp.user.clone(),
+                },
+            ]
+        }),
+        &param.addr,
+        &param_domain,
+    );
 
     progress!(ctx, 550);
 
@@ -364,7 +360,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     .await;
 
     progress!(ctx, 940);
-    //update_device_chats_handle.await?;
+    update_device_chats_handle.await?;
 
     Ok(())
 }
@@ -564,9 +560,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
     let first_err = if let Some(f) = errors.first() {
         f
     } else {
-        // This means configuration failed but no errors have been captured. This should never
-        // happen, but if it does, the user will see classic "Error: no error".
-        return "no error".to_string();
+        return "".to_string();
     };
 
     if errors

src/imex.rs

@@ -503,10 +503,20 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
 
     let backup_file = File::open(backup_to_import).await?;
+    let file_size = backup_file.metadata().await?.len();
     let archive = Archive::new(backup_file);
+
     let mut entries = archive.entries()?;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
+
+        let current_pos = f.raw_file_position();
+        let progress = 1000 * current_pos / file_size;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            context.emit_event(EventType::ImexProgress(progress as usize));
+        }
+
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
@@ -515,7 +525,6 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
                 context.get_dbfile(),
             )
             .await?;
-            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -715,11 +724,32 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
         .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
-    context.emit_event(EventType::ImexProgress(500));
+    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let count = read_dir.len();
+    let mut written_files = 0;
 
-    builder
-        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
-        .await?;
+    for entry in read_dir.into_iter() {
+        let entry = entry?;
+        let name = entry.file_name();
+        if !entry.file_type().await?.is_file() {
+            warn!(
+                context,
+                "Export: Found dir entry {} that is not a file, ignoring",
+                name.to_string_lossy()
+            );
+            continue;
+        }
+        let mut file = File::open(entry.path()).await?;
+        let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(name);
+        builder.append_file(path_in_archive, &mut file).await?;
+
+        written_files += 1;
+        let progress = 1000 * written_files / count;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            emit_event!(context, EventType::ImexProgress(progress));
+        }
+    }
 
     builder.finish().await?;
     Ok(())

src/job.rs

@@ -627,9 +627,6 @@ impl Job {
     /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
     /// and show them in the chat list.
     async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if context.get_config_bool(Config::Bot).await {
-            return Status::Finished(Ok(())); // Bots don't want those messages
-        }
         if let Err(err) = imap.connect_configured(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;

src/message.rs

@@ -1638,16 +1638,14 @@ pub(crate) async fn handle_ndn(
     context: &Context,
     failed: &FailureReport,
     error: Option<impl AsRef<str>>,
-) -> anyhow::Result<()> {
+) {
     if failed.rfc724_mid.is_empty() {
-        return Ok(());
+        return;
     }
 
-    // The NDN might be for a message-id that had attachments and was sent from a non-Delta Chat client.
-    // In this case we need to mark multiple "msgids" as failed that all refer to the same message-id.
-    let msgs: Vec<_> = context
+    let res = context
         .sql
-        .query_map(
+        .query_row(
             concat!(
                 "SELECT",
                 "    m.id AS msg_id,",
@@ -1664,42 +1662,37 @@ pub(crate) async fn handle_ndn(
                     row.get::<_, Chattype>("type")?,
                 ))
             },
-            |rows| Ok(rows.collect::<Vec<_>>()),
         )
-        .await?;
-
-    for (i, msg) in msgs.into_iter().enumerate() {
-        let (msg_id, chat_id, chat_type) = msg?;
-        set_msg_failed(context, msg_id, error.as_ref()).await;
-        if i == 0 {
-            // Add only one info msg for all failed messages
-            ndn_maybe_add_info_msg(context, failed, chat_id, chat_type).await?;
-        }
+        .await;
+    if let Err(ref err) = res {
+        info!(context, "Failed to select NDN {:?}", err);
     }
 
-    Ok(())
-}
+    if let Ok((msg_id, chat_id, chat_type)) = res {
+        set_msg_failed(context, msg_id, error).await;
 
-async fn ndn_maybe_add_info_msg(
-    context: &Context,
-    failed: &FailureReport,
-    chat_id: ChatId,
-    chat_type: Chattype,
-) -> anyhow::Result<()> {
-    if chat_type == Chattype::Group {
-        if let Some(failed_recipient) = &failed.failed_recipient {
-            let contact_id =
-                Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
-            let contact = Contact::load_from_db(context, contact_id).await?;
-            // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
-            let text = context
-                .stock_string_repl_str(StockMessage::FailedSendingTo, contact.get_display_name())
-                .await;
-            chat::add_info_msg(context, chat_id, text).await;
-            context.emit_event(EventType::ChatModified(chat_id));
+        if chat_type == Chattype::Group {
+            if let Some(failed_recipient) = &failed.failed_recipient {
+                let contact_id =
+                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+                    chat::add_info_msg(
+                        context,
+                        chat_id,
+                        context
+                            .stock_string_repl_str(
+                                StockMessage::FailedSendingTo,
+                                contact.get_display_name(),
+                            )
+                            .await,
+                    )
+                    .await;
+                    context.emit_event(EventType::ChatModified(chat_id));
+                }
+            }
         }
     }
-    Ok(())
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)

src/mimeparser.rs

@@ -1071,9 +1071,7 @@ impl MimeMessage {
                 .iter()
                 .find(|p| p.typ == Viewtype::Text)
                 .map(|p| p.msg.clone());
-            if let Err(e) = message::handle_ndn(context, failure_report, error).await {
-                warn!(context, "Could not handle ndn: {}", e);
-            }
+            message::handle_ndn(context, failure_report, error).await
         }
     }