factor out imex tracking

CHANGELOG.md

@@ -1,14 +1,5 @@
 # Changelog 
 
-## 1.26.0
-
-- change generated key type back to RSA as shipped versions
-  have problems to encrypt to Ed25519 keys
-
-- update rPGP to encrypt reliably to Ed25519 keys;
-  one of the next versions can finally use Ed25519 keys then
-
-
 ## 1.25.0
 
 - save traffic by downloading only messages that are really displayed #1236

Cargo.lock

@@ -627,7 +627,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.26.0"
+version = "1.25.0"
 dependencies = [
  "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-native-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -659,7 +659,7 @@ dependencies = [
  "num-derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "pgp 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pgp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "proptest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -693,9 +693,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.26.0"
+version = "1.25.0"
 dependencies = [
- "deltachat 1.26.0",
+ "deltachat 1.25.0",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1820,7 +1820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "pgp"
-version = "0.4.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "aes 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1841,6 +1841,7 @@ dependencies = [
  "des 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1858,7 +1859,6 @@ dependencies = [
  "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "thiserror 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "twofish 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "x25519-dalek 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3384,7 +3384,7 @@ dependencies = [
 "checksum parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc"
 "checksum parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
-"checksum pgp 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "43b7d0c384368631ebc10d9484a71cdc728966dfb25fd82cb1d51b19dda3ec2e"
+"checksum pgp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b16d0ab0f228058bcd3f20a1ee0da6b443ff142988693247a08b799b6c520fb"
 "checksum pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c"
 "checksum pin-project-internal 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f"
 "checksum pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.26.0"
+version = "1.25.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.26.0"
+version = "1.25.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

python/src/deltachat/account.py

@@ -38,7 +38,7 @@ class Account(object):
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        self._evlogger = EventLogger(self, logid, debug)
+        self._evlogger = EventLogger(self._dc_context, logid, debug)
         self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
 
         # register event call back and initialize plugin system
@@ -386,7 +386,15 @@ class Account(object):
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             if not self._threads.is_started():
                 lib.dc_perform_imap_jobs(self._dc_context)
-            return imex_tracker.wait_finish()
+            files_written = []
+            while True:
+                ev = imex_tracker.get()
+                if isinstance(ev, str):
+                    files_written.append(ev)
+                elif isinstance(ev, bool):
+                    if not ev:
+                        raise ValueError("export failed, exp-files: {}".format(files_written))
+                    return files_written
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
@@ -408,7 +416,8 @@ class Account(object):
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             if not self._threads.is_started():
                 lib.dc_perform_imap_jobs(self._dc_context)
-            imex_tracker.wait_finish()
+            if not imex_tracker.get():
+                raise ValueError("import from path '{}' failed".format(path))
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -511,7 +520,7 @@ class Account(object):
             deltachat.clear_context_callback(self._dc_context)
             del self._dc_context
             atexit.unregister(self.shutdown)
-        self.pluggy.unregister(self._evlogger)
+        self.pluggy.unregister(self)
 
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
@@ -533,6 +542,16 @@ class ImexTracker:
         self._imex_events = Queue()
         self.account = account
 
+    @hookimpl
+    def process_low_level_event(self, account, event_name, data1, data2):
+        # there could be multiple accounts instantiated
+        if self.account is not account:
+            return
+        method = getattr(self, "on_" + event_name.lower(), None)
+        if method is not None:
+            print("*** on_ -> ", event_name.lower())
+            method(data1, data2)
+
     def __enter__(self):
         self.account.pluggy.register(self)
         return self
@@ -540,27 +559,17 @@ class ImexTracker:
     def __exit__(self, *args):
         self.account.pluggy.unregister(self)
 
-    @hookimpl
-    def process_low_level_event(self, account, event_name, data1, data2):
-        # there could be multiple accounts instantiated
-        if self.account is not account:
-            return
-        if event_name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(data1)
-        elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(data1)
+    def get(self, timeout=60):
+        return self._imex_events.get(timeout=timeout)
 
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ValueError("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
+    def on_dc_event_imex_progress(self, data1, data2):
+        if data1 == 1000:
+            self._imex_events.put(True)
+        elif data1 == 0:
+            self._imex_events.put(False)
+
+    def on_dc_event_imex_file_written(self, data1, data2):
+        self._imex_events.put(data1)
 
 
 class IOThreads:

python/src/deltachat/eventlogger.py

@@ -8,28 +8,27 @@ from .hookspec import hookimpl
 class EventLogger:
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, debug=True):
-        self.account = account
+    def __init__(self, dc_context, logid=None, debug=True):
+        self._dc_context = dc_context
         self._event_queue = Queue()
         self._debug = debug
         if logid is None:
-            logid = str(self.account._dc_context).strip(">").split()[-1]
+            logid = str(self._dc_context).strip(">").split()[-1]
         self.logid = logid
         self._timeout = None
         self.init_time = time.time()
 
     @hookimpl
-    def process_low_level_event(self, account, event_name, data1, data2):
-        if self.account == account:
-            self._log_event(event_name, data1, data2)
-            self._event_queue.put((event_name, data1, data2))
+    def process_low_level_event(self, event_name, data1, data2):
+        self._log_event(event_name, data1, data2)
+        self._event_queue.put((event_name, data1, data2))
 
     def set_timeout(self, timeout):
         self._timeout = timeout
 
     def consume_events(self, check_error=True):
         while not self._event_queue.empty():
-            self.get(check_error=check_error)
+            self.get()
 
     def get(self, timeout=None, check_error=True):
         timeout = timeout or self._timeout

python/src/deltachat/hookspec.py

@@ -1,25 +0,0 @@
-""" Hooks for python bindings """
-
-import pluggy
-
-name = "deltachat"
-
-hookspec = pluggy.HookspecMarker(name)
-hookimpl = pluggy.HookimplMarker(name)
-_plugin_manager = None
-
-
-def get_plugin_manager():
-    global _plugin_manager
-    if _plugin_manager is None:
-        _plugin_manager = pluggy.PluginManager(name)
-        _plugin_manager.add_hookspecs(DeltaChatHookSpecs)
-    return _plugin_manager
-
-
-class DeltaChatHookSpecs:
-    """ Plugin Hook specifications for Python bindings to Delta Chat CFFI. """
-
-    @hookspec
-    def process_low_level_event(self, account, event_name, data1, data2):
-        """ process a CFFI low level events for a given account. """

python/tests/test_lowlevel.py

@@ -2,6 +2,7 @@ from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
 from deltachat.capi import ffi
 from deltachat.capi import lib
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -17,13 +18,21 @@ def test_callback_None2int():
 
 
 def test_dc_close_events(tmpdir):
-    from deltachat.account import Account
+    ctx = ffi.gc(
+        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    evlog = EventLogger(ctx)
+    evlog.set_timeout(5)
+    set_context_callback(
+        ctx,
+        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
+    )
     p = tmpdir.join("hello.db")
-    ac1 = Account(p.strpath)
-    ac1.shutdown()
+    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+    capi.lib.dc_close(ctx)
 
     def find(info_string):
-        evlog = ac1._evlogger
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
             data2 = ev[2]

src/dc_receive_imf.rs

@@ -1464,7 +1464,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
         return false;
     }
 
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = mailparse::addrparse(mid_list) {
         for id in ids.iter() {
             if is_known_rfc724_mid(context, id) {
                 return true;
@@ -1476,7 +1476,8 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
 }
 
 /// Check if a message is a reply to a known message (messenger or non-messenger).
-fn is_known_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
+fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
+    let addr = extract_single_from_addr(rfc724_mid);
     context
         .sql
         .exists(
@@ -1484,7 +1485,7 @@ fn is_known_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
              LEFT JOIN chats c ON m.chat_id=c.id  \
              WHERE m.rfc724_mid=?  \
              AND m.chat_id>9 AND c.blocked=0;",
-            params![rfc724_mid],
+            params![addr],
         )
         .unwrap_or_default()
 }
@@ -1511,7 +1512,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -
 }
 
 pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = mailparse::addrparse(mid_list) {
         for id in ids.iter() {
             if is_msgrmsg_rfc724_mid(context, id) {
                 return true;
@@ -1521,13 +1522,21 @@ pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -
     false
 }
 
+fn extract_single_from_addr(addr: &mailparse::MailAddr) -> &String {
+    match addr {
+        mailparse::MailAddr::Group(infos) => &infos.addrs[0].addr,
+        mailparse::MailAddr::Single(info) => &info.addr,
+    }
+}
+
 /// Check if a message is a reply to any messenger message.
-fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
+fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
+    let addr = extract_single_from_addr(rfc724_mid);
     context
         .sql
         .exists(
             "SELECT id FROM msgs  WHERE rfc724_mid=?  AND msgrmsg!=0  AND chat_id>9;",
-            params![rfc724_mid],
+            params![addr],
         )
         .unwrap_or_default()
 }

src/imap/mod.rs

@@ -1306,6 +1306,17 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     }
 }
 
+fn parse_message_id(value: &str) -> crate::error::Result<String> {
+    let addrs = mailparse::addrparse(value)
+        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+
+    if let Some(info) = addrs.extract_single_info() {
+        return Ok(info.addr);
+    }
+
+    bail!("could not parse message_id: {}", value);
+}
+
 fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
     let header_bytes = match prefetch_msg.header() {
         Some(header_bytes) => header_bytes,
@@ -1317,7 +1328,7 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>>
 
 fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String> {
     if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId)? {
-        Ok(crate::mimeparser::parse_message_id(&message_id)?)
+        Ok(parse_message_id(&message_id)?)
     } else {
         Err(Error::Other("prefetch: No message ID found".to_string()))
     }
@@ -1373,3 +1384,20 @@ fn prefetch_should_download(
     let show = show && !blocked_contact;
     Ok(show)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_message_id() {
+        assert_eq!(
+            parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+        assert_eq!(
+            parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+    }
+}

src/mimeparser.rs

@@ -741,7 +741,7 @@ impl MimeMessage {
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
         self.get(HeaderDef::MessageId)
-            .and_then(|msgid| parse_message_id(msgid).ok())
+            .and_then(|msgid| parse_message_id(msgid))
     }
 
     fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
@@ -779,16 +779,14 @@ impl MimeMessage {
                 .get_header_value(HeaderDef::OriginalMessageId)
                 .ok()
                 .flatten()
-                .and_then(|v| parse_message_id(&v).ok())
+                .and_then(|v| parse_message_id(&v))
             {
                 let additional_message_ids = report_fields
                     .get_header_value(HeaderDef::AdditionalMessageIds)
                     .ok()
                     .flatten()
                     .map_or_else(Vec::new, |v| {
-                        v.split(' ')
-                            .filter_map(|s| parse_message_id(s).ok())
-                            .collect()
+                        v.split(' ').filter_map(parse_message_id).collect()
                     });
 
                 return Ok(Some(Report {
@@ -911,20 +909,14 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
-pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
-    let ids = mailparse::msgidparse(value)
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
-
-    if ids.len() == 1 {
-        let id = &ids[0];
-        if id.starts_with('<') && id.ends_with('>') {
-            Ok(id.chars().skip(1).take(id.len() - 2).collect())
-        } else {
-            bail!("message-ID {} is not enclosed in < and >", value);
+fn parse_message_id(field: &str) -> Option<String> {
+    if let Ok(addrs) = mailparse::addrparse(field) {
+        // Assume the message id is a single id in the form of <id>
+        if let mailparse::MailAddr::Single(mailparse::SingleInfo { ref addr, .. }) = addrs[0] {
+            return Some(addr.clone());
         }
-    } else {
-        bail!("could not parse message_id: {}", value);
     }
+    None
 }
 
 fn is_known(key: &str) -> bool {

src/pgp.rs

@@ -153,8 +153,8 @@ pub(crate) fn create_keypair(
     keygen_type: KeyGenType,
 ) -> std::result::Result<KeyPair, PgpKeygenError> {
     let (secret_key_type, public_key_type) = match keygen_type {
-        KeyGenType::Rsa2048 | KeyGenType::Default => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
-        KeyGenType::Ed25519 => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
+        KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
+        KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
     };
 
     let user_id = format!("<{}>", addr);

src/provider/data.rs

@@ -22,9 +22,9 @@ lazy_static::lazy_static! {
 
     // aol.md: aol.com
     static ref P_AOL: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "To log in to AOL with Delta Chat, you need to set up an app password in the AOL web interface.",
-        after_login_hint: "",
+        status: Status::BROKEN,
+        before_login_hint: "You can't use Delta Chat to login to AOL.",
+        after_login_hint: "Seems like you logged in anyway, congratulations! 🎉 Feel free to tell us at https://github.com/deltachat/provider-db/issues that AOL works again.",
         overview_page: "https://providers.delta.chat/aol",
         server: vec![
         ],