Merge pull request #169 from deltachat/imex_tests

Imex tests and fix for export-blobs

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-alpha.1"
+version = "1.0.0-alpha.2"
 authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
 edition = "2018"
 license = "MPL"

ci_scripts/run_all.sh

@@ -37,7 +37,7 @@ if [ -n "$TESTS" ]; then
     export PYTHONDONTWRITEBYTECODE=1
 
     # run tox
-    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37,auditwheels
+    tox --workdir "$TOXWORKDIR" -e lint,py27,py35,py36,py37,auditwheels
     popd
 fi
 

python/setup.py

@@ -12,7 +12,7 @@ def main():
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
         setup_requires=['cffi>=1.0.0'],
-        install_requires=['cffi>=1.0.0', 'requests', 'attrs', 'six'],
+        install_requires=['cffi>=1.0.0', 'attrs', 'six'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ from deltachat import capi, const
 from deltachat.capi import ffi
 from deltachat.account import Account  # noqa
 
-__version__ = "0.10.0.dev2"
+__version__ = "0.10.0.dev3"
 
 
 _DC_CALLBACK_MAP = {}

python/src/deltachat/account.py

@@ -2,16 +2,14 @@
 
 from __future__ import print_function
 import threading
+import os
 import re
 import time
-import requests
 from array import array
 try:
     from queue import Queue
 except ImportError:
     from Queue import Queue
-import attr
-from attr import validators as v
 
 import deltachat
 from . import const
@@ -41,11 +39,11 @@ class Account(object):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
-        self._evhandler = EventHandler(self._dc_context)
         self._evlogger = EventLogger(self._dc_context, logid)
         deltachat.set_context_callback(self._dc_context, self._process_event)
         self._threads = IOThreads(self._dc_context)
         self._configkeys = self.get_config("sys.config_keys").split()
+        self._imex_completed = threading.Event()
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -182,15 +180,19 @@ class Account(object):
         return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
 
     def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact.
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
         :returns: a :class:`deltachat.chatting.Chat` object.
         """
-        contact_id = getattr(contact, "id", contact)
-        assert isinstance(contact_id, int)
-        chat_id = lib.dc_create_chat_by_contact_id(
-                        self._dc_context, contact_id)
+        if hasattr(contact, "id"):
+            if contact._dc_context != self._dc_context:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
         return Chat(self._dc_context, chat_id)
 
     def create_chat_by_message(self, message):
@@ -200,8 +202,13 @@ class Account(object):
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chatting.Chat` object.
         """
-        msg_id = getattr(message, "id", message)
-        assert isinstance(msg_id, int)
+        if hasattr(message, "id"):
+            if self._dc_context != message._dc_context:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
         chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
         return Chat(self._dc_context, chat_id)
 
@@ -272,6 +279,32 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
+    def export_to_dir(self, backupdir):
+        """return after all delta chat state is exported to a new file in
+        the specified directory.
+        """
+        snap_files = os.listdir(backupdir)
+        self._imex_completed.clear()
+        lib.dc_imex(self._dc_context, 11, as_dc_charpointer(backupdir), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        self._imex_completed.wait()
+        for x in os.listdir(backupdir):
+            if x not in snap_files:
+                return os.path.join(backupdir, x)
+
+    def import_from_file(self, path):
+        """import delta chat state from the specified backup file.
+
+        The account must be in unconfigured state for import to attempted.
+        """
+        assert not self.is_configured(), "cannot import into configured account"
+        self._imex_completed.clear()
+        lib.dc_imex(self._dc_context, 12, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        self._imex_completed.wait()
+
     def start_threads(self):
         """ start IMAP/SMTP threads (and configure account if it hasn't happened).
 
@@ -289,11 +322,15 @@ class Account(object):
     def _process_event(self, ctx, evt_name, data1, data2):
         assert ctx == self._dc_context
         self._evlogger(evt_name, data1, data2)
-        method = getattr(self._evhandler, evt_name.lower(), None)
+        method = getattr(self, "on_" + evt_name.lower(), None)
         if method is not None:
-            return method(data1, data2) or 0
+            method(data1, data2)
         return 0
 
+    def on_dc_event_imex_progress(self, data1, data2):
+        if data1 == 1000:
+            self._imex_completed.set()
+
 
 class IOThreads:
     def __init__(self, dc_context):
@@ -301,8 +338,11 @@ class IOThreads:
         self._thread_quitflag = False
         self._name2thread = {}
 
+    def is_started(self):
+        return len(self._name2thread) > 0
+
     def start(self, imap=True, smtp=True):
-        assert not self._name2thread
+        assert not self.is_started()
         if imap:
             self._start_one_thread("imap", self.imap_thread_run)
         if smtp:
@@ -322,43 +362,19 @@ class IOThreads:
                 thread.join()
 
     def imap_thread_run(self):
-        print ("starting imap thread")
+        print("starting imap thread")
         while not self._thread_quitflag:
             lib.dc_perform_imap_jobs(self._dc_context)
             lib.dc_perform_imap_fetch(self._dc_context)
             lib.dc_perform_imap_idle(self._dc_context)
 
     def smtp_thread_run(self):
-        print ("starting smtp thread")
+        print("starting smtp thread")
         while not self._thread_quitflag:
             lib.dc_perform_smtp_jobs(self._dc_context)
             lib.dc_perform_smtp_idle(self._dc_context)
 
 
-@attr.s
-class EventHandler(object):
-    _dc_context = attr.ib(validator=v.instance_of(ffi.CData))
-
-    def read_url(self, url):
-        try:
-            r = requests.get(url)
-        except requests.ConnectionError:
-            return ''
-        else:
-            return r.content
-
-    def dc_event_http_get(self, data1, data2):
-        url = data1
-        content = self.read_url(url)
-        if not isinstance(content, bytes):
-            content = content.encode("utf8")
-        # we need to return a fresh pointer that the core owns
-        return lib.dupstring_helper(content)
-
-    def dc_event_is_offline(self, data1, data2):
-        return 0  # always online
-
-
 class EventLogger:
     _loglock = threading.RLock()
 

python/tests/test_account.py

@@ -145,6 +145,14 @@ class TestOfflineAccount:
         assert not msg_state.is_out_delivered()
         assert not msg_state.is_out_mdn_received()
 
+    def test_create_chat_by_mssage_id(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        chat = ac1.create_chat_by_contact(contact1)
+        msg = chat.send_text("msg1")
+        assert chat == ac1.create_chat_by_message(msg)
+        assert chat == ac1.create_chat_by_message(msg.id)
+
     def test_message_image(self, acfactory, data, lp):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
@@ -180,6 +188,17 @@ class TestOfflineAccount:
         assert msg.filename.endswith(msg.basename)
         assert msg.filemime == typeout
 
+    def test_create_chat_mismatch(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        ac2 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        with pytest.raises(ValueError):
+            ac2.create_chat_by_contact(contact1)
+        chat1 = ac1.create_chat_by_contact(contact1)
+        msg = chat1.send_text("hello")
+        with pytest.raises(ValueError):
+            ac2.create_chat_by_message(msg)
+
     def test_chat_message_distinctions(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
@@ -202,6 +221,37 @@ class TestOfflineAccount:
         with pytest.raises(ValueError):
             ac1.configure(addr="123@example.org")
 
+    def test_import_export_one_contact(self, acfactory, tmpdir):
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        chat = ac1.create_chat_by_contact(contact1)
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+        assert not backupdir.listdir()
+
+        path = ac1.export_to_dir(backupdir.strpath)
+        assert os.path.exists(path)
+        ac2 = acfactory.get_unconfigured_account()
+        ac2.import_from_file(path)
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@hello.com"
+        chat2 = ac2.create_chat_by_contact(contact2)
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
 
 class TestOnlineAccount:
     def test_one_account_init(self, acfactory):
@@ -228,9 +278,9 @@ class TestOnlineAccount:
         c2 = ac1.create_contact(email=ac2.get_config("addr"))
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
-        #wait_successful_IMAP_SMTP_connection(ac1)
+        wait_successful_IMAP_SMTP_connection(ac1)
         wait_configuration_progress(ac1, 1000)
-        #wait_successful_IMAP_SMTP_connection(ac2)
+        wait_successful_IMAP_SMTP_connection(ac2)
         wait_configuration_progress(ac2, 1000)
 
         msg_out = chat.send_text("message1")
@@ -369,3 +419,25 @@ class TestOnlineAccount:
         assert msg_in.view_type.is_image()
         assert os.path.exists(msg_in.filename)
         assert os.stat(msg_in.filename).st_size == os.stat(path).st_size
+
+    def test_import_export_online(self, acfactory, tmpdir):
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        chat = ac1.create_chat_by_contact(contact1)
+        chat.send_text("msg1")
+        path = ac1.export_to_dir(backupdir.strpath)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account()
+        ac2.import_from_file(path)
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@hello.com"
+        chat2 = ac2.create_chat_by_contact(contact2)
+        messages = chat2.get_messages()
+        assert len(messages) == 1
+        assert messages[0].text == "msg1"

python/tox.ini

@@ -27,7 +27,7 @@ commands =
 [testenv:lint]
 skipsdist = True
 usedevelop = True
-basepython = python2.7
+basepython = python3.5
 deps =
     flake8
     # pygments required by rst-lint

src/dc_imex.rs

@@ -915,6 +915,7 @@ unsafe fn import_backup(context: &Context, backup_to_import: *const libc::c_char
         backup_to_import,
         context.get_dbfile(),
     );
+
     if 0 != dc_is_configured(context) {
         dc_log_error(
             context,
@@ -942,6 +943,10 @@ unsafe fn import_backup(context: &Context, backup_to_import: *const libc::c_char
                 );
                 sqlite3_step(stmt);
                 total_files_cnt = sqlite3_column_int(stmt, 0i32);
+                info!(
+                    context,
+                    0, "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt
+                );
                 sqlite3_finalize(stmt);
                 stmt = dc_sqlite3_prepare(
                     context,
@@ -1084,12 +1089,12 @@ unsafe fn export_backup(context: &Context, dir: *const libc::c_char) -> libc::c_
     );
     context.sql.close(&context);
     closed = 1i32;
-    dc_log_info(
+    info!(
         context,
-        0i32,
-        b"Backup \"%s\" to \"%s\".\x00" as *const u8 as *const libc::c_char,
-        context.get_dbfile(),
-        dest_pathNfilename,
+        0,
+        "Backup \"{}\" to \"{}\".",
+        as_str(context.get_dbfile()),
+        as_str(dest_pathNfilename),
     );
     if !(0 == dc_copy_file(context, context.get_dbfile(), dest_pathNfilename)) {
         context.sql.open(&context, as_path(context.get_dbfile()), 0);
@@ -1132,17 +1137,16 @@ unsafe fn export_backup(context: &Context, dir: *const libc::c_char) -> libc::c_
                         let dir_handle = dir_handle.unwrap();
                         total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
 
+                        info!(context, 0, "EXPORT: total_files_cnt={}", total_files_cnt);
                         if total_files_cnt > 0 {
                             // scan directory, pass 2: copy files
                             let dir_handle = std::fs::read_dir(dir);
                             if dir_handle.is_err() {
-                                dc_log_error(
+                                error!(
                                     context,
-                                    0i32,
-                                    b"Backup: Cannot copy from blob-directory \"%s\".\x00"
-                                        as *const u8
-                                        as *const libc::c_char,
-                                    context.get_blobdir(),
+                                    0,
+                                    "Backup: Cannot copy from blob-directory \"{}\".",
+                                    as_str(context.get_blobdir()),
                                 );
                             } else {
                                 let dir_handle = dir_handle.unwrap();
@@ -1190,7 +1194,9 @@ unsafe fn export_backup(context: &Context, dir: *const libc::c_char) -> libc::c_
                                         let name_f = entry.file_name();
                                         let name = name_f.to_string_lossy();
                                         if name.starts_with("delt-chat") && name.ends_with(".bak") {
-                                            // dc_log_info(context, 0, "Backup: Skipping \"%s\".", name);
+                                            continue;
+                                        } else {
+                                            info!(context, 0, "EXPORTing filename={}", name);
                                             free(curr_pathNfilename as *mut libc::c_void);
                                             let name_c = to_cstring(name);
                                             curr_pathNfilename = dc_mprintf(
@@ -1243,13 +1249,7 @@ unsafe fn export_backup(context: &Context, dir: *const libc::c_char) -> libc::c_
                                 }
                             }
                         } else {
-                            dc_log_info(
-                                context,
-                                0i32,
-                                b"Backup: No files to copy.\x00" as *const u8
-                                    as *const libc::c_char,
-                                context.get_blobdir(),
-                            );
+                            info!(context, 0, "Backup: No files to copy.");
                             current_block = 2631791190359682872;
                         }
                         match current_block {

src/dc_tools.rs

@@ -1258,6 +1258,7 @@ pub unsafe fn dc_write_file(
 ) -> libc::c_int {
     let mut success = 0;
     let pathNfilename_abs = dc_get_abs_path(context, pathNfilename);
+
     if pathNfilename_abs.is_null() {
         return 0;
     }
@@ -1270,15 +1271,17 @@ pub unsafe fn dc_write_file(
 
     match fs::write(p, bytes) {
         Ok(_) => {
+            info!(context, 0, "wrote file {}", as_str(pathNfilename));
+
             success = 1;
         }
         Err(_err) => {
-            dc_log_warning(
+            warn!(
                 context,
-                0i32,
-                b"Cannot write %lu bytes to \"%s\".\x00" as *const u8 as *const libc::c_char,
-                buf_bytes as libc::c_ulong,
-                pathNfilename,
+                0,
+                "Cannot write {} bytes to \"{}\".",
+                buf_bytes,
+                as_str(pathNfilename),
             );
         }
     }