Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-17 13:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix imex race condition, (#2255)

Browse Source 
fix #2254: if the DB was closed without calling stop_io() and then an interrupt arrives (e.g. incoming message), the db was corrupted.

* Add result.log() for logging with less boilerplate code

* Bugfix: Resultify housekeeping() to make it abort if the db is closed instead of just deleting everything

* Require the UI to call dc_stop_io() before backup export

* Prepare a bit better for closed-db: Resultify get_uidvalidity and get_uid_next and let job::load_next() wait until the db is open

About the bug (before this PR):
if the DB was closed without calling stop_io() and then an interrupt arrives (e.g. incoming message):
- I don't know if it downloads the message, but of course at some point the process of receiving the message will fail
- In my test, DC is just in the process of moving a message when the imex starts, but then can't delete the job or update the msg server_uid
- Then, when job::load_next() is called, no job can be loaded. That's why it calls `load_housekeeping_job()`. As `load_housekeeping_job()` can't load the time of the last housekeeping, it assumes we never ran housekeeping and returns a new Housekeeping job, which is immediately executed.
- housekeeping can't find any blobs referenced in the db and therefore deletes almost all blobs.
This commit is contained in:
Hocuri
2021-03-02 10:25:02 +01:00
committed by GitHub
parent a698a8dd84
commit 2a39dc06e9
13 changed files with 231 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
python/src/deltachat/_build.py
View File

@@ -150,6 +150,7 @@ def extract_defines(flags):
| DC_CHAT
| DC_PROVIDER
| DC_KEY_GEN
| DC_IMEX
) # End of prefix matching
_[\w_]+ # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
) # Close the capturing group, this contains

39
python/src/deltachat/account.py
View File

@@ -412,23 +412,23 @@ class Account(object):
Note that the account does not have to be started.
"""
return self._export(path, imex_cmd=1)
return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
def export_all(self, path):
"""return new file containing a backup of all database state
(chats, contacts, keys, media, ...). The file is created in the
the `path` directory.
Note that the account does not have to be started.
Note that the account has to be stopped; call stop_io() if necessary.
"""
export_files = self._export(path, 11)
export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
if len(export_files) != 1:
raise RuntimeError("found more than one new file")
return export_files[0]
def _export(self, path, imex_cmd):
with self.temp_plugin(ImexTracker()) as imex_tracker:
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
self.imex(path, imex_cmd)
return imex_tracker.wait_finish()
def import_self_keys(self, path):
@@ -438,7 +438,7 @@ class Account(object):
Note that the account does not have to be started.
"""
self._import(path, imex_cmd=2)
self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
def import_all(self, path):
"""import delta chat state from the specified backup `path` (a file).
@@ -446,13 +446,16 @@ class Account(object):
The account must be in unconfigured state for import to attempted.
"""
assert not self.is_configured(), "cannot import into configured account"
self._import(path, imex_cmd=12)
self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
def _import(self, path, imex_cmd):
with self.temp_plugin(ImexTracker()) as imex_tracker:
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
self.imex(path, imex_cmd)
imex_tracker.wait_finish()
def imex(self, path, imex_cmd):
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
def initiate_key_transfer(self):
"""return setup code after a Autocrypt setup message
has been successfully sent to our own e-mail address ("self-sent message").
@@ -577,6 +580,28 @@ class Account(object):
raise ValueError("account not configured, cannot start io")
lib.dc_start_io(self._dc_context)
def maybe_network(self):
"""This function should be called when there is a hint
that the network is available again,
e.g. as a response to system event reporting network availability.
The library will try to send pending messages out immediately.
Moreover, to have a reliable state
when the app comes to foreground with network available,
it may be reasonable to call the function also at that moment.
It is okay to call the function unconditionally when there is
network available, however, calling the function
_without_ having network may interfere with the backoff algorithm
and will led to let the jobs fail faster, with fewer retries
and may avoid messages being sent out.
Finally, if the context was created by the dc_accounts_t account manager
(currently not implemented in the Python bindings),
use dc_accounts_maybe_network() instead of this function
"""
lib.dc_maybe_network(self._dc_context)
def configure(self, reconfigure=False):
""" Start configuration process and return a Configtracker instance
on which you can block with wait_finish() to get a True/False success

48
python/tests/test_account.py
View File

@@ -476,6 +476,7 @@ class TestOfflineChat:
contact = msg.get_sender_contact()
assert contact == ac1.get_self_contact()
assert not backupdir.listdir()
ac1.stop_io()
path = ac1.export_all(backupdir.strpath)
assert os.path.exists(path)
ac2 = acfactory.get_unconfigured_account()
@@ -1497,21 +1498,42 @@ class TestOnlineAccount:
original_image_path = data.get_path("d.png")
chat1.send_image(original_image_path)
def assert_account_is_proper(ac):
contacts = ac.get_contacts(query="some1")
assert len(contacts) == 1
contact2 = contacts[0]
assert contact2.addr == "some1@example.org"
chat2 = contact2.create_chat()
messages = chat2.get_messages()
assert len(messages) == 2
assert messages[0].text == "msg1"
lp.sec("dbg file"+messages[1].filename)
assert messages[1].filemime == "image/png"
assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
ac.set_config("displayname", "new displayname")
assert ac.get_config("displayname") == "new displayname"
assert_account_is_proper(ac1)
backupdir = tmpdir.mkdir("backup")
lp.sec("export all to {}".format(backupdir))
with ac1.temp_plugin(ImexTracker()) as imex_tracker:
path = ac1.export_all(backupdir.strpath)
assert os.path.exists(path)
ac1.stop_io()
ac1.imex(backupdir.strpath, const.DC_IMEX_EXPORT_BACKUP)
# check progress events for export
assert imex_tracker.wait_progress(1, progress_upper_limit=249)
assert imex_tracker.wait_progress(250, progress_upper_limit=499)
assert imex_tracker.wait_progress(500, progress_upper_limit=749)
assert imex_tracker.wait_progress(750, progress_upper_limit=999)
assert imex_tracker.wait_progress(1000)
t = time.time()
paths = imex_tracker.wait_finish()
assert len(paths) == 1
path = paths[0]
assert os.path.exists(path)
ac1.start_io()
lp.sec("get fresh empty account")
ac2 = acfactory.get_unconfigured_account()
@@ -1530,23 +1552,11 @@ class TestOnlineAccount:
assert imex_tracker.wait_progress(750, progress_upper_limit=999)
assert imex_tracker.wait_progress(1000)
contacts = ac2.get_contacts(query="some1")
assert len(contacts) == 1
contact2 = contacts[0]
assert contact2.addr == "some1@example.org"
chat2 = contact2.create_chat()
messages = chat2.get_messages()
assert len(messages) == 2
assert messages[0].text == "msg1"
assert messages[1].filemime == "image/png"
assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
assert_account_is_proper(ac1)
assert_account_is_proper(ac2)
# wait until a second passed since last backup
# because get_latest_backupfile() shall return the latest backup
# from a UI it's unlikely anyone manages to export two
# backups in one second.
time.sleep(max(0, 1 - (time.time() - t)))
lp.sec("Second-time export all to {}".format(backupdir))
ac1.stop_io()
path2 = ac1.export_all(backupdir.strpath)
assert os.path.exists(path2)
assert path2 != path