Add some information about mailing list managers.

Remove additional chat type MailingList, make them GroupLists with a special param, make can_send() return false, code style and deduplication

.circleci/config.yml

@@ -0,0 +1,232 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
+jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: pyenv versions
+      - run: pyenv global 3.5.2
+      - run: ls -laR workspace
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy 
+
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      # - cargo_fetch
+
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+             - remote_tests_rust 
+          filters:
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
+
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.*/
+
+      # Linux Desktop 64bit
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -1,143 +0,0 @@
-name: Rust CI
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - staging
-      - trying
-
-env:
-  RUSTFLAGS: -Dwarnings
-
-jobs:
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: stable
-            components: clippy
-            override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
-
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
-
-  build_and_test:
-    name: Build and test
-    strategy:
-      matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.54.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.54.0
-            python: false # Python bindings compilation on Windows is not supported.
-
-          # Minimum Supported Rust Version = 1.51.0
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.51.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      with:
-        command: check
-        args: --all --bins --examples --tests --features repl --benches
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3

.github/workflows/code-quality.yml

@@ -0,0 +1,48 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --workspace --examples --tests --all-features
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2020-03-12
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -26,6 +25,3 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
-
-coverage/
-.DS_Store

CHANGELOG.md

@@ -1,982 +1,4 @@
-# Changelog
-
-## Unreleased
-
-### API changes
-
-- Removed `mvbox_watch` option. #2906
-  It is automatically enabled whenever `mvbox_move` is enabled.
-
-## 1.70.0
-
-### Fixes
-- fix: do not abort Param parsing on unknown keys #2856
-- fix: execute `Chat-Group-Member-Removed:` even when arriving disordered #2857
-
-
-## 1.69.0
-
-### Fixes
-- fix group-related system messages in multi-device setups #2848
-- fix "Google Workspace" (former "G Suite") issues related to bad resolvers #2852
-
-
-## 1.68.0
-
-### Fixes
-- fix chat assignment when forwarding #2843
-- fix layout issues with the generated QR code svg #2842
-
-
-## 1.67.0
-
-### API changes
-- `dc_get_securejoin_qr_svg(chat_id)` added #2815
-- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
-
-
-## 1.66.0
-
-### API changes
-- `dc_contact_get_last_seen()` added #2823
-- python: `Contact.last_seen` added #2823
-- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
-
-### Changes
-- python: add cutil.from_optional_dc_charpointer() #2824
-- refactorings #2807 #2822 #2825
-
-
-## 1.65.0
-
-### Changes
-- python: add mypy support and some type hints #2809
-
-### Fixes
-- do not disable ephemeral timer when downloading a message partially #2811
-- apply existing ephemeral timer also to partially downloaded messages;
-  after full download, the ephemeral timer starts over #2811
-- replace user-visible error on verification failure with warning;
-  the error is logged to the corresponding chat anyway #2808
-
-
-## 1.64.0
-
-### Fixes
-- add 'waiting for being added to the group' only for group-joins,
-  not for setup-contact #2797
-- prioritize In-Reply-To: and References: headers over group IDs when assigning
-  messages to chats to fix incorrect assignment of Delta Chat replies to
-  classic email threads #2795
-
-
-## 1.63.0
-
-### API changes
-- `dc_get_last_error()` added #2788
-
-### Changes
-- Optimize Autocrypt gossip #2743
-
-### Fixes
-- fix permanently hiding of one-to-one chats after secure-join #2791
-
-
-## 1.62.0
-
-### API Changes
-- `dc_join_securejoin()` now always returns immediately;
-  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
-  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
-- introduce multi-device-sync-messages;
-  as older cores display them as files in self-chat,
-  they are currently only sent if config option `send_sync_msgs` is set #2669
-- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
-
-### Changes
-- use system DNS instead of google for MX queries #2780
-- improve error logging #2758
-- improve tests #2764 #2781
-- improve ci #2770
-- refactorings #2677 #2728 #2740 #2729 #2766 #2778
-
-### Fixes
-- add Let's Encrypt certificate to core as it may be missing older devices #2752
-- prioritize certificate setting from user over the one from provider-db #2749
-- fix "QR process failed" error #2725
-- do not update quota in endless loop #2726
-
-
-## 1.61.0
-
-### API Changes
-- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
-  and `download_limit` config option #2631 #2696
-- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
-- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
-  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
-- emit warnings and errors from account manager with account-id 0 #2712
-
-### Changes
-- notify about incoming contact requests #2690
-- messages are marked as read on first read receipt #2699
-- quota warning reappears after import, rewarning at 95% #2702
-- lock strict TLS if certificate checks are automatic #2711
-- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
-- `Accounts` is not cloneable anymore #2654 #2658
-- update chat/contact data only when there was no newer update #2642
-- better detection of mailing list names #2665 #2685
-- log all decisions when applying ephemeral timer to chats #2679
-- connectivity view now translatable #2694 #2723
-- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
-- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
-- update provider database #2618
-
-### Fixes
-- ephemeral timer rollback protection #2693 #2709
-- recreate configured folders if they are deleted #2691
-- ignore MDNs sent to self #2674
-- recognize NDNs that put headers into "message/global-headers" part #2598
-- avoid `dc_get_contacts()` returning duplicate contact ids #2591
-- do not leak group names on forwarding messages #2719
-- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
-- fix pkg-config file #2660
-- fix "QR process failed" error #2725
-
-
-## 1.60.0
-
-### Added
-- add device message to warn about QUOTA #2621
-- add SOCKS5 support #2474 #2620
-
-### Changes
-- don't emit multiple events with the same import/export progress number #2639
-- reduce message length limit to 5000 chars #2615
-
-### Fixes
-- keep event emitter from closing when there are no accounts #2636
-
-
-## 1.59.0
-
-### Added
-- add quota information to `dc_get_connectivity_html()`
-
-### Changes
-- refactorings #2592 #2570 #2581
-- add 'device chat about' to now existing status #2613
-- update provider database #2608
-
-### Fixes
-- provider database supports socket=PLAIN and dotless domains now #2604 #2608
-- add migrated accounts to events emitter #2607
-- fix forwarding quote-only mails #2600
-- do not set WantsMdn param for outgoing messages #2603
-- set timestamps for system messages #2593
-- do not treat gmail labels as folders #2587
-- avoid timing problems in `dc_maybe_network_lost()` #2551
-- only set smtp to "connected" if the last message was actually sent #2541
-
-
-## 1.58.0
-
-### Fixes
-- move WAL file together with database
-  and avoid using data if the database was not closed correctly before #2583
-
-
-## 1.57.0
-
-### API Changes
-
-- breaking change: removed deaddrop chat #2514 #2563
-
-  Contact request chats are not merged into a single virtual
-  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
-  same way as other chats, but sending of messages to them is not
-  allowed and MDNs are not sent automatically until the chat is
-  "accepted" by the user.
-
-  New API:
-  - `dc_chat_is_contact_request()`: returns true if chat is a contact
-    request.  In this case an option to accept the chat via
-    `dc_accept_chat()` should be shown in the UI.
-  - `dc_accept_chat()`: unblock the chat or accept contact request
-  - `dc_block_chat()`: block the chat, currently works only for mailing
-    lists.
-
-  Removed API:
-  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_decide_on_contact_request()`: this call requires a message ID
-    from deaddrop chat as input. As deaddrop chat is removed, this
-    call can't be used anymore.
-  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
-    only difference between these calls was in handling of deaddrop
-    chat
-  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
-
-- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
-  Instead, there is a new api `dc_get_connectivity()`
-  and `dc_get_connectivity_html()`;
-  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
-
-- breaking change: removed `dc_accounts_import_account()`
-  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
-  on its context
-
-- update account api, 2 new methods:
-  `int dc_all_work_done (dc_context_t* context);`
-  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
-
-- add api to check if a message was `Auto-Submitted`
-  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
-  python: `Message.is_bot()`
-
-- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
-  now returns `NULL` if there is no selected account
-
-- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
-  connectivity loss on its own (eg. iOS) #2550
-
-### Added
-- use Auto-Submitted: auto-generated header to identify bots #2502
-- allow sending stickers via repl tool
-- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
-- withdraw/revive own qr-codes #2512
-- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
-
-### Changes
-- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
-- update documentation #2548 #2561 #2569
-- breaking: `Accounts::create` does not also create an default account anymore #2500
-- remove "forwarded" from stickers, as the primary way of getting stickers
-  is by asking a bot and then forwarding them currently #2526
-- mimeparser: use mailparse to parse RFC 2231 filenames #2543
-- allow email addresses without dot in the domain part #2112
-- allow installing lib and include under different prefixes #2558
-- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
-- improve tests #2487 #2491 #2497
-- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
-- improve ci #2494
-- update provider-database #2565
-
-### Removed
-- remove `dc_accounts_import_account()` api #2521
-- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
-
-### Fixes
-- allow stickers with gif-images #2481
-- fix database migration #2486
-- do not count hidden messages in get_msg_cnt(). #2493
-- improve drafts detection #2489
-- fix panic when removing last, selected account from account manager #2500
-- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
-- avoid hiding outgoing classic emails #2505
-- fixes for message timestamps #2517
-- do not process names, avatars, location XMLs, message signature etc.
-  for duplicate messages #2513
-- fix `can_send` for users not in group #2479
-- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
-- fix which chats messages are assigned to #2465
-- fix: don't create chats when MDNs are received #2578
-
-
-## 1.56.0
-
-- fix downscaling images #2469
-
-- fix outgoing messages popping up in selfchat #2456
-
-- securejoin: display error reason if there is any #2470
-
-- do not allow deleting contacts with ongoing chats #2458
-
-- fix: ignore drafts folder when scanning #2454
-
-- fix: scan folders also when inbox is not watched #2446
-
-- more robust In-Reply-To parsing #2182
-
-- update dependencies #2441 #2438 #2439 #2440 #2447 #2448 #2449 #2452 #2453 #2460 #2464 #2466
-
-- update provider-database #2471
-
-- refactorings #2459 #2457
-
-- improve tests and ci #2445 #2450 #2451
-
-
-## 1.55.0
-
-- fix panic when receiving some HTML messages #2434
-
-- fix downloading some messages multiple times #2430
-
-- fix formatting of read receipt texts #2431
-
-- simplify SQL error handling #2415
-
-- explicit rust API for creating chats with blocked status #2282
-
-- debloat the binary by using less AsRef arguments #2425
-
-
-## 1.54.0
-
-- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
-
-- global search performance improvement #2364 #2365 #2366
-
-- improve SQLite performance with `PRAGMA synchronous=normal` #2382
-
-- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
-
-- python: list `requests` as a requirement #2390
-
-- fix creation of many delete jobs when being offline #2372
-
-- synchronize status between devices #2386
-
-- deaddrop (contact requests) chat improvements #2373
-
-- add "Forwarded:" to notification and chatlist summaries #2310
-
-- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
-
-- improve tests #2360 #2362 #2370 #2377 #2387
-
-- cleanup #2359 #2361 #2374 #2376 #2379 #2388
-
-
-## 1.53.0
-
-- fix sqlx performance regression #2355 2356
-
-- add a `ci_scripts/coverage.sh` #2333 #2334
-
-- refactorings and tests #2348 #2349 #2350
-
-- improve python bindings #2332 #2326
-
-
-## 1.52.0
-
-- database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340
-
-- add alias support: UIs should check for `dc_msg_get_override_sender_name()`
-  also in single-chats now and display divergent names and avatars #2297
-
-- parse blockquote-tags for better quote detection #2313
-
-- ignore unknown classical emails from spam folder #2311
-
-- support "Mixed Up” encryption repairing #2321
-
-- fix single chat search #2344
-
-- fix nightly clippy and rustc errors #2341
-
-- update dependencies #2350
-
-- improve ci #2342
-
-- improve python bindings #2332 #2326
-
-
-## 1.51.0
-
-- breaking change: You have to call `dc_stop_io()`/`dc_start_io()`
-  before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`:
-  fix race condition and db corruption
-  when a message was received during backup #2253
-
-- save subject for messages: new api `dc_msg_get_subject()`,
-  when quoting, use the subject of the quoted message as the new subject,
-  instead of the last subject in the chat #2274 #2283
-
-- new apis to get full or html message,
-  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151 #2264 #2279
-
-- new chat type and apis for the new mailing list support,
-  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
-  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
-  #2241 #2243 #2258 #2259 #2261 #2267 #2270 #2272 #2290
-
-- new api `dc_decide_on_contact_request()`,
-  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
-
-- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
-
-- new api `dc_get_chat_encrinfo()` #2186
-
-- new api `dc_contact_get_status()`, returning the recent footer #2218 #2307
-
-- improve contact name update rules,
-  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
-
-- new api for bots: `dc_msg_set_html()` #2153
-
-- new api for bots: `dc_msg_set_override_sender_name()` #2231
-
-- api removed: `dc_is_io_running()` #2139
-
-- api removed: `dc_contact_get_first_name()` #2165 #2171
-
-- improve compatibility with providers changing the Message-ID
-  (as Outlook.com) #2250 #2265
-
-- correctly show emails that were sent to an alias and then bounced 
-
-- implement Consistent Color Generation (XEP-0392),
-  that results in contact colors be be changed #2228 #2229 #2239
-
-- fetch recent existing messages
-  and create corresponding chats after configure #2106
-
-- improve e-mail compatibility
-  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
-
-- better support videochat-services not supporting random rooms #2191
-
-- export backups as .tar files #2023
-
-- scale avatars based on media_quality, fix avatar rotation #2063
-
-- compare ephemeral timer to parent message to deal with reordering better #2100
-
-- better ephemeral system messages #2183
-
-- read quotes out of html messages #2104
-
-- prepend subject to messages with attachments, if needed #2111
-
-- run housekeeping at least once a day #2114
-
-- resolve MX domain only once per OAuth2 provider #2122
-
-- configure provider based on MX record #2123 #2134
-
-- make transient bad destination address error permanent
-  after n tries #2126 #2202
-
-- enable strict TLS for known providers by default #2121
-
-- improve and harden secure join #2154 #2161 #2251
-
-- update `dc_get_info()` to return more information #2156
-
-- prefer In-Reply-To/References
-  over group-id stored in Message-ID #2164 #2172 #2173
-
-- apply gossiped encryption preference to new peerstates #2174
-
-- fix: do not return quoted messages from the trash chat #2221
-
-- fix: allow emojis for location markers #2177
-
-- fix encoding of Chat-Group-Name-Changed messages that could even lead to
-  messages not being delivered #2141
-
-- fix error when no temporary directory is available #1929
-
-- fix marking read receipts as seen #2117
-
-- fix read-notification for mixed-case addresses #2103
-
-- fix decoding of attachment filenames #2080 #2094 #2102
-
-- fix downloading ranges of message #2061
-
-- fix parsing quoted encoded words in From: header #2193 #2204
-
-- fix import/export race condition #2250
-
-- fix: exclude muted chats from notified-list #2269 #2275
-
-- fix: update uid_next if the server rewind it #2288
-
-- fix: return error on fingerprint mismatch on qr-scan #2295
-
-- fix ci #2217 #2226 #2244 #2245 #2249 #2277 #2286
-
-- try harder on backup opening #2148
-
-- trash messages more thoroughly #2273
-
-- nicer logging #2284
-
-- add CMakeLists.txt #2260
-
-- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
-
-- improve python bindings #2113 #2115 #2133 #2214
-
-- improve documentation #2143 #2160 #2175 #2146
-
-- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
-  #2200 #2230 #2262 #2203
-
-- update provider-database #2299
-
-
-## 1.50.0
-
-- do not fetch emails in between inbox_watch disabled and enabled again #2087
-
-- fix: do not fetch from INBOX if inbox_watch is disabled #2085
-
-- fix: do not use STARTTLS when PLAIN connection is requested
-  and do not allow downgrade if STARTTLS is not available #2071
-
-
-## 1.49.0
-
-- add timestamps to image and video filenames #2068
-
-- forbid quoting messages from another context #2069
-
-- fix: preserve quotes in messages with attachments #2070
-
-
-## 1.48.0
-
-- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
-  #2035 #2042
-
-- skip fetch existing messages/contacts if config-option `bot` set #2017
-
-- always log why a message is sorted to trash #2045
-
-- display a quote if top posting is detected #2047
-
-- add ephemeral task cancellation to `dc_stop_io()`;
-  before, there was no way to quickly terminate pending ephemeral tasks #2051
-
-- when saved-messages chat is deleted,
-  a device-message about recreation is added #2050
-
-- use `max_smtp_rcpt_to` from provider-db,
-  sending messages to many recipients in configurable chunks #2056
-
-- fix handling of empty autoconfigure files #2027
-
-- fix adding saved messages to wrong chats on multi-device #2034 #2039
-
-- fix hang on android4.4 and other systems
-  by adding a workaround to executer-blocking-handling bug #2040
-
-- fix secret key export/import roundtrip #2048
-
-- fix mistakenly unarchived chats #2057
-
-- fix outdated-reminder test that fails only 7 days a year,
-  including halloween :) #2059
-
-- improve python bindings #2021 #2036 #2038
-
-- update provider-database #2037
-
-
-## 1.47.0
-
-- breaking change: `dc_update_device_chats()` removed;
-  this is now done automatically during configure
-  unless the new config-option `bot` is set #1957
-
-- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
-  and remove `dc_marknoticed_all_chats()` #1942 #1981
-
-- breaking change: remove unused starring options #1965
-
-- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
-  `dc_chat_is_protected()`; also single-chats may be protected now, this may
-  happen over the wire even if the UI do not offer an option for that #1968
-
-- breaking change: split quotes off message text,
-  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
-
-- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
-  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
-
-- require quorum to enable encryption #1946
-
-- speed up and clean up account creation #1912 #1927 #1960 #1961
-
-- configure now collects recent contacts and fetches last messages
-  unless disabled by `fetch_existing` config-option #1913 #2003
-  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
-
-- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
-  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
-
-- add `dc_set_chat_protection()`; the `protect` parameter in
-  `dc_create_group_chat()` will be removed in an upcoming release;
-  up to then, UIs using the "verified group" paradigm
-  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
-
-- remove unneeded `DC_STR_COUNT` #1991
-
-- mark all failed messages as failed when receiving an NDN #1993
-
-- check some easy cases for bad system clock and outdated app #1901
-
-- fix import temporary directory usage #1929
-
-- fix forcing encryption for reset peers #1998
-
-- fix: do not allow to save drafts in non-writeable chats #1997
-
-- fix: do not show HTML if there is no content and there is an attachment #1988
-
-- fix recovering offline/lost connections, fixes background receive bug #1983
-
-- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
-
-- fix whitespace for summaries #1938
-
-- fix: improve sentbox name guessing #1941
-
-- fix: avoid manual poll impl for accounts events #1944
-
-- fix encoding newlines in param as a preparation for storing quotes #1945
-
-- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
-
-- fix ci #1928 #1931 #1932 #1933 #1934 #1943
-
-- update provider-database #1940 #2005 #2006
-
-- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
-
-
-## 1.46.0
-
-- breaking change: `dc_configure()` report errors in
-  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
-  #1886 #1905
-
-- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
-  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
-
-- parse multiple servers in Mozilla autoconfig #1860
-
-- try multiple servers for each protocol #1871
-
-- do IMAP and SMTP configuration in parallel #1891
-
-- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
-
-- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
-
-- do not reset peerstate on encrypted messages,
-  ignore reordered autocrypt headers #1885 #1890
-
-- always sort message replies after parent message #1852
-
-- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
-
-- improve mimetype guessing for PDF and many other formats #1857 #1861
-
-- improve accepting invalid html #1851
-
-- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
-
-- tweak HELO command #1908
-
-- make `dc_accounts_get_all()` return accounts sorted #1909
-
-- fix KML coordinates precision used for location streaming #1872
-
-- fix cancelling import/export #1855
-
-
-## 1.45.0
-
-- add `dc_accounts_t` account manager object and related api functions #1784
-
-- add capability to import backups as .tar files,
-  which will become the default in a subsequent release #1749
-
-- try various server domains on configuration #1780 #1838
-
-- recognize .tgs files as stickers #1826
-
-- remove X-Mailer debug header #1819
-
-- improve guessing message types from extension #1818
-
-- fix showing unprotected subjects in encrypted messages #1822
-
-- fix threading in interaction with non-delta-clients #1843
-
-- fix handling if encryption degrades #1829
-
-- fix webrtc-servers names set by the user #1831
-
-- update provider database #1828
-
-- update async-imap to fix Oauth2 #1837
-
-- optimize jpeg assets with trimage #1840
-
-- add tests and documentations #1809 #1820
-
-
-## 1.44.0
-
-- fix peerstate issues #1800 #1805
-
-- fix a crash related to muted chats #1803
-
-- fix incorrect dimensions sometimes reported for images #1806
-
-- fixed `dc_chat_get_remaining_mute_duration` function #1807
-
-- handle empty tags (e.g. `<br/>`) in HTML mails #1810
-
-- always translate the message about disappearing messages timer change #1813
-
-- improve footer detection in plain text email #1812
-
-- update device chat icon to fix warnings in iOS logs #1802
-
-- fix deletion of multiple messages #1795
-
-
-## 1.43.0
-
-- improve using own jitsi-servers #1785
-
-- fix smtp-timeout tweaks for larger mails #1797
-
-- more bug fixes and updates #1794 #1792 #1789 #1787
-
-
-## 1.42.0
-
-- new qr-code type `DC_QR_WEBRTC` #1779
-
-- new `dc_chatlist_get_summary2()` api #1771
-
-- tweak smtp-timeout for larger mails #1782
-
-- optimize read-receipts #1765
-
-- Allow http scheme for DCACCOUNT URLs #1770
-
-- improve tests #1769
-
-- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
-
-
-## 1.41.0
-
-- new apis to initiate video chats #1718 #1735
-
-- new apis `dc_msg_get_ephemeral_timer()`
-  and `dc_msg_get_ephemeral_timestamp()`
-
-- new api `dc_chatlist_get_summary2()` #1771
-
-- improve IMAP handling #1703 #1704
-
-- improve ephemeral messages #1696 #1705
-
-- mark location-messages as auto-generated #1715
-
-- multi-device avatar-sync #1716 #1717
-
-- improve python bindings #1732 #1733 #1738 #1769
-
-- Allow http scheme for DCACCOUNT urls #1770
-
-- more fixes #1702 #1706 #1707 #1710 #1719 #1721
-  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
-
-- refactorings #1712 #1714 #1757
-
-- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
-
-
-## 1.40.0
-
-- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
-
-- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
-
-- improve idle #1690 #1688
-
-- fix message processing issues by sequential processing #1694
-
-- refactorings #1670 #1673
-
-
-## 1.39.0
-
-- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
-
-- fix potential panics, update dependencies #1650 #1655
-
-
-## 1.38.0
-
-- fix sorting, esp. for multi-device
-
-
-## 1.37.0
-
-- improve ndn heuristics #1630
-
-- get oauth2 authorizer from provider-db #1641
-
-- removed linebreaks and spaces from generated qr-code #1631
-
-- more fixes #1633 #1635 #1636 #1637
-
-
-## 1.36.0
-
-- parse ndn (network delivery notification) reports
-  and report failed messages as such #1552 #1622 #1630
-
-- add oauth2 support for gsuite domains #1626
-
-- read image orientation from exif before recoding #1619
-
-- improve logging #1593 #1598
-
-- improve python and bot bindings #1583 #1609
-
-- improve imap logout #1595
-
-- fix sorting #1600 #1604
-
-- fix qr code generation #1631
-
-- update rustcrypto releases #1603
-
-- refactorings #1617
-
-
-## 1.35.0
-
-- enable strict-tls from a new provider-db setting #1587
-
-- new subject 'Message from USER' for one-to-one chats #1395
-
-- recode images #1563
-
-- improve reconnect handling #1549 #1580
-
-- improve importing addresses #1544
-
-- improve configure and folder detection #1539 #1548
-
-- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
-
-- fix ad-hoc groups #1566
-
-- preventions against being marked as spam #1575
-
-- refactorings #1542 #1569
-
-
-## 1.34.0
-
-- new api for io, thread and event handling #1356,
-  see the example atop of `deltachat.h` to get an overview
-
-- LOTS of speed improvements due to async processing #1356
-
-- enable WAL mode for sqlite #1492
-
-- process incoming messages in bulk #1527
-
-- improve finding out the sent-folder #1488
-
-- several bug fixes
-
-
-## 1.33.0
-
-- let `dc_set_muted()` also mute one-to-one chats #1470
-
-- fix a bug that led to load and traffic if the server does not use sent-folder
-  #1472
-
-
-## 1.32.0
-
-- fix endless loop when trying to download messages with bad RFC Message-ID,
-  also be more reliable on similar errors #1463 #1466 #1462
-
-- fix bug with comma in contact request #1438
-
-- do not refer to hidden messages on replies #1459
-
-- improve error handling #1468 #1465 #1464
-
-
-## 1.31.0
-
-- always describe the context of the displayed error #1451
-
-- do not emit `DC_EVENT_ERROR` when message sending fails;
-  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
-
-- new config-option `media_quality` #1449
-
-- try over if writing message to database fails #1447
-
-
-## 1.30.0
-
-- expunge deleted messages #1440
-
-- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
-
-
-## 1.29.0
-
-- new config options `delete_device_after` and `delete_server_after`,
-  each taking an amount of seconds after which messages
-  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
-
-- new api `dc_estimate_deletion_cnt()` to estimate the effect
-  of `delete_device_after` and `delete_server_after`
-
-- use Ed25519 keys by default, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1362
-
-- improve message ellipsizing #1397 #1430
-
-- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
-
-- do not show badly formatted non-delta-messages as empty #1384
-
-- try over SMTP on potentially recoverable error 5.5.0 #1379
-
-- remove device-chat from forward-to-chat-list #1367
-
-- improve group-handling #1368
-
-- `dc_get_info()` returns uptime (how long the context is in use)
-
-- python improvements and adaptions #1408 #1415
-
-- log to the stdout and stderr in tests #1416
-
-- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
-
-- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
-
-- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
-
+# Changelog 
 
 ## 1.28.0
 
@@ -1399,3 +421,4 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

CMakeLists.txt

@@ -1,42 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-include(GNUInstallDirs)
-
-find_program(CARGO cargo)
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,13 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.70.0"
+version = "1.28.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
-resolver = "2"
-
-[profile.dev]
-debug = 0
 
 [profile.release]
 lto = true
@@ -15,77 +11,61 @@ lto = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
-backtrace = "0.3"
-base64 = "0.13"
-bitflags = "1.3"
-byteorder = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+libc = "0.2.51"
+pgp = { version = "0.5.1", default-features = false } 
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-kamadak-exif = "0.5"
+sha2 = "0.8.0"
+rand = "0.7.0"
+smallvec = "1.0.0"
+reqwest = { version = "0.10.0", features = ["blocking", "json"] }
+num-derive = "0.3.0"
+num-traits = "0.2.6"
+async-smtp = "0.2"
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
-once_cell = "1.9.0"
+async-imap = "0.2"
+async-native-tls = "0.3.1"
+async-std = { version = "1.4", features = ["unstable"] }
+base64 = "0.11"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.19"
-rand = "0.7"
-regex = "1.5"
-rusqlite = { version = "0.26", features = ["sqlcipher"] }
-rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.10"
-sha2 = "0.10"
-smallvec = "1"
-stop-token = "0.6"
-strum = "0.23"
-strum_macros = "0.23"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "3.3.0"
-textwrap = "0.14.2"
+serde_json = "1.0"
+chrono = "0.4.6"
+indexmap = "1.3.0"
+lazy_static = "1.4.0"
+regex = "1.1.6"
+rusqlite = { version = "0.21", features = ["bundled"] }
+r2d2_sqlite = "0.13.0"
+r2d2 = "0.8.5"
+strum = "0.16.0"
+strum_macros = "0.16.0"
+thread-local-object = "0.1.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.8.0"
+image-meta = "0.1.0"
+quick-xml = "0.17.1"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.12.0"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+pretty_env_logger = "0.3.1"
+
+rustyline = { version = "4.1.0", optional = true }
+thiserror = "1.0.14"
+anyhow = "1.0.28"
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
-futures-lite = "1.12"
-log = "0.4"
-pretty_env_logger = "0.4"
-proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.3.0"
+proptest = "0.9.4"
 
 [workspace]
 members = [
@@ -96,29 +76,15 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
-required-features = ["repl"]
 
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["repl"]
+required-features = ["rustyline"]
 
 
-[[bench]]
-name = "create_account"
-harness = false
-
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
 [features]
-default = ["vendored"]
-internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+default = ["nightly"]
+vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
+

README.md

@@ -2,14 +2,14 @@
 
 > Deltachat-core written in Rust 
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
 ## Installing Rust and Cargo
 
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
 
 ```
-$ curl https://sh.rustup.rs -sSf | sh
+curl https://sh.rustup.rs -sSf | sh
 ```
 
 ## Using the CLI client
@@ -17,9 +17,8 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -79,16 +78,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -105,8 +94,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
-SMTP tracing in addition to info messages.
+
 
 ### Expensive tests
 
@@ -121,6 +109,11 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
@@ -128,8 +121,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
-- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
+- [Go](https://github.com/hugot/go-deltachat/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -139,5 +131,4 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
-- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -0,0 +1,19 @@
+environment:
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init -yv --default-toolchain nightly-2020-03-12
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+  - rustc -vV
+  - cargo -vV
+
+build: false
+
+test_script:
+  - cargo test --release --all
+
+cache:
+  - target
+  - C:\Users\appveyor\.cargo\registry

assets/icon-broadcast.svg

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   enable-background="new 0 0 128 128"
-   viewBox="0 0 60 60"
-   version="1.1"
-   id="svg878"
-   sodipodi:docname="icon-broadcast.svg"
-   width="60"
-   height="60"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
-  <metadata
-     id="metadata884">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs882" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1329"
-     inkscape:window-height="847"
-     id="namedview880"
-     showgrid="false"
-     inkscape:zoom="5.21875"
-     inkscape:cx="36.598802"
-     inkscape:cy="32.191617"
-     inkscape:window-x="111"
-     inkscape:window-y="205"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg878"
-     inkscape:document-rotation="0" />
-  <radialGradient
-     id="c"
-     cx="65.25"
-     cy="89"
-     r="26.440001"
-     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
-     gradientUnits="userSpaceOnUse">
-    <stop
-       stop-color="#FFC107"
-       offset="0"
-       id="stop833" />
-    <stop
-       stop-color="#FFBD06"
-       offset=".3502"
-       id="stop835" />
-    <stop
-       stop-color="#FFB104"
-       offset=".6938"
-       id="stop837" />
-    <stop
-       stop-color="#FFA000"
-       offset="1"
-       id="stop839" />
-  </radialGradient>
-  <radialGradient
-     id="b"
-     cx="52.5"
-     cy="19.75"
-     r="92.975998"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)">
-    <stop
-       stop-color="#EF5350"
-       offset="0"
-       id="stop848" />
-    <stop
-       stop-color="#EB4F4C"
-       offset=".246"
-       id="stop850" />
-    <stop
-       stop-color="#E04341"
-       offset=".4878"
-       id="stop852" />
-    <stop
-       stop-color="#CD302F"
-       offset=".7272"
-       id="stop854" />
-    <stop
-       stop-color="#C62828"
-       offset=".8004"
-       id="stop856" />
-    <stop
-       stop-color="#C62828"
-       offset="1"
-       id="stop858" />
-  </radialGradient>
-  <radialGradient
-     id="a"
-     cx="16.979"
-     cy="92"
-     r="24.165001"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)"
-     xlink:href="#b">
-    <stop
-       stop-color="#E0E0E0"
-       offset="0"
-       id="stop863" />
-    <stop
-       stop-color="#CFCFCF"
-       offset=".3112"
-       id="stop865" />
-    <stop
-       stop-color="#A4A4A4"
-       offset=".9228"
-       id="stop867" />
-    <stop
-       stop-color="#9E9E9E"
-       offset="1"
-       id="stop869" />
-  </radialGradient>
-  <rect
-     y="0"
-     x="0"
-     height="60"
-     width="60"
-     id="rect1420"
-     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
-  <path
-     id="path872"
-     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
-     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
-</svg>

assets/qrcode_logo_footer.svg

@@ -1,10 +0,0 @@
-<text
-     xml:space="preserve"
-     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     x="42.325161"
-     y="23.32255"
-     id="text72398">get.delta.chat</text>
-  <path
-     id="path84310"
-     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -1,37 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, &book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/create_account.rs

@@ -1,26 +0,0 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::accounts::Accounts;
-use tempfile::tempdir;
-
-async fn create_accounts(n: u32) {
-    let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
-
-    let mut accounts = Accounts::new(p.clone()).await.unwrap();
-
-    for expected_id in 2..n {
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, expected_id);
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,27 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -1,12 +1,11 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Continuous Integration, run through CircleCI and an own build machine.
 
 ## Description of scripts 
 
-- `../.github/workflows` contains jobs run by GitHub Actions.
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -27,8 +26,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -42,10 +41,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd scripts  # where all CI things are 
+    cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

ci_scripts/ci_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install -U pip
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*
+
+popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        clear_index = not projectnames
+        assert projectnames == ["deltachat"]
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,11 +60,8 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

ci_scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux1_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust
+# Install Rust nightly
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -1,9 +1,12 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,17 +20,16 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
-
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -41,5 +43,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash scripts/run-python-test.sh 
+    bash ci_scripts/run-python-test.sh 
 _HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -4,7 +4,6 @@
 # and tox/pytest. 
 
 set -e -x
-shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -ex
-shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,19 +15,16 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
-export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.7
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -40,8 +37,9 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
+unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.70.0"
+version = "1.28.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,16 +17,13 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1"
-num-traits = "0.2"
+human-panic = "1.0.1"
+num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1"
-anyhow = "1"
-thiserror = "1"
-rand = "0.7"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
-default = ["vendored"]
+default = ["vendored", "nightly"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
-

deltachat-ffi/Doxyfile

@@ -236,6 +236,12 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all
@@ -583,7 +589,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,16 +4,4 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
-	border-radius: 6px;
 }
-
-code {
-	background-color: #e0e0e0;
-	padding-left: .5em;
-	padding-right: .5em;
-	border-radius: 6px;
-}
-
-li {
-	margin-bottom: .5em;
-}

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/build.rs

@@ -19,17 +19,15 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
+        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/dc_array.rs

@@ -1,56 +1,46 @@
-use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
-use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
-    MsgIds(Vec<MsgId>),
-    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub(crate) fn get_id(&self, index: usize) -> u32 {
+    pub fn new(capacity: usize) -> Self {
+        dc_array_t::Uint(Vec::with_capacity(capacity))
+    }
+
+    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
+    pub fn new_locations(capacity: usize) -> Self {
+        dc_array_t::Locations(Vec::with_capacity(capacity))
+    }
+
+    pub fn add_id(&mut self, item: u32) {
+        if let Self::Uint(array) = self {
+            array.push(item);
+        } else {
+            panic!("Attempt to add id to array of other type");
+        }
+    }
+
+    pub fn add_location(&mut self, location: Location) {
+        if let Self::Locations(array) = self {
+            array.push(location)
+        } else {
+            panic!("Attempt to add a location to array of other type");
+        }
+    }
+
+    pub fn get_id(&self, index: usize) -> u32 {
         match self {
-            Self::MsgIds(array) => array[index].to_u32(),
-            Self::Chat(array) => match array[index] {
-                ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
-                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
-            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index],
+            Self::Uint(array) => array[index] as u32,
         }
     }
 
-    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(array) => array.get(index).and_then(|item| match item {
-                ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
-                ChatItem::DayMarker { timestamp } => Some(*timestamp),
-            }),
-            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(_) => None,
-            Self::Locations(array) => array
-                .get(index)
-                .and_then(|location| location.marker.as_deref()),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_location(&self, index: usize) -> &Location {
+    pub fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -58,18 +48,55 @@ impl dc_array_t {
         }
     }
 
-    /// Returns the number of elements in the array.
-    pub(crate) fn len(&self) -> usize {
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Locations(array) => array.is_empty(),
+            Self::Uint(array) => array.is_empty(),
+        }
+    }
+
+    /// Returns the number of elements in the array.
+    pub fn len(&self) -> usize {
         match self {
-            Self::MsgIds(array) => array.len(),
-            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
-        (0..self.len()).find(|i| self.get_id(*i) == needle)
+    pub fn clear(&mut self) {
+        match self {
+            Self::Locations(array) => array.clear(),
+            Self::Uint(array) => array.clear(),
+        }
+    }
+
+    pub fn search_id(&self, needle: u32) -> Option<usize> {
+        if let Self::Uint(array) = self {
+            for (i, &u) in array.iter().enumerate() {
+                if u == needle {
+                    return Some(i);
+                }
+            }
+            None
+        } else {
+            panic!("Attempt to search for id in array of other type");
+        }
+    }
+
+    pub fn sort_ids(&mut self) {
+        if let dc_array_t::Uint(v) = self {
+            v.sort();
+        } else {
+            panic!("Attempt to sort array of something other than uints");
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const u32 {
+        if let dc_array_t::Uint(v) = self {
+            v.as_ptr()
+        } else {
+            panic!("Attempt to convert array of something other than uints to raw");
+        }
     }
 }
 
@@ -79,18 +106,6 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
-impl From<Vec<MsgId>> for dc_array_t {
-    fn from(array: Vec<MsgId>) -> Self {
-        dc_array_t::MsgIds(array)
-    }
-}
-
-impl From<Vec<ChatItem>> for dc_array_t {
-    fn from(array: Vec<ChatItem>) -> Self {
-        dc_array_t::Chat(array)
-    }
-}
-
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -103,11 +118,12 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let arr: dc_array_t = Vec::<u32>::new().into();
-        assert!(arr.len() == 0);
+        let mut arr = dc_array_t::new(7);
+        assert!(arr.is_empty());
 
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
 
         assert_eq!(arr.len(), 1000);
 
@@ -115,15 +131,31 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        assert_eq!(arr.search_id(10), Some(8));
-        assert_eq!(arr.search_id(1), None);
+        arr.clear();
+
+        assert!(arr.is_empty());
+
+        arr.add_id(13);
+        arr.add_id(7);
+        arr.add_id(666);
+        arr.add_id(0);
+        arr.add_id(5000);
+
+        arr.sort_ids();
+
+        assert_eq!(arr.get_id(0), 0);
+        assert_eq!(arr.get_id(1), 7);
+        assert_eq!(arr.get_id(2), 13);
+        assert_eq!(arr.get_id(3), 666);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        let mut arr = dc_array_t::new(7);
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lot.rs

@@ -1,245 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use crate::message::MessageState;
-use crate::qr::Qr;
-use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Debug)]
-pub enum Lot {
-    Summary(Summary),
-    Qr(Qr),
-    Error(String),
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
-            },
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::FprOk { .. } => None,
-                Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
-                Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-            },
-            Self::Error(err) => Some(err),
-        }
-    }
-
-    pub fn get_text2(&self) -> Option<Cow<str>> {
-        match self {
-            Self::Summary(summary) => Some(summary.truncated_text(160)),
-            Self::Qr(_) => None,
-            Self::Error(_) => None,
-        }
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => Meaning::None,
-                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
-                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
-                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
-            },
-            Self::Qr(_qr) => Meaning::None,
-            Self::Error(_err) => Meaning::None,
-        }
-    }
-
-    pub fn get_state(&self) -> LotState {
-        match self {
-            Self::Summary(summary) => summary.state.into(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
-                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
-                Qr::FprOk { .. } => LotState::QrFprOk,
-                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
-                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
-                Qr::Account { .. } => LotState::QrAccount,
-                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Addr { .. } => LotState::QrAddr,
-                Qr::Url { .. } => LotState::QrUrl,
-                Qr::Text { .. } => LotState::QrText,
-                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
-                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
-                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
-                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
-            },
-            Self::Error(_err) => LotState::QrError,
-        }
-    }
-
-    pub fn get_id(&self) -> u32 {
-        match self {
-            Self::Summary(_) => Default::default(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
-                Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
-                Qr::FprWithoutAddr { .. } => Default::default(),
-                Qr::Account { .. } => Default::default(),
-                Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
-                Qr::Url { .. } => Default::default(),
-                Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
-            },
-            Self::Error(_) => Default::default(),
-        }
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        match self {
-            Self::Summary(summary) => summary.timestamp,
-            Self::Qr(_) => Default::default(),
-            Self::Error(_) => Default::default(),
-        }
-    }
-}
-
-#[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// text1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        Lot::Summary(summary)
-    }
-}
-
-impl From<Qr> for Lot {
-    fn from(qr: Qr) -> Self {
-        Lot::Qr(qr)
-    }
-}
-
-// Make it easy to convert errors into the final `Lot`.
-impl From<Error> for Lot {
-    fn from(error: Error) -> Self {
-        Lot::Error(error.to_string())
-    }
-}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -102,9 +105,8 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         })
     }
 
@@ -120,9 +122,8 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         }),
         None => Err(CStringError::NotUnicode),
     }
@@ -167,20 +168,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +184,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name
@@ -250,7 +239,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] slice can not outlive
+/// Because this returns a reference the [Path] silce can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1"
-quote = "1"
+syn = "1.0.13"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -35,11 +35,7 @@ pub fn from_sql_derive(input: TokenStream) -> TokenStream {
         impl rusqlite::types::FromSql for #name {
             fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
                 let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
+                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
             }
         }
     };

draft/aeap-mvp.rst

@@ -1,33 +0,0 @@
-AEAP MVP
-========
-
-Changes to the UIs
-------------------
-
-- The secondary self addresses (see below) are shown in the UI, but not editable.
-
-- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
-
-Changes in the core
--------------------
-
-- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
-
-- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
-
-  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
-
-  - The key stays the same.
-
-- No changes for 1:1 chats, there simply is a new one
-
-- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
-  
-  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
-
-  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
-
-Other
------
-
-- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/group-sync.rst

@@ -5,95 +5,69 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-1. Alice and Bob are in a two-member group
+- Alice and Bob are in a two-member group
 
-2. Then Alice adds Carol, while concurrently Bob adds Doris
+- Alice adds Carol, concurrently Bob adds Doris
 
-Right now, the group has inconsistent memberships:
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
 
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
 
 
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
 
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
 
 
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
 
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
 
-For brevity let's abbreviate "group membership modification" as **GMM**.
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
 
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
+Notes:
 
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
+- mechanism works for both encrypted and unencrypted add/del messages
 
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
 
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
+- multiple member-added/removed messages can be attached in a single
+  correction message
 
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
 
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
 
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
 
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
   b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -109,3 +83,44 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

draft/mailing_list.rst

@@ -0,0 +1,19 @@
+2. Mailing lists are now only detected by the ListId header, because how should DC sort mailing lists if the ListId header is unset and only the precedence header says it's a mailing list (?). We could just hide these emails (with "junk" or "list" precedence), as we did before, if there is a reason to do so. 
+
+   I do not actually understand why such emails were hidden in the first place, though; if there is an automatic answer stating that someone is out of office or if an email could not be delivered (these are the usual reasons why such emails are sent), I would want to know about this as a user.
+   
+  Björn: to the ListId-header: it is already an improvement to use only that and to keep ignoring mails with Precedence-header. historically, i ignored all these mails to reduce noise that is created from them (eg. contacts should not be added to the suggestion list...) and to be able to concentrate on other things first.
+   
+   
+3. for `List-Id: foo <bla>`, bla now is the id and foo is the name. For GitHub and GitLab notifications this is bad because all notifications will go into one chat. Maybe we should instead take the "in-reply-to" header to find out what messages belong together. For normal mailing lists, of course, this will create a second chat if someone does not use the "Reply" button to write to that mailing list.
+
+  Björn: well, having all notifications in one chat is not that bad. i would just follow the guidelines for now, use the ID and not the Name.
+
+
+4. Currently a mailing list is shown as an empty group (ChatType `Group`) ("0 members"). 
+
+   Maybe we should change the ChatType to `Single` because this way, the UI would fit better. Disadvantage: We can't show the different senders of the messages. 
+   
+   Or we introduce a `ChatType` `MailingList`. Very big disadvantage: We would have to adapt all UI project and it would be nice if we could keep the changes within the core.
+
+5. Contacts from mailings lists stay "unknown" now and are not shown in the contacts suggestion list..

draft/mailing_list_current_state.rst

@@ -0,0 +1,13 @@
+1. Mailing lists with Precedence-header and without List-Id are shown as normal messages.
+   
+2. for `List-Id: foo <bla>`, bla now is the id and foo is the name. If foo is not present, bla is taken as the name as well. For GitHub and GitLab notifications all notifications will go into one chat. To distinguish, all subjects are shown in mailing lists.
+
+3. Currently a mailing list is shown as a group with SELF and the List-Id. I could not find any stable way to get a mail address that could be called the "mailing list address". To get this to work, I disabled the may_be_valid_addr check. (to be discussed...)
+
+4. Contacts from mailings lists stay "unknown" and are not shown in the contacts suggestion list.
+
+5. In general, only the From: address is taken as the author. If the domain matches with the List-Id domain (like `deltachat.github.com` and `Hocuri <notifications@notification.github.com>`) then the display name is added to the email address -> `Hocuri - notifications@notification.github.com`. This is not really nice, but as the UIs distinguish bettween contacts only based on the address this was the only way I could find without changing the API. 
+
+   TODO: Prevent the user from sending emails to such contacts (like Hocuri - notifications@notification.github.com)
+
+6. You can't send to mailing lists.

draft/mailing_list_managers.md

@@ -0,0 +1,71 @@
+# Mailing list integration of Delta Chat
+
+A collection of information to help with the integration of mailing lists into Delta Chat.
+
+## Chat name
+
+How should the chat be titled?
+
+It could be taken from the email header `List-Id`:
+
+### Mailman 
+
+* Schema: `${list-description} <${list-local-part}.${list-domain}>`
+* $list-description could be many words, should be truncated. Could also be empty, though.
+
+### Schleuder
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Could be absent: <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L357-358>.
+
+
+### Sympa
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Always present <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/Spindle/TransformOutgoing.pm#L110-L118>, <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/List.pm#L6832-L6833>.
+
+
+## Sender name
+
+What's the name of the person that actually sent the message?
+
+Some MLM change the sender information to avoid problems with DMARC.
+
+### Mailman
+
+* Since v2.1.16 the `From` depends on the list's configuration and possibly on the sender-domain's DMARC-configuration — i.e. messages from the same list can arrive one of the Variants A-D!
+* Documentation of config options:
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-544>,
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-163>,
+ * <https://wiki.list.org/DEV/DMARC>
+* Variant A: `From` is unchanged.
+* Variant B. `From` is mangled like this: `${sender-name} via ${list-name} <${list-addr-spec}>`. The original sender is put into `Reply-To`.
+* Variant C. `From` is mangled like this: `${sender-name} <${encoded-sender-addr-spec}@${list-domain}>`. The original sender is put into `Reply-To`.
+* Variant D: `From` is set to ${list-name-addr}, the originally sent message is included as mime-part.
+* Variant E: `From` is set to ${list-name-addr}, original sender information is removed.
+
+### Schleuder
+
+* Visible only in mime-body (which is possibly encrypted), or not at all.
+* The first `text/plain` mime-part may include the information, as taken from the original message: `From: ${sender-name-addr}`.
+
+### Sympa
+* Depends on the list's configuration.
+* Documentation:
+ * <https://sympa-community.github.io/manual/customize/dmarc-protection.html>,
+ * <https://sympa-community.github.io/gpldoc/man/sympa.conf.5.html#dmarc-protection>.
+* Variant A: `From` is unchanged.
+* Variant B: `From` is mangled like Mailman Variant B, but the original sender information is put into `X-Original-From`.
+
+
+## Autocrypt
+
+### Mailman
+* Not supported.
+
+### Schleuder
+* A list's key is included in sent messages (as of version 3.5.0) (optional, by default active): <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L349-355>.
+* Incoming keys are not yet looked at (that feature is planned: <https://0xacab.org/schleuder/schleuder/issues/435>).
+
+### Sympa
+* Not supported.

draft/msgwork_new_imap_jobs.rst

@@ -1,66 +0,0 @@
-
-simplify/streamline mark-seen/delete/move/send-mdn job handling
----------------------------------------------------------------
-
-Idea: Introduce a new "msgwork" sql table that looks very
-much like the jobs table but has a primary key "msgid"
-and no job id and no foreign-id anymore. This opens up
-bulk-processing by looking at the whole table and combining
-flag-setting to reduce imap-roundtrips and select-folder calls.
-
-Concretely, these IMAP jobs:
-
-    DeleteMsgOnImap
-    MarkseenMsgOnImap
-    MoveMsg
-
-Would be replaced by a few per-message columns in the new msgwork table:
-
-- needs_mark_seen: (bool) message shall be marked as seen on imap
-- needs_to_move: (bool) message should be moved to mvbox_folder
-- deletion_time: (target_time or 0) message shall be deleted at specified time
-- needs_send_mdn: (bool) MDN shall be sent
-
-The various places that currently add the (replaced) jobs
-would now add/modify the respective message record in the message-work table.
-
-Looking at a single message-work entry conceptually looks like this::
-
-    if msg.server_uid==0:
-        return RetryLater  # nothing can be done without server_uid
-
-    if msg.deletion_time > current_time:
-        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
-        clear(needs_deletion)
-        clear(mark_seen)
-
-    if needs_mark_seen:
-        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
-        clear(needs_mark_seen)
-
-    if needs_send_mdn:
-        schedule_smtp_send_mdn(msg)
-        clear(needs_send_mdn)
-
-    if any_flag_set():
-        retrylater
-    # remove msgwork entry from table
-
-
-Notes/Questions:
-
-- it's unclear how much we need per-message retry-time tracking/backoff
-
-- drafting bulk processing algo is useful before
-  going for the implementation, i.e. including select_folder calls etc.
-
-- maybe it's better to not have bools for the flags but
-
-  0 (no change)
-  1 (set the imap flag)
-  2 (clear the imap flag)
-
-  and design such that we can cover all imap flags.
-
-- It might not be neccessary to keep needs_send_mdn state in this table
-  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/main.rs

@@ -6,120 +6,211 @@
 
 #[macro_use]
 extern crate deltachat;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
 
-use ansi_term::Color;
 use anyhow::{bail, Error};
-use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
+use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
-use log::{error, info, warn};
+use deltachat::Event;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
-/// Event Handler
-fn receive_event(event: EventType) {
-    let yellow = Color::Yellow.normal();
+// Event Handler
+
+fn receive_event(_context: &Context, event: Event) {
     match event {
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            info!("{}", msg);
+            println!("{}", msg);
         }
-        EventType::SmtpConnected(msg) => {
-            info!("[SMTP_CONNECTED] {}", msg);
+        Event::SmtpConnected(msg) => {
+            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
         }
-        EventType::ImapConnected(msg) => {
-            info!("[IMAP_CONNECTED] {}", msg);
+        Event::ImapConnected(msg) => {
+            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
         }
-        EventType::SmtpMessageSent(msg) => {
-            info!("[SMTP_MESSAGE_SENT] {}", msg);
+        Event::SmtpMessageSent(msg) => {
+            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
         }
-        EventType::Warning(msg) => {
-            warn!("{}", msg);
+        Event::Warning(msg) => {
+            println!("[Warning] {}", msg);
         }
-        EventType::Error(msg) => {
-            error!("{}", msg);
+        Event::Error(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
         }
-        EventType::ErrorSelfNotInGroup(msg) => {
-            error!("[SELF_NOT_IN_GROUP] {}", msg);
+        Event::ErrorNetwork(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
         }
-        EventType::MsgsChanged { chat_id, msg_id } => {
-            info!(
-                "{}",
-                yellow.paint(format!(
-                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
-                    chat_id, msg_id,
-                ))
+        Event::ErrorSelfNotInGroup(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
+        }
+        Event::MsgsChanged { chat_id, msg_id } => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
+                chat_id, msg_id,
             );
         }
-        EventType::ContactsChanged(_) => {
-            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
+        Event::ContactsChanged(_) => {
+            print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
         }
-        EventType::LocationChanged(contact) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+        Event::LocationChanged(contact) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
+                contact,
             );
         }
-        EventType::ConfigureProgress { progress, comment } => {
-            if let Some(comment) = comment {
-                info!(
-                    "{}",
-                    yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
-                    ))
-                );
-            } else {
-                info!(
-                    "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-                );
-            }
-        }
-        EventType::ImexProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+        Event::ConfigureProgress(progress) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
+                progress,
             );
         }
-        EventType::ImexFileWritten(file) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
+        Event::ImexProgress(progress) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
+                progress,
             );
         }
-        EventType::ChatModified(chat) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
+        Event::ImexFileWritten(file) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
+                file.display()
+            );
+        }
+        Event::ChatModified(chat) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
+                chat
             );
         }
         _ => {
-            info!("Received {:?}", event);
+            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
 }
 
+// Threads for waiting for messages and for jobs
+
+lazy_static! {
+    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
+    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
+}
+
+struct Handle {
+    handle_imap: Option<std::thread::JoinHandle<()>>,
+    handle_mvbox: Option<std::thread::JoinHandle<()>>,
+    handle_sentbox: Option<std::thread::JoinHandle<()>>,
+    handle_smtp: Option<std::thread::JoinHandle<()>>,
+}
+
+macro_rules! while_running {
+    ($code:block) => {
+        if IS_RUNNING.load(Ordering::Relaxed) {
+            $code
+        } else {
+            break;
+        }
+    };
+}
+
+fn start_threads(c: Arc<RwLock<Context>>) {
+    if HANDLE.clone().lock().unwrap().is_some() {
+        return;
+    }
+
+    println!("Starting threads");
+    IS_RUNNING.store(true, Ordering::Relaxed);
+
+    let ctx = c.clone();
+    let handle_imap = std::thread::spawn(move || loop {
+        while_running!({
+            perform_inbox_jobs(&ctx.read().unwrap());
+            perform_inbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                let context = ctx.read().unwrap();
+                perform_inbox_idle(&context);
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_mvbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_mvbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_mvbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_sentbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_sentbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_sentbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c;
+    let handle_smtp = std::thread::spawn(move || loop {
+        while_running!({
+            perform_smtp_jobs(&ctx.read().unwrap());
+            while_running!({
+                perform_smtp_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    *HANDLE.clone().lock().unwrap() = Some(Handle {
+        handle_imap: Some(handle_imap),
+        handle_mvbox: Some(handle_mvbox),
+        handle_sentbox: Some(handle_sentbox),
+        handle_smtp: Some(handle_smtp),
+    });
+}
+
+fn stop_threads(context: &Context) {
+    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
+        println!("Stopping threads");
+        IS_RUNNING.store(false, Ordering::Relaxed);
+
+        interrupt_inbox_idle(context);
+        interrupt_mvbox_idle(context);
+        interrupt_sentbox_idle(context);
+        interrupt_smtp_idle(context);
+
+        handle.handle_imap.take().unwrap().join().unwrap();
+        handle.handle_mvbox.take().unwrap().join().unwrap();
+        handle.handle_sentbox.take().unwrap().join().unwrap();
+        handle.handle_smtp.take().unwrap().join().unwrap();
+    }
+}
+
 // === The main loop
 
 struct DcHelper {
@@ -156,27 +247,28 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 11] = [
     "info",
+    "open",
+    "close",
     "set",
     "get",
     "oauth2",
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 35] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "createchatbymsg",
     "creategroup",
-    "createbroadcast",
-    "createprotected",
+    "createverified",
     "addmember",
     "removemember",
     "groupname",
@@ -189,49 +281,36 @@ const CHAT_COMMANDS: [&str; 35] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
-    "sendsyncmsg",
-    "videochat",
     "draft",
     "listmedia",
     "archive",
     "unarchive",
     "pin",
     "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
-    "download",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 10] = [
     "getqr",
-    "getqrsvg",
     "getbadqr",
     "checkqr",
-    "joinqr",
     "event",
     "fileinfo",
     "clear",
@@ -242,9 +321,7 @@ const MISC_COMMANDS: [&str; 12] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -266,10 +343,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -290,83 +368,70 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
-async fn start(args: Vec<String>) -> Result<(), Error> {
+fn main_0(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
-
-    let events = context.get_event_emitter();
-    async_std::task::spawn(async move {
-        while let Some(event) = events.recv().await {
-            receive_event(event.typ);
-        }
-    });
+    let context = Context::new(
+        Box::new(receive_event),
+        "CLI".into(),
+        Path::new(&args[1]).to_path_buf(),
+    )?;
 
     println!("Delta Chat Core is awaiting your commands.");
 
+    let ctx = Arc::new(RwLock::new(context));
+
     let config = Config::builder()
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
-    let input_loop = async_std::task::spawn_blocking(move || {
-        let h = DcHelper {
-            completer: FilenameCompleter::new(),
-            highlighter: MatchingBracketHighlighter::new(),
-            hinter: HistoryHinter {},
-        };
-        let mut rl = Editor::with_config(config);
-        rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
-        if rl.load_history(".dc-history.txt").is_err() {
-            println!("No previous history.");
-        }
+    let h = DcHelper {
+        completer: FilenameCompleter::new(),
+        highlighter: MatchingBracketHighlighter::new(),
+        hinter: HistoryHinter {},
+    };
+    let mut rl = Editor::with_config(config);
+    rl.set_helper(Some(h));
+    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+    if rl.load_history(".dc-history.txt").is_err() {
+        println!("No previous history.");
+    }
 
-        loop {
-            let p = "> ";
-            let readline = rl.readline(p);
-
-            match readline {
-                Ok(line) => {
-                    // TODO: ignore "set mail_pw"
-                    rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
-                }
-                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                    println!("Exiting...");
-                    drop(reader_s);
-                    break;
-                }
-                Err(err) => {
-                    println!("Error: {}", err);
-                    drop(reader_s);
-                    break;
+    loop {
+        let p = "> ";
+        let readline = rl.readline(&p);
+        match readline {
+            Ok(line) => {
+                // TODO: ignore "set mail_pw"
+                rl.add_history_entry(line.as_str());
+                let ctx = ctx.clone();
+                match handle_cmd(line.trim(), ctx) {
+                    Ok(ExitResult::Continue) => {}
+                    Ok(ExitResult::Exit) => break,
+                    Err(err) => println!("Error: {}", err),
                 }
             }
-        }
-
-        rl.save_history(".dc-history.txt")?;
-        println!("history saved");
-        Ok::<_, Error>(())
-    });
-
-    while let Ok(line) = reader_r.recv().await {
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
+            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                println!("Exiting...");
+                break;
+            }
+            Err(err) => {
+                println!("Error: {}", err);
+                break;
+            }
         }
     }
-    context.stop_io().await;
-    input_loop.await?;
+    rl.save_history(".dc-history.txt")?;
+    println!("history saved");
+    {
+        stop_threads(&ctx.read().unwrap());
+    }
 
     Ok(())
 }
@@ -377,29 +442,43 @@ enum ExitResult {
     Exit,
 }
 
-async fn handle_cmd(
-    line: &str,
-    ctx: Context,
-    selected_chat: &mut ChatId,
-) -> Result<ExitResult, Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
 
     match arg0 {
         "connect" => {
-            ctx.start_io().await;
+            start_threads(ctx);
         }
         "disconnect" => {
-            ctx.stop_io().await;
+            stop_threads(&ctx.read().unwrap());
+        }
+        "smtp-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("smtp-jobs are already running in a thread.",);
+            } else {
+                perform_smtp_jobs(&ctx.read().unwrap());
+            }
+        }
+        "imap-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("inbox-jobs are already running in a thread.");
+            } else {
+                perform_inbox_jobs(&ctx.read().unwrap());
+            }
         }
         "configure" => {
-            ctx.configure().await?;
+            start_threads(ctx.clone());
+            ctx.read().unwrap().configure();
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
-                let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+            if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
+                let oauth2_url = dc_get_oauth2_url(
+                    &ctx.read().unwrap(),
+                    &addr,
+                    "chat.delta:/com.b44t.messenger",
+                );
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -414,54 +493,43 @@ async fn handle_cmd(
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
-            if !qr.is_empty() {
-                if arg0 == "getbadqr" && qr.len() > 40 {
-                    qr.replace_range(12..22, "0000000000")
-                }
-                println!("{}", qr);
-                let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
-            }
-        }
-        "getqrsvg" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            match get_securejoin_qr_svg(&ctx, group).await {
-                Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get QR code svg: {}", err);
+            start_threads(ctx.clone());
+            if let Some(mut qr) = dc_get_securejoin_qr(
+                &ctx.read().unwrap(),
+                ChatId::new(arg1.parse().unwrap_or_default()),
+            ) {
+                if !qr.is_empty() {
+                    if arg0 == "getbadqr" && qr.len() > 40 {
+                        qr.replace_range(12..22, "0000000000")
+                    }
+                    println!("{}", qr);
+                    let output = Command::new("qrencode")
+                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                        .output()
+                        .expect("failed to execute process");
+                    io::stdout().write_all(&output.stdout).unwrap();
+                    io::stderr().write_all(&output.stderr).unwrap();
                 }
             }
         }
         "joinqr" => {
-            ctx.start_io().await;
+            start_threads(ctx.clone());
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                dc_join_securejoin(&ctx.read().unwrap(), arg1);
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
-        _ => cmdline(ctx.clone(), line, selected_chat).await?,
+        _ => dc_cmdline(&ctx.read().unwrap(), line)?,
     }
 
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+pub fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
-    let args = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    let args: Vec<String> = std::env::args().collect();
+    main_0(args)?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,100 +1,111 @@
+extern crate deltachat;
+
+use std::sync::{Arc, RwLock};
+use std::{thread, time};
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::job::{
+    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
+    perform_smtp_jobs,
+};
+use deltachat::Event;
+
+fn cb(_ctx: &Context, event: Event) {
+    print!("[{:?}]", event);
 
-fn cb(event: EventType) {
     match event {
-        EventType::ConfigureProgress { progress, .. } => {
-            log::info!("progress: {}", progress);
+        Event::ConfigureProgress(progress) => {
+            println!("  progress: {}", progress);
         }
-        EventType::Info(msg) => {
-            log::info!("{}", msg);
+        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            println!("  {}", msg);
         }
-        EventType::Warning(msg) => {
-            log::warn!("{}", msg);
-        }
-        EventType::Error(msg) => {
-            log::error!("{}", msg);
-        }
-        event => {
-            log::info!("{:?}", event);
+        _ => {
+            println!();
         }
     }
 }
 
-/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
-#[async_std::main]
-async fn main() {
-    pretty_env_logger::try_init_timed().ok();
-
+fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
-        .await
-        .expect("Failed to create context");
-    let info = ctx.get_info().await;
-    log::info!("info: {:#?}", info);
+    println!("creating database {:?}", dbfile);
+    let ctx =
+        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+    let running = Arc::new(RwLock::new(true));
+    let info = ctx.get_info();
+    let duration = time::Duration::from_millis(4000);
+    println!("info: {:#?}", info);
 
-    let events = ctx.get_event_emitter();
-    let events_spawn = async_std::task::spawn(async move {
-        while let Some(event) = events.recv().await {
-            cb(event.typ);
+    let ctx = Arc::new(ctx);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t1 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_inbox_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_inbox_fetch(&ctx1);
+
+                if *r1.read().unwrap() {
+                    perform_inbox_idle(&ctx1);
+                }
+            }
         }
     });
 
-    log::info!("configuring");
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t2 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_smtp_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_smtp_idle(&ctx1);
+            }
+        }
+    });
+
+    println!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 3, "requires email password");
-    let email = args[1].clone();
-    let pw = args[2].clone();
-    ctx.set_config(config::Config::Addr, Some(&email))
-        .await
-        .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw))
-        .await
+    assert_eq!(args.len(), 2, "missing password");
+    let pw = args[1].clone();
+    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+    ctx.configure();
 
-    ctx.configure().await.unwrap();
+    thread::sleep(duration);
 
-    log::info!("------ RUN ------");
-    ctx.start_io().await;
-    log::info!("--- SENDING A MESSAGE ---");
+    println!("sending a message");
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
 
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
-        .await
-        .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
-
-    for i in 0..1 {
-        log::info!("sending message {}", i);
-        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
-            .await
-            .unwrap();
-    }
-
-    // wait for the message to be sent out
-    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
-
-    log::info!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
+    println!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
-            .await
-            .unwrap();
-        log::info!("[{}] msg: {:?}", i, msg);
+        let summary = chats.get_summary(&ctx, 0, None);
+        let text1 = summary.get_text1();
+        let text2 = summary.get_text2();
+        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
     }
 
-    log::info!("stopping");
-    ctx.stop_io().await;
-    log::info!("closing");
-    drop(ctx);
-    events_spawn.await;
+    thread::sleep(duration);
+
+    println!("stopping threads");
+
+    *running.write().unwrap() = false;
+    deltachat::job::interrupt_inbox_idle(&ctx);
+    deltachat::job::interrupt_smtp_idle(&ctx);
+
+    println!("joining");
+    t1.join().unwrap();
+    t2.join().unwrap();
+
+    println!("closing");
 }

proptest-regressions/dc_tools.txt

@@ -7,5 +7,3 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
-cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
-cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/CHANGELOG

@@ -1,29 +1,6 @@
-1.51.0
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
-1.44.0
-------
-
-- fix Chat.get_mute_duration()
-
-1.40.1
+0.900.0 (DRAFT)
 ---------------
 
-- emit "ac_member_removed" event (with 'actor' being the removed contact) 
-  for when a user leaves a group.
-
-- fix create_contact(addr) when addr is the self-contact. 
-
-
-1.40.0
----------------
-
-- uses latest 1.40+ Delta Chat core
-
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -33,7 +10,6 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
-
 0.800.0
 -------
 

python/README.rst

@@ -7,14 +7,76 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -41,79 +103,6 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Running tests
-=============
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-This will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _livetests:
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
-
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
-
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
-
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
-
-Each test run creates new accounts.
-
-
-.. _sourceinstall:
-
-Installing bindings from source (Updated: July 2020)
-=========================================================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   python install_python_bindings.py
-
-The core compilation and bindings building might take a while,
-depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
 Code examples
 =============
 
@@ -124,10 +113,10 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux based wheels
-====================================
+Building manylinux1 based wheels
+================================
 
-Building portable manylinux wheels which come with libdeltachat.so
+Building portable manylinux1 wheels which come with libdeltachat.so
 can be done with docker-tooling.
 
 using docker pull / premade images
@@ -143,7 +132,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -152,9 +141,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/doc/_templates/globaltoc.html

@@ -9,7 +9,8 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
+  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/echo_and_quit.py

@@ -12,13 +12,10 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
-            if message.is_system_message():
-                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
-            else:
-                text = message.text
-                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
     @account_hookimpl
     def ac_message_delivered(self, message):

python/examples/group_tracking.py

@@ -12,36 +12,24 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
     @account_hookimpl
     def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
+        print("*** ac_configure_completed:", success)
 
     @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
+    def ac_member_added(self, chat, contact):
+        print("*** ac_member_added", contact.addr, "from", chat)
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, actor, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, actor, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+    def ac_member_removed(self, chat, contact):
+        print("*** ac_member_removed", contact.addr, "from", chat)
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -3,7 +3,7 @@ import pytest
 import py
 import echo_and_quit
 import group_tracking
-from deltachat.events import FFIEventLogger
+from deltachat.eventlogger import FFIEventLogger
 
 
 @pytest.fixture(scope='session')
@@ -17,48 +17,41 @@ def datadir():
         pytest.skip('test-data directory not found')
 
 
-def test_echo_quit_plugin(acfactory, lp):
-    lp.sec("creating one echo_and_quit bot")
+def test_echo_quit_plugin(acfactory):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
-    lp.sec("creating a temp account to contact the bot")
     ac1 = acfactory.get_one_online_account()
-
-    lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    bot_chat = bot_contact.create_chat()
-    bot_chat.send_text("hello")
-
-    lp.sec("waiting for the reply message from the bot to arrive")
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
     reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    lp.sec("send quit sequence")
-    bot_chat.send_text("/quit")
+    assert reply.chat == ch1
+    ch1.send_text("/quit")
     botproc.wait()
 
 
 def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
+    botproc = acfactory.run_bot_process(group_tracking)
 
     ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
-        *ac_configure_completed*
+        *ac_configure_completed: True*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1))
-    ac2.add_account_plugin(FFIEventLogger(ac2))
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
 
-    lp.sec("creating bot test group with bot")
+    lp.sec("creating bot test group with all three")
     bot_contact = ac1.create_contact(botproc.addr)
     ch = ac1.create_group_chat("bot test group")
     ch.add_contact(bot_contact)
     ch.send_text("hello")
 
     botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
+        *ac_member_added {}*
+    """.format(ac1.get_config("addr")))
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))
@@ -69,11 +62,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_added {}*
+    """.format(contact3.addr))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -17,12 +17,8 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-
     if target == 'release':
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
-
-    print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 

python/mypy.ini

@@ -1,19 +0,0 @@
-[mypy]
-
-[mypy-deltachat.capi.*]
-ignore_missing_imports = True
-
-[mypy-pluggy.*]
-ignore_missing_imports = True
-
-[mypy-cffi.*]
-ignore_missing_imports = True
-
-[mypy-imapclient.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-_pytest.*]
-ignore_missing_imports = True

python/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,11 +8,17 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
+        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
+        use_scm_version = {
+            "root": "..",
+            "relative_to": __file__,
+            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
+        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
-        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -1,13 +1,13 @@
 import sys
 
-from . import capi, const, hookspec # noqa
-from .capi import ffi  # noqa
+from . import capi, const, hookspec
+from .capi import ffi
 from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa
 from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import events
+from . import eventlogger
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -17,11 +17,69 @@ except DistributionNotFound:
     __version__ = "0.0.0.dev0-unknown"
 
 
+_DC_CALLBACK_MAP = {}
+
+
+@capi.ffi.def_extern()
+def py_dc_callback(ctx, evt, data1, data2):
+    """The global event handler.
+
+    CFFI only allows us to set one global event handler, so this one
+    looks up the correct event handler for the given context.
+    """
+    try:
+        callback = _DC_CALLBACK_MAP.get(ctx, lambda *a: 0)
+    except AttributeError:
+        # we are in a deep in GC-free/interpreter shutdown land
+        # nothing much better to do here than:
+        return 0
+
+    # the following code relates to the deltachat/_build.py's helper
+    # function which provides us signature info of an event call
+    evt_name = get_dc_event_name(evt)
+    event_sig_types = capi.lib.dc_get_event_signature_types(evt)
+    if data1 and event_sig_types & 1:
+        data1 = ffi.string(ffi.cast('char*', data1)).decode("utf8")
+    if data2 and event_sig_types & 2:
+        data2 = ffi.string(ffi.cast('char*', data2)).decode("utf8")
+        try:
+            if isinstance(data2, bytes):
+                data2 = data2.decode("utf8")
+        except UnicodeDecodeError:
+            # XXX ignoring the decode error is not quite correct but for now
+            # i don't want to hunt down encoding problems in the c lib
+            pass
+    try:
+        ret = callback(ctx, evt_name, data1, data2)
+        if ret is None:
+            ret = 0
+        assert isinstance(ret, int), repr(ret)
+        if event_sig_types & 4:
+            return ffi.cast('uintptr_t', ret)
+        elif event_sig_types & 8:
+            return ffi.cast('int', ret)
+    except:  # noqa
+        raise
+        ret = 0
+    return ret
+
+
+def set_context_callback(dc_context, func):
+    _DC_CALLBACK_MAP[dc_context] = func
+
+
+def clear_context_callback(dc_context):
+    try:
+        _DC_CALLBACK_MAP.pop(dc_context, None)
+    except AttributeError:
+        pass
+
+
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name in dir(const):
+        for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[getattr(const, name)] = name
+                _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -39,7 +97,7 @@ def unregister_global_plugin(plugin):
     gm.unregister(plugin)
 
 
-register_global_plugin(events)
+register_global_plugin(eventlogger)
 
 
 def run_cmdline(argv=None, account_plugins=None):
@@ -60,14 +118,9 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
+        log = eventlogger.FFIEventLogger(ac, "bot")
         ac.add_account_plugin(log)
 
-    for plugin in account_plugins or []:
-        print("adding plugin", plugin)
-        ac.add_account_plugin(plugin)
-
     if not ac.is_configured():
         assert args.email and args.password, (
             "you must specify --email and --password once to configure this database/account"
@@ -75,13 +128,14 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
+
+    for plugin in account_plugins or []:
+        ac.add_account_plugin(plugin)
 
     # start IO threads and configure if neccessary
-    ac.start_io()
+    ac.start()
 
     print("{}: waiting for message".format(ac.get_config("addr")))
 

python/src/deltachat/_build.py

@@ -1,63 +1,79 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import os
-import platform
-import re
-import shutil
-import subprocess
 import tempfile
-import textwrap
-import types
-
+import platform
+import os
 import cffi
+import shutil
+from os.path import dirname as dn
+from os.path import abspath
 
 
-def local_build_flags(projdir, target):
-    """Construct build flags for building against a checkout.
-
-    :param projdir: The root directory of the deltachat-core-rust project.
-    :param target: The rust build target, `debug` or `release`.
-    """
-    flags = types.SimpleNamespace()
-    if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
-        ]
-    elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        if platform.system() == 'Darwin':
+            libs = ['resolv', 'dl']
+            extra_link_args = [
+                    '-framework', 'CoreFoundation',
+                    '-framework', 'CoreServices',
+                    '-framework', 'Security',
+            ]
+        elif platform.system() == 'Linux':
+            libs = ['rt', 'dl', 'm']
+            extra_link_args = []
+        else:
+            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
-        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-    target_dir = os.environ.get("CARGO_TARGET_DIR")
-    if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
-    return flags
-
-
-def system_build_flags():
-    """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-    return flags
-
-
-def extract_functions(flags):
-    """Extract the function definitions from deltachat.h.
-
-    This creates a .h file with a single `#include <deltachat.h>` line
-    in it.  It then runs the C preprocessor to create an output file
-    which contains all function definitions found in `deltachat.h`.
-    """
+        libs = ['deltachat']
+        objs = []
+        incs = []
+        extra_link_args = []
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            const char * dupstring_helper(const char* string)
+            {
+                return strdup(string);
+            }
+            int dc_get_event_signature_types(int e)
+            {
+                int result = 0;
+                if (DC_EVENT_DATA1_IS_STRING(e))
+                    result |= 1;
+                if (DC_EVENT_DATA2_IS_STRING(e))
+                    result |= 2;
+                if (DC_EVENT_RETURNS_STRING(e))
+                    result |= 4;
+                if (DC_EVENT_RETURNS_INT(e))
+                    result |= 8;
+                return result;
+            }
+        """,
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
+        extra_link_args=extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern const char * dupstring_helper(const char* string);
+        extern int dc_get_event_signature_types(int);
+    """)
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -69,134 +85,20 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            return dst_fp.read()
+            builder.cdef(dst_fp.read())
     finally:
         shutil.rmtree(tmpdir)
 
-
-def find_header(flags):
-    """Use the compiler to find the deltachat.h header location.
-
-    This uses a small utility in deltachat.h to find the location of
-    the header file location.
-    """
-    distutils.log.set_verbosity(distutils.log.INFO)
-    cc = distutils.ccompiler.new_compiler(force=True)
-    distutils.sysconfig.customize_compiler(cc)
-    tmpdir = tempfile.mkdtemp()
-    try:
-        src_name = os.path.join(tmpdir, "where.c")
-        obj_name = os.path.join(tmpdir, "where.o")
-        dst_name = os.path.join(tmpdir, "where")
-        with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
-                #include <stdio.h>
-                #include <deltachat.h>
-
-                int main(void) {
-                    printf("%s", _dc_header_file_location());
-                    return 0;
-                }
-            """))
-        cwd = os.getcwd()
-        try:
-            os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
-                       macros=[("PY_CFFI_INC", "1")])
-        finally:
-            os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
-        return subprocess.check_output(dst_name)
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-def extract_defines(flags):
-    """Extract the required #DEFINEs from deltachat.h.
-
-    Since #DEFINEs are interpreted by the C preprocessor we can not
-    use the compiler to extract these and need to parse the header
-    file ourselves.
-
-    The defines are returned in a string that can be passed to CFFIs
-    cdef() method.
-    """
-    header = find_header(flags)
-    defines_re = re.compile(r"""
-        \#define\s+   # The start of a define.
-        (             # Begin capturing group which captures the define name.
-            (?:       # A nested group which is not captured, this allows us
-                      #    to build the list of prefixes to extract without
-                      #    creation another capture group.
-                DC_EVENT
-                | DC_QR
-                | DC_MSG
-                | DC_LP
-                | DC_EMPTY
-                | DC_CERTCK
-                | DC_STATE
-                | DC_STR
-                | DC_CONTACT_ID
-                | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
-                | DC_CHAT
-                | DC_PROVIDER
-                | DC_KEY_GEN
-                | DC_IMEX
-                | DC_CONNECTIVITY
-            )         # End of prefix matching
-            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
-        )             # Close the capturing group, this contains
-                      # the entire name e.g. DC_MSG_TEXT.
-        \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
-    defines = []
-    with open(header) as fp:
-        for line in fp:
-            match = defines_re.match(line)
-            if match:
-                defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
-
-
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
-        flags = local_build_flags(projdir, target)
-    else:
-        flags = system_build_flags()
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
-    )
     builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
+        extern "Python" uintptr_t py_dc_callback(
+            dc_context_t* context,
+            int event,
+            uintptr_t data1,
+            uintptr_t data2);
     """)
-    function_defs = extract_functions(flags)
-    defines = extract_defines(flags)
-    builder.cdef(function_defs)
-    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -1,21 +1,22 @@
 """ Account class implementation. """
 
 from __future__ import print_function
+import atexit
 from contextlib import contextmanager
 from email.utils import parseaddr
+import queue
 from threading import Event
 import os
 from array import array
+import deltachat
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker, ConfigureTracker
-from . import hookspec
-from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
+from .tracker import ImexTracker
+from . import hookspec, iothreads
 
 
 class MissingCredentials(ValueError):
@@ -29,7 +30,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -38,48 +39,49 @@ class Account(object):
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
-        self._logging = logging
-
         self.add_account_plugin(self)
 
+        self._dc_context = ffi.gc(
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
+            _destroy_dc_context,
+        )
+
+        hook = hookspec.Global._get_plugin_manager().hook
+
+        self._threads = iothreads.IOThreads(self)
+        self._hook_event_queue = queue.Queue()
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
+
+        # open database
         self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
-
-        self._dc_context = ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
-            lib.dc_context_unref,
-        )
-        if self._dc_context == ffi.NULL:
-            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
-
-        self._shutdown_event = Event()
-        self._event_thread = EventThread(self)
+        if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
+            raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
-        hook = hookspec.Global._get_plugin_manager().hook
+        atexit.register(self.shutdown)
         hook.dc_account_init(account=self)
 
-    def disable_logging(self) -> None:
-        """ disable logging. """
-        self._logging = False
-
-    def enable_logging(self) -> None:
-        """ re-enable logging. """
-        self._logging = True
+    @hookspec.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        name, kwargs = self._map_ffi_event(ffi_event)
+        if name is not None:
+            ev = HookEvent(self, name=name, kwargs=kwargs)
+            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
-    def log(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
+    def ac_log_line(self, msg):
+        self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name: str) -> None:
+    def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self) -> Dict[str, str]:
+    def get_info(self):
         """ return dictionary of built config parameters. """
         lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
         d = {}
@@ -90,35 +92,19 @@ class Account(object):
             d[key.lower()] = value
         return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
-    def set_stock_translation(self, id: int, string: str) -> None:
+    def set_stock_translation(self, id, string):
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        bytestring = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name: str, value: Optional[str]) -> None:
+    def set_config(self, name, value):
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -126,16 +112,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
+        name = name.encode("utf8")
+        if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            valuebytes = value.encode("utf8")
+            value = value.encode("utf8")
         else:
-            valuebytes = ffi.NULL
-        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
+            value = ffi.NULL
+        lib.dc_set_config(self._dc_context, name, value)
 
-    def get_config(self, name: str) -> str:
+    def get_config(self, name):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -144,12 +130,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, namebytes)
+        name = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, name)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
+    def _preconfigure_keypair(self, addr, public, secret):
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -161,7 +147,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs: Dict[str, Any]) -> None:
+    def update_config(self, kwargs):
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -171,15 +157,15 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self) -> bool:
+    def is_configured(self):
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return bool(lib.dc_is_configured(self._dc_context))
 
-    def set_avatar(self, img_path: Optional[str]) -> None:
+    def set_avatar(self, img_path):
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -191,18 +177,31 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self) -> None:
+    def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir) -> Optional[str]:
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
+    def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        return from_optional_dc_charpointer(res)
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
-    def get_blobdir(self) -> str:
+    def get_blobdir(self):
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -210,100 +209,44 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self) -> Contact:
+    def get_self_contact(self):
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
-        """create a (new) Contact or return an existing one.
+    def create_contact(self, email, name=None):
+        """ create a (new) Contact. If there already is a Contact
+        with that e-mail address, it is unblocked and its name is
+        updated.
 
-        Calling this method will always result in the same
-        underlying contact id.  If there already is a Contact
-        with that e-mail address, it is unblocked and its display
-        `name` is updated if specified.
-
-        :param obj: email-address, Account or Contact instance.
-        :param name: (optional) display name for this contact
+        :param email: email-address (text type)
+        :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return Contact(self, contact_id)
 
-    def get_contact(self, obj) -> Optional[Contact]:
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
-        if isinstance(obj, Account):
-            if not obj.is_configured():
-                raise ValueError("can only add addresses from configured accounts")
-            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
-        elif isinstance(obj, Contact):
-            if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
-            addr, displayname = obj.addr, obj.name
-        elif isinstance(obj, str):
-            displayname, addr = parseaddr(obj)
-        else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
-
-        if name is None and displayname:
-            name = displayname
-        return (name, addr)
-
-    def delete_contact(self, contact: Contact) -> bool:
+    def delete_contact(self, contact):
         """ delete a Contact.
 
         :param contact: contact object obtained
         :returns: True if deletion succeeded (contact was deleted)
         """
         contact_id = contact.id
-        assert contact.account == self
+        assert contact._dc_context == self._dc_context
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-        return None
-
-    def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
-        :param contact_id: integer id of this contact.
-        :returns: :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
-    def get_blocked_contacts(self) -> List[Contact]:
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
-    def get_contacts(
-        self,
-        query: Optional[str] = None,
-        with_self: bool = False,
-        only_verified: bool = False,
-    ) -> List[Contact]:
-        """get a (filtered) list of contacts.
+    def get_contacts(self, query=None, with_self=False, only_verified=False):
+        """ get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -323,41 +266,55 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self) -> Generator[Message, None, None]:
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
+    def create_chat_by_contact(self, contact):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
-    def create_chat(self, obj) -> Chat:
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
-        return self.create_contact(obj).create_chat()
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(contact, "id"):
+            if contact._dc_context != self._dc_context:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
+        return Chat(self, chat_id)
 
-    def create_group_chat(
-        self,
-        name: str,
-        contacts: Optional[List[Contact]] = None,
-        verified: bool = False,
-    ) -> Chat:
-        """create a new group chat object.
+    def create_chat_by_message(self, message):
+        """ create or get an existing chat object for the
+        the specified message.
+
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
+        :param message: messsage id or message instance.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(message, "id"):
+            if self._dc_context != message._dc_context:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
+        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
+        return Chat(self, chat_id)
+
+    def create_group_chat(self, name, verified=False):
+        """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
-        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        chat = Chat(self, chat_id)
-        if contacts is not None:
-            for contact in contacts:
-                chat.add_contact(contact)
-        return chat
+        return Chat(self, chat_id)
 
-    def get_chats(self) -> List[Chat]:
+    def get_chats(self):
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -374,17 +331,24 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self) -> Chat:
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
+    def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -396,21 +360,19 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
+    def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            if isinstance(msg, Message):
-                arr.append(msg.id)
-            else:
-                arr.append(msg)
+            msg = getattr(msg, "id", msg)
+            arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
+    def forward_messages(self, messages, chat):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -420,7 +382,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages):
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -434,23 +396,25 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account has to be stopped; call stop_io() if necessary.
+        Note that the account does not have to be started.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -460,7 +424,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
+        self._import(path, imex_cmd=2)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -468,28 +432,29 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-
-    def initiate_key_transfer(self) -> str:
+    def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self._threads.is_started():
+            raise RuntimeError("threads not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self) -> str:
+    def get_setup_contact_qr(self):
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -539,9 +504,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -567,10 +530,6 @@ class Account(object):
         self._pm.check_pending()
         return plugin
 
-    def remove_account_plugin(self, plugin, name=None):
-        """ remove an account plugin. """
-        self._pm.unregister(plugin, name=name)
-
     @contextmanager
     def temp_plugin(self, plugin):
         """ run a with-block with the given plugin temporarily registered. """
@@ -582,117 +541,111 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
+    def start(self, callback_thread=True):
+        """ start this account (activate imap/smtp threads etc.)
+        and return immediately.
 
-    def get_connectivity_html(self) -> str:
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
+        If this account is not configured, an internal configuration
+        job will be scheduled if config values are sufficiently specified.
 
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
-    def start_io(self):
-        """ start this account's IO scheduling (Rust-core async scheduler)
-
-        If this account is not configured an Exception is raised.
-        You need to call account.configure() and account.wait_configure_finish()
-        before.
-
-        You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
-        account is started.
-
-        If you are using this from a test, you may want to call
-        wait_all_initial_fetches() afterwards.
+        You may call `wait_shutdown` or `shutdown` after the
+        account is in started mode.
 
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
-        :raises ConfigureFailed: if the account could not be configured.
 
         :returns: None
         """
         if not self.is_configured():
-            raise ValueError("account not configured, cannot start io")
-        lib.dc_start_io(self._dc_context)
+            if not self.get_config("addr") or not self.get_config("mail_pw"):
+                raise MissingCredentials("addr or mail_pwd not set in config")
+            lib.dc_configure(self._dc_context)
+        self._threads.start(callback_thread=callback_thread)
 
-    def maybe_network(self):
-        """This function should be called when there is a hint
-        that the network is available again,
-        e.g. as a response to system event reporting network availability.
-        The library will try to send pending messages out immediately.
-
-        Moreover, to have a reliable state
-        when the app comes to foreground with network available,
-        it may be reasonable to call the function also at that moment.
-
-        It is okay to call the function unconditionally when there is
-        network available, however, calling the function
-        _without_ having network may interfere with the backoff algorithm
-        and will led to let the jobs fail faster, with fewer retries
-        and may avoid messages being sent out.
-
-        Finally, if the context was created by the dc_accounts_t account manager
-        (currently not implemented in the Python bindings),
-        use dc_accounts_maybe_network() instead of this function
-        """
-        lib.dc_maybe_network(self._dc_context)
-
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
-        """ Start configuration process and return a Configtracker instance
-        on which you can block with wait_finish() to get a True/False success
-        value for the configuration process.
-        """
-        assert self.is_configured() == reconfigure
-        if not self.get_config("addr") or not self.get_config("mail_pw"):
-            raise MissingCredentials("addr or mail_pwd not set in config")
-        configtracker = ConfigureTracker(self)
-        self.add_account_plugin(configtracker)
-        lib.dc_configure(self._dc_context)
-        return configtracker
-
-    def wait_shutdown(self) -> None:
+    def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self) -> None:
-        """ stop core IO scheduler if it is running. """
-        self.log("stop_ongoing")
-        self.stop_ongoing()
-
-        self.log("dc_stop_io (stop core IO scheduler)")
-        lib.dc_stop_io(self._dc_context)
-
-    def shutdown(self) -> None:
-        """ shutdown and destroy account (stop callback thread, close and remove
-        underlying dc_context)."""
-        if self._dc_context is None:
+    def shutdown(self, wait=True):
+        """ shutdown account, stop threads and close and remove
+        underlying dc_context and callbacks. """
+        dc_context = self._dc_context
+        if dc_context is None:
             return
 
-        self.stop_io()
-
-        self.log("remove dc_context references")
-
-        # if _dc_context is unref'ed the event thread should quickly
-        # receive the termination signal. However, some python code might
-        # still hold a reference and so we use a secondary signal
-        # to make sure the even thread terminates if it receives any new
-        # event, indepedently from waiting for the core to send NULL to
-        # get_next_event().
-        self._event_thread.mark_shutdown()
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=False)
+        lib.dc_close(dc_context)
+        self._hook_event_queue.put(None)
+        self._threads.stop(wait=wait)  # to wait for threads
         self._dc_context = None
-
-        self.log("wait for event thread to finish")
-        try:
-            self._event_thread.wait(timeout=2)
-        except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
-
-        if self._event_thread.is_alive():
-            self.log("WARN: event thread did not terminate yet, ignoring.")
-
+        atexit.unregister(self.shutdown)
         self._shutdown_event.set()
-
         hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self)
-        self.log("shutdown finished")
+        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
+
+    def _handle_current_events(self):
+        """ handle all currently queued events and then return. """
+        while 1:
+            try:
+                event = self._hook_event_queue.get(block=False)
+            except queue.Empty:
+                break
+            else:
+                event.call_hook()
+
+    def iter_events(self, timeout=None):
+        """ yield hook events until shutdown.
+
+        It is not allowed to call iter_events() from multiple threads.
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
+        while 1:
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
+
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                return "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = self.get_message_by_id(ffi_event.data2)
+                if msg.is_in_fresh():
+                    return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_MEMBER_ADDED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_added", dict(chat=chat, contact=contact)
+        elif name == "DC_EVENT_MEMBER_REMOVED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_removed", dict(chat=chat, contact=contact)
+        return None, {}
+
+
+def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
+    # destructor for dc_context
+    dc_context_unref(dc_context)
+    try:
+        deltachat.clear_context_callback(dc_context)
+    except (TypeError, AttributeError):
+        # we are deep into Python Interpreter shutdown,
+        # so no need to clear the callback context mapping.
+        pass
 
 
 class ScannedQRCode:
@@ -708,3 +661,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -3,13 +3,12 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
-from typing import Optional
 
 
 class Chat(object):
@@ -18,30 +17,29 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
+    def __init__(self, account, id):
         self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
-               self.account._dc_context == other.account._dc_context
+               self._dc_context == getattr(other, "_dc_context", None)
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other):
         return not (self == other)
 
-    def __repr__(self) -> str:
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+    def __repr__(self):
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
 
     @property
     def _dc_chat(self):
         return ffi.gc(
-            lib.dc_get_chat(self.account._dc_context, self.id),
+            lib.dc_get_chat(self._dc_context, self.id),
             lib.dc_chat_unref
         )
 
-    def delete(self) -> None:
+    def delete(self):
         """Delete this chat and all its messages.
 
         Note:
@@ -49,39 +47,34 @@ class Chat(object):
         - does not delete messages on server
         - the chat or contact is not blocked, new message will arrive
         """
-        lib.dc_delete_chat(self.account._dc_context, self.id)
-
-    def block(self) -> None:
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self) -> None:
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
+        lib.dc_delete_chat(self._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self) -> bool:
+    def is_group(self):
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
 
-    def is_muted(self) -> bool:
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
+    def is_muted(self):
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -91,38 +84,30 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self) -> bool:
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+    def is_verified(self):
+        """ return True if this chat is a verified group.
 
-        :returns: True if the chat is writable, False otherwise
+        :returns: True if chat is verified, False otherwise.
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return lib.dc_chat_is_verified(self._dc_chat)
 
-    def is_protected(self) -> bool:
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
-
-    def get_name(self) -> Optional[str]:
+    def get_name(self):
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name: str) -> bool:
+    def set_name(self, name):
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: True on success, False otherwise
+        :returns: None
         """
         name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        return lib.dc_set_chat_name(self._dc_context, self.id, name)
 
-    def mute(self, duration: Optional[int] = None) -> None:
+    def mute(self, duration=None):
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -132,70 +117,47 @@ class Chat(object):
             mute_duration = -1
         else:
             mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self) -> None:
+    def unmute(self):
         """ unmutes the chat
 
         :returns: None
         """
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, 0)
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self) -> int:
+    def get_mute_duration(self):
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
-    def get_ephemeral_timer(self) -> int:
-        """ get ephemeral timer.
-
-        :returns: ephemeral timer value in seconds
-        """
-        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
-
-    def set_ephemeral_timer(self, timer: int) -> bool:
-        """ set ephemeral timer.
-
-        :param: timer value in seconds
-
-        :returns: True on success, False otherwise
-        """
-        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
-
-    def get_type(self) -> int:
+    def get_type(self):
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self) -> Optional[str]:
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
-    def get_join_qr(self) -> Optional[str]:
+    def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
         where account.join_with_qrcode(qr) needs to be called.
         """
-        res = lib.dc_get_securejoin_qr(self.account._dc_context, self.id)
+        res = lib.dc_get_securejoin_qr(self._dc_context, self.id)
         return from_dc_charpointer(res)
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg: Message) -> Message:
+    def send_msg(self, msg):
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -212,7 +174,7 @@ class Chat(object):
             assert msg.id != 0
             # get a fresh copy of dc_msg, the core needs it
             msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         # modify message in place to avoid bad state for the caller
@@ -227,7 +189,7 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         msg = as_dc_charpointer(text)
-        msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
+        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
         if msg_id == 0:
             raise ValueError("message could not be send, does chat exist?")
         return Message.from_db(self.account, msg_id)
@@ -242,7 +204,7 @@ class Chat(object):
         """
         msg = Message.new_empty(self.account, view_type="file")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -257,25 +219,23 @@ class Chat(object):
         mime_type = mimetypes.guess_type(path)[0]
         msg = Message.new_empty(self.account, view_type="image")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """ create a new prepared message.
 
         :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
+        :returns: :class:`deltachat.message.Message` instance.
         """
-        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
+        msg_id = lib.dc_prepare_msg(self._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -306,7 +266,7 @@ class Chat(object):
         msg = Message.from_db(self.account, message.id)
 
         # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, 0, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         assert sent_id == msg.id
@@ -320,9 +280,9 @@ class Chat(object):
         :returns: None
         """
         if message is None:
-            lib.dc_set_draft(self.account._dc_context, self.id, ffi.NULL)
+            lib.dc_set_draft(self._dc_context, self.id, ffi.NULL)
         else:
-            lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
+            lib.dc_set_draft(self._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
         """ get draft message for this chat.
@@ -330,7 +290,7 @@ class Chat(object):
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
         """
-        x = lib.dc_get_draft(self.account._dc_context, self.id)
+        x = lib.dc_get_draft(self._dc_context, self.id)
         if x == ffi.NULL:
             return None
         dc_msg = ffi.gc(x, lib.dc_msg_unref)
@@ -342,7 +302,7 @@ class Chat(object):
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
+            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
             lib.dc_array_unref
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
@@ -352,69 +312,60 @@ class Chat(object):
 
         :returns: number of fresh messages
         """
-        return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
+        return lib.dc_get_fresh_msg_cnt(self._dc_context, self.id)
 
     def mark_noticed(self):
         """ mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
-        return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
+        return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
     def get_summary(self):
         """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
+        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
         s = from_dc_charpointer(dc_res)
         return json.loads(s)
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, obj):
+    def add_contact(self, contact):
         """ add a contact to this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        contact = self.account.create_contact(obj)
-        ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
+        ret = lib.dc_add_contact_to_chat(self._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
-        return contact
 
-    def remove_contact(self, obj):
+    def remove_contact(self, contact):
         """ remove a contact from this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.get_contact(obj)
-        ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
+        ret = lib.dc_remove_contact_from_chat(self._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
+        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
+
         """
         from .contact import Contact
         dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
         )
         return list(iter_array(
             dc_array, lambda id: Contact(self.account, id))
         )
 
-    def num_contacts(self):
-        """ return number of contacts in this chat. """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return lib.dc_array_get_cnt(dc_array)
-
     def set_profile_image(self, img_path):
         """Set group profile image.
 
@@ -427,7 +378,7 @@ class Chat(object):
         """
         assert os.path.exists(img_path), img_path
         p = as_dc_charpointer(img_path)
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, p)
         if res != 1:
             raise ValueError("Setting Profile Image {!r} failed".format(p))
 
@@ -440,7 +391,7 @@ class Chat(object):
         :raises ValueError: if profile image could not be reset
         :returns: None
         """
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, ffi.NULL)
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, ffi.NULL)
         if res != 1:
             raise ValueError("Removing Profile Image failed")
 
@@ -464,13 +415,19 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
         """return True if this chat has location-sending enabled currently.
         :returns: True if location sending is enabled.
         """
-        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
 
     def is_archived(self):
         """return True if this chat is archived.
@@ -483,7 +440,7 @@ class Chat(object):
 
         all subsequent messages will carry a location with them.
         """
-        lib.dc_send_locations_to_chat(self.account._dc_context, self.id, seconds)
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
 
     def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
         """return list of locations for the given contact in the given timespan.
@@ -507,30 +464,24 @@ class Chat(object):
         else:
             contact_id = contact.id
 
-        dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
         return [
             Location(
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
-                ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
-        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -1,13 +1,202 @@
-from typing import Any, List
+import sys
+import re
+import os
+from os.path import dirname, abspath
+from os.path import join as joinpath
 
-from .capi import lib
+# the following const are generated from deltachat.h
+# this works well when you in a git-checkout
+# run "python deltachat/const.py" to regenerate events
+# begin const generated
+DC_GCL_ARCHIVED_ONLY = 0x01
+DC_GCL_NO_SPECIALS = 0x02
+DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_VERIFIED_ONLY = 0x01
+DC_GCL_ADD_SELF = 0x02
+DC_QR_ASK_VERIFYCONTACT = 200
+DC_QR_ASK_VERIFYGROUP = 202
+DC_QR_FPR_OK = 210
+DC_QR_FPR_MISMATCH = 220
+DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
+DC_QR_ADDR = 320
+DC_QR_TEXT = 330
+DC_QR_URL = 332
+DC_QR_ERROR = 400
+DC_CHAT_ID_DEADDROP = 1
+DC_CHAT_ID_TRASH = 3
+DC_CHAT_ID_MSGS_IN_CREATION = 4
+DC_CHAT_ID_STARRED = 5
+DC_CHAT_ID_ARCHIVED_LINK = 6
+DC_CHAT_ID_ALLDONE_HINT = 7
+DC_CHAT_ID_LAST_SPECIAL = 9
+DC_CHAT_TYPE_UNDEFINED = 0
+DC_CHAT_TYPE_SINGLE = 100
+DC_CHAT_TYPE_GROUP = 120
+DC_CHAT_TYPE_VERIFIED_GROUP = 130
+DC_MSG_ID_MARKER1 = 1
+DC_MSG_ID_DAYMARKER = 9
+DC_MSG_ID_LAST_SPECIAL = 9
+DC_STATE_UNDEFINED = 0
+DC_STATE_IN_FRESH = 10
+DC_STATE_IN_NOTICED = 13
+DC_STATE_IN_SEEN = 16
+DC_STATE_OUT_PREPARING = 18
+DC_STATE_OUT_DRAFT = 19
+DC_STATE_OUT_PENDING = 20
+DC_STATE_OUT_FAILED = 24
+DC_STATE_OUT_DELIVERED = 26
+DC_STATE_OUT_MDN_RCVD = 28
+DC_CONTACT_ID_SELF = 1
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
+DC_CONTACT_ID_LAST_SPECIAL = 9
+DC_MSG_TEXT = 10
+DC_MSG_IMAGE = 20
+DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
+DC_MSG_AUDIO = 40
+DC_MSG_VOICE = 41
+DC_MSG_VIDEO = 50
+DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
+DC_EVENT_INFO = 100
+DC_EVENT_SMTP_CONNECTED = 101
+DC_EVENT_IMAP_CONNECTED = 102
+DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
+DC_EVENT_WARNING = 300
+DC_EVENT_ERROR = 400
+DC_EVENT_ERROR_NETWORK = 401
+DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
+DC_EVENT_MSGS_CHANGED = 2000
+DC_EVENT_INCOMING_MSG = 2005
+DC_EVENT_MSG_DELIVERED = 2010
+DC_EVENT_MSG_FAILED = 2012
+DC_EVENT_MSG_READ = 2015
+DC_EVENT_CHAT_MODIFIED = 2020
+DC_EVENT_CONTACTS_CHANGED = 2030
+DC_EVENT_LOCATION_CHANGED = 2035
+DC_EVENT_CONFIGURE_PROGRESS = 2041
+DC_EVENT_IMEX_PROGRESS = 2051
+DC_EVENT_IMEX_FILE_WRITTEN = 2052
+DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
+DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
+DC_EVENT_MEMBER_ADDED = 2063
+DC_EVENT_MEMBER_REMOVED = 2064
+DC_EVENT_FILE_COPIED = 2055
+DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
+# end const generated
 
 
-def __getattr__(name: str) -> Any:
-    if name.startswith("DC_"):
-        return getattr(lib, name)
-    return globals()[name]
+def read_event_defines(f):
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
+    for line in f:
+        m = rex.match(line)
+        if m:
+            yield m.groups()
 
 
-def __dir__() -> List[str]:
-    return sorted(name for name in dir(lib) if name.startswith("DC_"))
+if __name__ == "__main__":
+    here = abspath(__file__).rstrip("oc")
+    here_dir = dirname(here)
+    if len(sys.argv) >= 2:
+        deltah = sys.argv[1]
+    else:
+        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
+    assert os.path.exists(deltah)
+
+    lines = []
+    skip_to_end = False
+    for orig_line in open(here):
+        if skip_to_end:
+            if not orig_line.startswith("# end const"):
+                continue
+            skip_to_end = False
+        lines.append(orig_line)
+        if orig_line.startswith("# begin const"):
+            with open(deltah) as f:
+                for name, item in read_event_defines(f):
+                    lines.append("{} = {}\n".format(name, item))
+            skip_to_end = True
+
+    tmpname = here + ".tmp"
+    with open(tmpname, "w") as f:
+        f.write("".join(lines))
+    os.rename(tmpname, here)

python/src/deltachat/contact.py

@@ -1,12 +1,8 @@
 """ Contact object. """
 
-from datetime import date, datetime, timezone
-from typing import Optional
-
-from . import const, props
-from .capi import ffi, lib
-from .chat import Chat
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
 
 
 class Contact(object):
@@ -15,93 +11,54 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
-        return self.account._dc_context == other.account._dc_context and self.id == other.id
+        return self._dc_context == other._dc_context and self.id == other.id
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
 
     @property
     def _dc_contact(self):
         return ffi.gc(
-            lib.dc_get_contact(self.account._dc_context, self.id),
+            lib.dc_get_contact(self._dc_context, self.id),
             lib.dc_contact_unref
         )
 
     @props.with_doc
-    def addr(self) -> str:
+    def addr(self):
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self) -> str:
+    def display_name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
-    # deprecated alias
-    display_name = name
-
-    @props.with_doc
-    def last_seen(self) -> date:
-        """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
-    def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, block)
-
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self) -> Optional[str]:
+    def get_profile_image(self):
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        return from_optional_dc_charpointer(dc_res)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
 
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
-
-    def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        dc_context = self.account._dc_context
-        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
-        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
-        return Chat(self.account, chat_id)
-
-    # deprecated name
-    get_chat = create_chat
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/cutil.py

@@ -1,9 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
-
-T = TypeVar('T')
+from datetime import datetime
 
 
 def as_dc_charpointer(obj):
@@ -14,28 +11,20 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
+def iter_array(dc_array_t, constructor):
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj) -> str:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    raise ValueError
-
-
-def from_optional_dc_charpointer(obj) -> Optional[str]:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    return None
+def from_dc_charpointer(obj):
+    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:
-    def __init__(self, dc_lot) -> None:
+    def __init__(self, dc_lot):
         self._dc_lot = dc_lot
 
-    def id(self) -> int:
+    def id(self):
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -54,4 +43,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)

python/src/deltachat/direct_imap.py

@@ -1,266 +0,0 @@
-"""
-Internal Python-level IMAP handling used by the testplugin
-and for cleaning up inbox/mvbox for each test function run.
-"""
-
-import io
-import email
-import ssl
-import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
-import imaplib
-import deltachat
-from deltachat import const, Account
-
-
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
-ALL = "1:*"
-
-
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.delete_folder(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
-
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
-class DirectImap:
-    def __init__(self, account: Account) -> None:
-        self.account = account
-        self.logid = account.get_config("displayname") or id(account)
-        self._idling = False
-        self.connect()
-
-    def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
-
-        user = self.account.get_config("addr")
-        pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
-        self.conn.login(user, pw)
-
-        self.select_folder("INBOX")
-
-    def shutdown(self):
-        try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
-            pass
-        try:
-            self.conn.logout()
-        except (OSError, IMAPClientError):
-            print("Could not logout direct_imap conn")
-
-    def create_folder(self, foldername):
-        try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
-    def select_folder(self, foldername):
-        assert not self._idling
-        return self.conn.select_folder(foldername)
-
-    def select_config_folder(self, config_name):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
-        if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
-        foldername = self.account.get_config(config_name)
-        if foldername:
-            return self.select_folder(foldername)
-
-    def list_folders(self):
-        """ return list of all existing folder names"""
-        assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
-
-    def delete(self, range, expunge=True):
-        """ delete a range of messages (imap-syntax).
-        If expunge is true, perform the expunge-operation
-        to make sure the messages are really gone and not
-        just flagged as deleted.
-        """
-        self.conn.set_flags(range, [DELETED])
-        if expunge:
-            self.conn.expunge()
-
-    def get_all_messages(self):
-        assert not self._idling
-
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
-        assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
-
-    def mark_all_read(self):
-        messages = self.get_unread_messages()
-        if messages:
-            res = self.conn.set_flags(messages, [SEEN])
-            print("marked seen:", messages, res)
-
-    def get_unread_cnt(self):
-        return len(self.get_unread_messages())
-
-    def dump_imap_structures(self, dir, logfile):
-        assert not self._idling
-        stream = io.StringIO()
-
-        def log(*args, **kwargs):
-            kwargs["file"] = stream
-            print(*args, **kwargs)
-
-        empty_folders = []
-        for imapfolder in self.list_folders():
-            self.select_folder(imapfolder)
-            messages = list(self.get_all_messages())
-            if not messages:
-                empty_folders.append(imapfolder)
-                continue
-
-            log("---------", imapfolder, len(messages), "messages ---------")
-            # get message content without auto-marking it as seen
-            # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
-                    continue
-
-                flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
-                path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
-
-        if empty_folders:
-            log("--------- EMPTY FOLDERS:", empty_folders)
-
-        print(stream.getvalue(), file=logfile)
-
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False):
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
-        if terminate:
-            self.idle_done()
-        self.account.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
-        """
-        while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle_done()
-            self._idling = False
-            return res
-
-    def append(self, folder, msg):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
-        """
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
-
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]

python/src/deltachat/eventlogger.py

@@ -0,0 +1,137 @@
+import deltachat
+import threading
+import time
+import re
+from queue import Queue, Empty
+from .hookspec import account_hookimpl, global_hookimpl
+
+
+@global_hookimpl
+def dc_account_init(account):
+    # send all FFI events for this account to a plugin hook
+    def _ll_event(ctx, evt_name, data1, data2):
+        assert ctx == account._dc_context
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        account._pm.hook.ac_process_ffi_event(
+            account=account, ffi_event=ffi_event
+        )
+    deltachat.set_context_callback(account._dc_context, _ll_event)
+
+
+@global_hookimpl
+def dc_account_after_shutdown(dc_context):
+    deltachat.clear_context_callback(dc_context)
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
+    _loglock = threading.RLock()
+
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
+        self.account = account
+        self.logid = logid
+        self.init_time = time.time()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._log_event(ffi_event)
+
+    def _log_event(self, ffi_event):
+        # don't show events that are anyway empty impls now
+        if ffi_event.name == "DC_EVENT_GET_STRING":
+            return
+        self.account.ac_log_line(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout if timeout is not None else self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError(str(ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev.name), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev.name):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev.data2):
+                return ev
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
+
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/events.py

@@ -1,280 +0,0 @@
-import threading
-import time
-import re
-import os
-from queue import Queue, Empty
-
-import deltachat
-from .hookspec import account_hookimpl
-from contextlib import contextmanager
-from .capi import ffi, lib
-from .message import map_system_message
-from .cutil import from_optional_dc_charpointer
-
-
-class FFIEvent:
-    def __init__(self, name: str, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account) -> None:
-        self.account = account
-        self.logid = self.account.get_config("displayname")
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
-        self.account.log(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.current_thread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-
-        if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
-            if message.startswith("DC_EVENT_WARNING"):
-                s = WARN + s + ENDC
-            if message.startswith("DC_EVENT_ERROR"):
-                s = ERROR + s + ENDC
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError("unexpected event: {}".format(ev))
-        return ev
-
-    def iter_events(self, timeout=None, check_error=True):
-        while 1:
-            yield self.get(timeout=timeout, check_error=check_error)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        for ev in self.iter_events(timeout=timeout, check_error=check_error):
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_contains(self, regex: str) -> FFIEvent:
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.search(ev.data2):
-                return ev
-
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def wait_securejoin_inviter_progress(self, target):
-        while 1:
-            event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-            if event.data2 >= target:
-                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
-                break
-
-    def wait_all_initial_fetches(self):
-        """Has to be called after start_io() to wait for fetch_existing_msgs to run
-        so that new messages are not mistaken for old ones:
-        - ac1 and ac2 are created
-        - ac1 sends a message to ac2
-        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("Done fetching existing messages")
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)
-        return None
-
-    def wait_msg_delivered(self, msg):
-        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == msg.chat.id
-        assert ev.data2 == msg.id
-        assert msg.is_out_delivered()
-
-
-class EventThread(threading.Thread):
-    """ Event Thread for an account.
-
-    With each Account init this callback thread is started.
-    """
-    def __init__(self, account) -> None:
-        self.account = account
-        super(EventThread, self).__init__(name="events")
-        self.daemon = True
-        self._marked_for_shutdown = False
-        self.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.log(message + " START")
-        yield
-        self.account.log(message + " FINISHED")
-
-    def mark_shutdown(self) -> None:
-        self._marked_for_shutdown = True
-
-    def wait(self, timeout=None) -> None:
-        if self == threading.current_thread():
-            # we are in the callback thread and thus cannot
-            # wait for the thread-loop to finish.
-            return
-        self.join(timeout=timeout)
-
-    def run(self) -> None:
-        """ get and run events until shutdown. """
-        with self.log_execution("EVENT THREAD"):
-            self._inner_run()
-
-    def _inner_run(self):
-        event_emitter = ffi.gc(
-            lib.dc_get_event_emitter(self.account._dc_context),
-            lib.dc_event_emitter_unref,
-        )
-        while not self._marked_for_shutdown:
-            event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL:
-                break
-            if self._marked_for_shutdown:
-                break
-            evt = lib.dc_event_get_id(event)
-            data1 = lib.dc_event_get_data1_int(event)
-            # the following code relates to the deltachat/_build.py's helper
-            # function which provides us signature info of an event call
-            evt_name = deltachat.get_dc_event_name(evt)
-            if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
-            else:
-                data2 = lib.dc_event_get_data2_int(event)
-
-            lib.dc_event_unref(event)
-            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            try:
-                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-                for name, kwargs in self._map_ffi_event(ffi_event):
-                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
-                    hook = getattr(self.account._pm.hook, name)
-                    hook(**kwargs)
-            except Exception:
-                if self.account._dc_context is not None:
-                    raise
-
-    def _map_ffi_event(self, ffi_event: FFIEvent):
-        name = ffi_event.name
-        account = self.account
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = account.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = account.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)

python/src/deltachat/hookspec.py

@@ -15,9 +15,8 @@ global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 class PerAccount:
     """ per-Account-instance hook specifications.
 
-    All hooks are executed in a dedicated Event thread.
-    Hooks are generally not allowed to block/last long as this
-    blocks overall event processing on the python side.
+    Except for ac_process_ffi_event all hooks are executed
+    in the thread which calls Account.wait_shutdown().
     """
     @classmethod
     def _make_plugin_manager(cls):
@@ -31,6 +30,10 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -39,49 +42,23 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
+        """ Called when a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
-
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
-
-        :param message: Message that was just delivered.
-        """
+        """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
-
-        :param chat: Chat which was modified.
-        """
+    def ac_member_added(self, chat, contact):
+        """ Called for each contact added to a chat. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
-
-        :param chat: Chat where contact was added.
-        :param contact: Contact that was added.
-        :param actor: Who added the contact (None if it was our self-addr)
-        :param message: The original system message that reports the addition.
-        """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
-
-        :param chat: Chat where contact was removed.
-        :param contact: Contact that was removed.
-        :param actor: Who removed the contact (None if it was our self-addr)
-        :param message: The original system message that reports the removal.
-        """
+    def ac_member_removed(self, chat, contact):
+        """ Called for each contact removed from a chat. """
 
 
 class Global:
@@ -103,13 +80,5 @@ class Global:
         """ called when `Account::__init__()` function starts executing. """
 
     @global_hookspec
-    def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
-
-        This hook can be used to perform extra work before
-        ac_configure_completed is called.
-        """
-
-    @global_hookspec
-    def dc_account_after_shutdown(self, account):
+    def dc_account_after_shutdown(self, account, dc_context):
         """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -0,0 +1,106 @@
+
+import threading
+import time
+
+from contextlib import contextmanager
+
+from .capi import lib
+
+
+class IOThreads:
+    def __init__(self, account):
+        self.account = account
+        self._dc_context = account._dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+
+    def is_started(self):
+        return len(self._name2thread) > 0
+
+    def start(self, callback_thread):
+        assert not self.is_started()
+        self._start_one_thread("inbox", self.imap_thread_run)
+        self._start_one_thread("smtp", self.smtp_thread_run)
+
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
+        if int(self.account.get_config("mvbox_watch")):
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+
+        if int(self.account.get_config("sentbox_watch")):
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.ac_log_line(message + " START")
+        yield
+        self.account.ac_log_line(message + " FINISHED")
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
+
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if "mvbox" in self._name2thread:
+            lib.dc_interrupt_mvbox_idle(self._dc_context)
+        if "sentbox" in self._name2thread:
+            lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
+                ev.call_hook()
+
+    def imap_thread_run(self):
+        with self.log_execution("INBOX THREAD START"):
+            while not self._thread_quitflag:
+                lib.dc_perform_imap_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_idle(self._dc_context)
+
+    def mvbox_thread_run(self):
+        with self.log_execution("MVBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_mvbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_idle(self._dc_context)
+
+    def sentbox_thread_run(self):
+        with self.log_execution("SENTBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_sentbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_idle(self._dc_context)
+
+    def smtp_thread_run(self):
+        with self.log_execution("SMTP THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_smtp_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -1,13 +1,11 @@
 """ The Message object. """
 
 import os
-import re
 from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import datetime
 
 
 class Message(object):
@@ -18,22 +16,19 @@ class Message(object):
     """
     def __init__(self, account, dc_msg):
         self.account = account
-        assert isinstance(self.account._dc_context, ffi.CData)
+        self._dc_context = account._dc_context
+        assert isinstance(self._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        self.id = lib.dc_msg_get_id(dc_msg)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
-        typ = "outgoing" if self.is_outgoing() else "incoming"
-        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ, self.is_system_message(), repr(self.text[:10]),
-            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
 
     @classmethod
     def from_db(cls, account, id):
@@ -47,36 +42,26 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param: view_type is "text", "audio", "video", "file"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
         ))
 
-    def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
-
-        If the message is a contact request
-        the sender will become an accepted contact.
-
-        :returns: a :class:`deltachat.chat.Chat` object.
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
         """
-        self.chat.accept()
-        return self.chat
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_msg_unref
+        )
 
     @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
-
-    @props.with_doc
-    def text(self) -> str:
+    def text(self):
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -84,23 +69,6 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self) -> str:
-        """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -114,25 +82,20 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self) -> str:
+    def basename(self):
         """basename of the attachment if it exists, otherwise empty string. """
-        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self) -> str:
+    def filemime(self):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return bool(lib.dc_msg_is_info(self._dc_msg))
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self) -> str:
+    def get_setupcodebegin(self):
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -140,25 +103,21 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self) -> str:
+    def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
         """
-        return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
+        return from_dc_charpointer(lib.dc_get_msg_info(self._dc_context, self.id))
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
         res = lib.dc_continue_key_transfer(
-                self.account._dc_context,
+                self._dc_context,
                 self.id,
                 as_dc_charpointer(setup_code)
         )
@@ -172,7 +131,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def time_received(self):
@@ -182,52 +141,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @props.with_doc
-    def ephemeral_timer(self):
-        """Ephemeral timer in seconds
-
-        :returns: timer in seconds or None if there is no timer
-        """
-        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
-        if timer:
-            return timer
-
-    @props.with_doc
-    def ephemeral_timestamp(self):
-        """UTC time when the message will be deleted.
-
-        :returns: naive datetime.datetime() object or None if the timer is not started.
-        """
-        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
-        if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @property
-    def quoted_text(self) -> Optional[str]:
-        """Text inside the quote
-
-        :returns: Quoted text"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
-
-    @property
-    def quote(self):
-        """Quote getter
-
-        :returns: Quoted message, if found in the database"""
-        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
-        if msg:
-            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
-
-    @quote.setter
-    def quote(self, quoted_message):
-        """Quote setter"""
-        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
-
-    def force_plaintext(self) -> None:
-        """Force the message to be sent in plain text."""
-        lib.dc_msg_force_plaintext(self._dc_msg)
+            return datetime.utcfromtimestamp(ts)
 
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
@@ -238,18 +152,13 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
-        mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
+        mime_headers = lib.dc_get_mime_headers(self._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
-    @property
-    def error(self) -> Optional[str]:
-        """Error message"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
@@ -260,20 +169,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self) -> Optional[str]:
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_optional_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -300,7 +195,7 @@ class Message(object):
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
+                lib.dc_get_msg(self._dc_context, self.id),
                 lib.dc_msg_unref
             )
         return lib.dc_msg_get_state(dc_msg)
@@ -329,13 +224,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -386,10 +274,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -410,68 +294,18 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    const.DC_MSG_TEXT: 'text',
+    const.DC_MSG_IMAGE: 'image',
+    const.DC_MSG_GIF: 'gif',
+    const.DC_MSG_AUDIO: 'audio',
+    const.DC_MSG_VIDEO: 'video',
+    const.DC_MSG_FILE: 'file'
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
-    if code is not None:
-        return code
+    for code, value in _view_type_mapping.items():
+        if value == view_type_name:
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if not res:
-            return
-        action, affected, actor = res
-        affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
-        return "ac_member_" + res[0], d
-
-
-def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
-    if m:
-        text = m.group(1)
-    text = text.rstrip(".")
-    return text.strip()
-
-
-def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
-
-    returns a (action, affected, actor) triple """
-
-    # Member Me (x@y) removed by a@b.
-    # Member x@y added by a@b
-    # Member With space (tmp1@x.org) removed by tmp2@x.org.
-    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-    # Group left by some one (tmp1@x.org).
-    # Group left by tmp1@x.org.
-    text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
-    if m:
-        affected, action, actor = m.groups()
-        return action, extract_addr(affected), extract_addr(actor)
-    if text.startswith("group left by "):
-        addr = extract_addr(text[13:])
-        if addr:
-            return "removed", addr, addr
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr) -> None:
+    def __init__(self, account, addr):
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self) -> str:
+    def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self) -> str:
+    def get_before_login_hints(self):
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
-    def status(self) -> int:
+    def status(self):
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -1,26 +1,21 @@
 from __future__ import print_function
 import os
 import sys
-import io
 import subprocess
 import queue
 import threading
 import fnmatch
-import time
-import weakref
-import tempfile
-from typing import List, Dict, Callable
-
 import pytest
 import requests
-
+import time
 from . import Account, const
+from .tracker import ConfigureTracker
 from .capi import lib
-from .events import FFIEventLogger, FFIEventTracker
+from .eventlogger import FFIEventLogger, FFIEventTracker
+from _pytest.monkeypatch import MonkeyPatch
 from _pytest._code import Source
-from deltachat import direct_imap
 
-import deltachat
+import tempfile
 
 
 def pytest_addoption(parser):
@@ -33,87 +28,41 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    parser.addoption(
-        "--strict-tls", action="store_true",
-        help="Never accept invalid TLS certificates for test accounts",
-    )
 
 
 def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
         if cfg:
             config.option.liveconfig = cfg
 
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
 
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            if item.get_closest_marker("ignored"):
-                if not item.config.getvalue("ignored"):
-                    pytest.skip("use --ignored to run this test")
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
 
 
 def pytest_report_header(config, startdir):
     summary = []
 
     t = tempfile.mktemp()
+    m = MonkeyPatch()
     try:
+        m.setattr(sys.stdout, "write", lambda x: len(x))
         ac = Account(t)
         info = ac.get_info()
         ac.shutdown()
     finally:
+        m.undo()
         os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
+    summary.extend(['Deltachat core={} sqlite={}'.format(
          info['deltachat_core_version'],
          info['sqlite_version'],
-         info['journal_mode'],
      )])
 
     cfg = config.option.liveconfig
@@ -127,7 +76,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn) -> None:
+    def __init__(self, fn):
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -138,34 +87,32 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index: int):
+    def get(self, index):
         return self.configlist[index]
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    configlist: List[Dict[str, str]]
-
-    def __init__(self, url: str) -> None:
+    def __init__(self, url):
         self.configlist = []
         self.url = url
 
-    def get(self, index: int):
+    def get(self, index):
         try:
             return self.configlist[index]
         except IndexError:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
             return config
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
@@ -182,7 +129,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self) -> None:
+        def __init__(self):
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -213,51 +160,35 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        _finalizers: List[Callable[[], None]]
-        _accounts: List[Account]
-
-        def __init__(self) -> None:
+        def __init__(self):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
-            self._accounts = []
             self.init_time = time.time()
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
                 fin = self._finalizers.pop()
                 fin()
 
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
-
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
+            ac = Account(path)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
-            ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
+            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
             if not quiet:
-                logger = FFIEventLogger(ac)
-                logger.init_time = self.init_time
-                ac.add_account_plugin(logger)
-            self._accounts.append(ac)
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            self._finalizers.append(ac.shutdown)
             return ac
 
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -292,123 +223,83 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            if pytestconfig.getoption("--strict-tls"):
-                # Enable strict certificate checks for online accounts
-                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
-        def get_online_configuring_account(self, sentbox=False, move=False,
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
                                            pre_generated_key=True, quiet=False, config={}):
             ac, configdict = self.get_online_config(
                 pre_generated_key=pre_generated_key, quiet=quiet)
             configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac._configtracker = ac.configure()
+            ac.start()
             return ac
 
-        def get_one_online_account(self, pre_generated_key=True, move=False):
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, move=move)
-            self.wait_configure_and_start_io([ac1])
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            ac1._configtracker.wait_imap_connected()
+            ac1._configtracker.wait_smtp_connected()
+            ac1._configtracker.wait_finish()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
+            ac1._configtracker.wait_finish()
+            ac2._configtracker.wait_finish()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
-                        for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
-            for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
-            return accounts
-
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
-
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            if hasattr(account, "direct_imap"):
-                # Attach the existing direct_imap. If we did not do this, a new one would be created and
-                # delete existing messages (see dc_account_extra_configure(configure))
-                ac.direct_imap = account.direct_imap
-            ac._configtracker = ac.configure()
+            ac.start()
             return ac
 
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
-            started_accounts = []
-            for acc in accounts:
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
-            for acc in started_accounts:
-                acc._evtracker.wait_all_initial_fetches()
-
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
-
-        def run_bot_process(self, module, ffi=True):
+        def run_bot_process(self, module):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
             args = [
                 sys.executable,
-                "-u",
                 fn,
+                "--show-ffi",
                 "--email", bot_cfg["addr"],
                 "--password", bot_cfg["mail_pw"],
                 bot_ac.db_path,
             ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
             print("$", " ".join(args))
             popen = subprocess.Popen(
                 args=args,
                 stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
+                bufsize=1,                 # line buffering
                 close_fds=True,            # close all FDs other than 0/1/2
                 universal_newlines=True    # give back text
             )
@@ -416,48 +307,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1: Account, ac2: Account):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
+    return am
 
 
 class BotProcess:
-    stdout_queue: queue.Queue
-
-    def __init__(self, popen, bot_cfg) -> None:
+    def __init__(self, popen, bot_cfg):
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -465,10 +321,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.daemon = True
+        t.setDaemon(1)
         t.start()
 
-    def _run_stdout_thread(self) -> None:
+    def _run_stdout_thread(self):
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -476,35 +332,28 @@ class BotProcess:
                     break
                 line = line.strip()
                 self.stdout_queue.put(line)
-                print("bot-stdout: ", line)
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self) -> None:
+    def kill(self):
         self.popen.kill()
 
-    def wait(self, timeout=30) -> None:
+    def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
         patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
         for next_pattern in patterns:
             print("+++FNMATCH:", next_pattern)
-            ignored = []
             while 1:
                 line = self.stdout_queue.get(timeout=15)
                 if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
                     raise IOError("BOT stdout-thread terminated")
                 if fnmatch.fnmatch(line, next_pattern):
                     print("+++MATCHED:", line)
                     break
                 else:
                     print("+++IGN:", line)
-                    ignored.append(line)
 
 
 @pytest.fixture
@@ -515,26 +364,10 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg: str) -> None:
+        def sec(self, msg):
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg: str) -> None:
+        def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
-
-        def indent(self, msg: str) -> None:
-            print("  " + msg)
-
     return Printer()
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    # execute all other hooks to obtain the report object
-    outcome = yield
-    rep = outcome.get_result()
-
-    # set a report attribute for each phase of a call, which can
-    # be "setup", "call", "teardown"
-
-    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl, Global
+from .hookspec import account_hookimpl
 
 
 class ImexFailed(RuntimeError):
@@ -18,17 +18,7 @@ class ImexTracker:
         if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
             self._imex_events.put(ffi_event.data1)
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data2)
-
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
+            self._imex_events.put(ffi_event.data1)
 
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
@@ -50,14 +40,11 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self, account):
-        self.account = account
+    def __init__(self):
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
-        self._progress = Queue()
-        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -66,15 +53,10 @@ class ConfigureTracker:
             self._smtp_finished.set()
         elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
             self._imap_finished.set()
-        elif ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS":
-            self._progress.put(ffi_event.data1)
 
     @account_hookimpl
     def ac_configure_completed(self, success):
-        if success:
-            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
-        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """
@@ -84,12 +66,6 @@ class ConfigureTracker:
         """ wait until smtp is configured. """
         self._imap_finished.wait()
 
-    def wait_progress(self, data1=None):
-        while 1:
-            evdata = self._progress.get()
-            if data1 is None or evdata == data1:
-                break
-
     def wait_finish(self):
         """ wait until configure is completed.
 

python/tests/auditwheels.py

@@ -1,17 +1,13 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(
-                ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])

python/tests/conftest.py

@@ -0,0 +1,18 @@
+from __future__ import print_function
+
+
+def wait_configuration_progress(account, min_target, max_target=1001):
+    min_target = min(min_target, max_target)
+    while 1:
+        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if event.data1 >= min_target and event.data1 <= max_target:
+            print("** CONFIG PROGRESS {}".format(min_target), account)
+            break
+
+
+def wait_securejoin_inviter_progress(account, target):
+    while 1:
+        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if event.data2 >= target:
+            print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
+            break

python/tests/stress_test_db.py

@@ -1,135 +0,0 @@
-
-import time
-import threading
-import pytest
-import os
-from queue import Queue, Empty
-
-import deltachat
-
-
-def test_db_busy_error(acfactory, tmpdir):
-    starttime = time.time()
-    log_lock = threading.RLock()
-
-    def log(string):
-        with log_lock:
-            print("%3.2f %s" % (time.time() - starttime, string))
-
-    # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3, quiet=True)
-    log("created %s accounts" % len(accounts))
-
-    # put a bigfile into each account
-    for acc in accounts:
-        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
-        with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
-    log("created %s bigfiles" % len(accounts))
-
-    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
-    chat = accounts[0].create_group_chat("stress-group")
-    for addr in contact_addrs[1:]:
-        chat.add_contact(chat.account.create_contact(addr))
-
-    # setup auto-responder bots which report back failures/actions
-    report_queue = Queue()
-
-    def report_func(replier, report_type, *report_args):
-        report_queue.put((replier, report_type, report_args))
-
-    # each replier receives all events and sends report events to receive_queue
-    repliers = []
-    for acc in accounts:
-        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
-        acc.add_account_plugin(replier)
-        repliers.append(replier)
-
-    # kick off message sending
-    # after which repliers will reply to each other
-    chat.send_text("hello")
-
-    alive_count = len(accounts)
-    while alive_count > 0:
-        try:
-            replier, report_type, report_args = report_queue.get(timeout=10)
-        except Empty:
-            log("timeout waiting for next event")
-            pytest.fail("timeout exceeded")
-        if report_type == ReportType.exit:
-            replier.log("EXIT")
-        elif report_type == ReportType.ffi_error:
-            replier.log("ERROR: {}".format(report_args[0]))
-        elif report_type == ReportType.message_echo:
-            continue
-        else:
-            raise ValueError("{} unknown report type {}, args={}".format(
-                addr, report_type, report_args
-            ))
-        alive_count -= 1
-        replier.log("shutting down")
-        replier.account.shutdown()
-        replier.log("shut down complete, remaining={}".format(alive_count))
-
-
-class ReportType:
-    exit = "exit"
-    ffi_error = "ffi-error"
-    message_echo = "message-echo"
-
-
-class AutoReplier:
-    def __init__(self, account, log, num_send, num_bigfiles, report_func):
-        self.account = account
-        self._log = log
-        self.report_func = report_func
-        self.num_send = num_send
-        self.num_bigfiles = num_bigfiles
-        self.current_sent = 0
-        self.addr = self.account.get_self_contact().addr
-
-        self._thread = threading.Thread(
-            name="Stats{}".format(self.account),
-            target=self.thread_stats
-        )
-        self._thread.setDaemon(True)
-        self._thread.start()
-
-    def log(self, message):
-        self._log("{} {}".format(self.addr, message))
-
-    def thread_stats(self):
-        # XXX later use, for now we just quit
-        return
-        while 1:
-            time.sleep(1.0)
-            break
-
-    @deltachat.account_hookimpl
-    def ac_incoming_message(self, message):
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-        message.create_chat()
-        message.mark_seen()
-        self.log("incoming message: {}".format(message))
-
-        self.current_sent += 1
-        # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
-            message.chat.send_text("send big file as reply to: {}".format(message.text))
-            msg = message.chat.send_file(self.account.bigfile)
-        else:
-            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
-            assert msg.text
-        self.log("message-sent: {}".format(msg))
-        self.report_func(self, ReportType.message_echo)
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-
-    @deltachat.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self.log(ffi_event)
-        if ffi_event.name == "DC_EVENT_ERROR":
-            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -6,35 +6,27 @@ import shutil
 import pytest
 from filecmp import cmp
 
-
-def wait_msg_delivered(account, msg_list):
-    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
-    msg_list = list(msg_list)
-    while msg_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        msg_list.remove((ev.data1, ev.data2))
+from conftest import wait_configuration_progress
+from deltachat import const
 
 
-def wait_msgs_changed(account, msgs_list):
-    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
-    account.log("waiting for msgs_list={}".format(msgs_list))
-    msgs_list = list(msgs_list)
-    while msgs_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        for i, (data1, data2) in enumerate(msgs_list):
-            if ev.data1 == data1:
-                if data2 is None or ev.data2 == data2:
-                    del msgs_list[i]
-                    break
-        else:
-            account.log("waiting mismatch data1={} data2={}".format(data1, data2))
-    return ev.data1, ev.data2
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev.data1 == chat_id
+    if msg_id is not None:
+        assert ev.data2 == msg_id
+    return ev.data2
 
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -43,8 +35,13 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -56,10 +53,15 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
 
-        chat = ac1.create_chat(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
@@ -68,16 +70,19 @@ class TestOnlineInCreation:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
+        chat2.add_contact(c2)
+        wait_msgs_changed(ac1, 0, 0)  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them
         # is the one caused by forwarding
-        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        forwarded_id = wait_msgs_changed(ac1, chat2.id)
+        if forwarded_id == 0:
+            forwarded_id = wait_msgs_changed(ac1, chat2.id)
+            assert forwarded_id
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 
@@ -86,24 +91,30 @@ class TestOnlineInCreation:
         shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
-        lp.sec("check that both forwarded and original message are proper.")
-        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
+        lp.sec("expect the forwarded message to be sent now too")
+        wait_msgs_changed(ac1, chat2.id, forwarded_id)
         fwd_msg = ac1.get_message_by_id(forwarded_id)
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
-        lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [
-            (chat2.id, forwarded_id),
-            (chat.id, prepared_original.id)
-        ])
+        lp.sec("wait for the messages to be delivered to SMTP")
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat.id
+        assert ev.data2 == prepared_original.id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat2.id
+        assert ev.data2 == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,52 +1,75 @@
 from __future__ import print_function
-
-from queue import Queue
-from deltachat import capi, cutil, const
+from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
 from deltachat import register_global_plugin
 from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
-# from deltachat.account import EventLogger
 
 
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_context_unref(ctx)
+    capi.lib.dc_close(ctx)
+
+
+def test_callback_None2int():
+    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    set_context_callback(ctx, lambda *args: None)
+    capi.lib.dc_close(ctx)
+    clear_context_callback(ctx)
 
 
 def test_dc_close_events(tmpdir, acfactory):
     ac1 = acfactory.get_unconfigured_account()
 
     # register after_shutdown function
-    shutdowns = Queue()
+    shutdowns = []
 
     class ShutdownPlugin:
         @global_hookimpl
         def dc_account_after_shutdown(self, account):
             assert account._dc_context is None
-            shutdowns.put(account)
+            shutdowns.append(account)
     register_global_plugin(ShutdownPlugin())
     assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
-    shutdowns.get(timeout=2)
+    assert shutdowns == [ac1]
+
+    def find(info_string):
+        evlog = ac1._evtracker
+        while 1:
+            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+            data2 = ev.data2
+            if info_string in data2:
+                return
+            else:
+                print("skipping event", ev)
+
+    find("disconnecting inbox-thread")
+    find("disconnecting sentbox-thread")
+    find("disconnecting mvbox-thread")
+    find("disconnecting SMTP")
+    find("Database closed")
 
 
 def test_wrong_db(tmpdir):
+    dc_context = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
     p = tmpdir.join("hello.db")
     # write an invalid database file
     p.write("x123" * 10)
-
-    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):
-    db_fname = tmpdir.join("hello.db")
     # Apparently some client code expects this to be the same as passing NULL.
     ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
-    assert ctx != ffi.NULL
+    db_fname = tmpdir.join("hello.db")
+    assert lib.dc_open(ctx, db_fname.strpath.encode("ascii"), b"")
 
 
 def test_event_defines():
@@ -55,22 +78,19 @@ def test_event_defines():
 
 
 def test_sig():
-    sig = capi.lib.dc_event_has_string_data
-    assert not sig(const.DC_EVENT_MSGS_CHANGED)
-    assert sig(const.DC_EVENT_INFO)
-    assert sig(const.DC_EVENT_WARNING)
-    assert sig(const.DC_EVENT_ERROR)
-    assert sig(const.DC_EVENT_SMTP_CONNECTED)
-    assert sig(const.DC_EVENT_IMAP_CONNECTED)
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
-    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
+    sig = capi.lib.dc_get_event_signature_types
+    assert sig(const.DC_EVENT_INFO) == 2
+    assert sig(const.DC_EVENT_WARNING) == 2
+    assert sig(const.DC_EVENT_ERROR) == 2
+    assert sig(const.DC_EVENT_SMTP_CONNECTED) == 2
+    assert sig(const.DC_EVENT_IMAP_CONNECTED) == 2
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT) == 2
 
 
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
-
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
+    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
+    chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
@@ -87,18 +107,47 @@ def test_get_special_message_id_returns_empty_message(acfactory):
 
 def test_provider_info_none():
     ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
-def test_get_info_open(tmpdir):
-    db_fname = tmpdir.join("test.db")
+def test_get_info_closed():
     ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
     assert 'deltachat_core_version' in info
+    assert 'database_dir' not in info
+
+
+def test_get_info_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
     assert 'database_dir' in info
+
+
+def test_is_open_closed():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_is_open(ctx) == 0
+
+
+def test_is_open_actually_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    assert lib.dc_is_open(ctx) == 1

python/tox.ini

@@ -1,19 +1,19 @@
 [tox]
-isolated_build = true
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py37
     lint
-    mypy
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
+    DCC_PY_LIVECONFIG
     DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
@@ -44,15 +44,6 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
-[testenv:mypy]
-deps =
-    mypy
-    typing
-    types-setuptools
-    types-requests
-commands =
-    mypy --no-incremental src/
-
 [testenv:doc]
 changedir=doc
 deps =
@@ -80,8 +71,6 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
-markers = 
-    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   scripts/run-integration-tests.sh -e py35
+#   ./run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   scripts/run-integration-tests.sh -e py35 -- -x
+#   ./run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,6 +23,9 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
+    export DCC_PY_LIVECONFIG=liveconfig
+fi
 tox "$@"
 ret=$?
 popd