more type fixes

.circleci/config.yml

@@ -0,0 +1,230 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
+jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: pyenv versions
+      - run: pyenv global 3.5.2
+      - run: ls -laR workspace
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy 
+
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      # - cargo_fetch
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
+
+      - build_doxygen:
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      # Linux Desktop 64bit
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -14,11 +14,11 @@ jobs:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.43.1
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -29,62 +29,25 @@ jobs:
   run_clippy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.43.1
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
 
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.54.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.54.0
-            python: false # Python bindings compilation on Windows is not supported.
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
 
-          # Minimum Supported Rust Version = 1.51.0
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.51.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -114,40 +77,13 @@ jobs:
 
     - name: check
       uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
       with:
-        command: check
-        args: --all --bins --examples --tests --features repl
+        command:  check
+        args: --workspace --all --bins --examples --tests
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3
+        args: --workspace
+          

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -26,6 +25,3 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
-
-coverage/
-.DS_Store

ASYNC-API-TODO.txt

@@ -0,0 +1,54 @@
+
+Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
+
+- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
+
+APIs:
+    dc_context_new                 # opens the database
+    dc_open                             # FFI only
+      -> drop it and move parameters to dc_context_new()
+
+    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
+    dc_imex NEVER goes through the job system
+    dc_imex import_backup needs to ensure dc_stop_jobs()
+
+    dc_start_io                # start smtp/imap and job handling subsystems
+    dc_stop_io                 # stop smtp/imap and job handling subsystems
+    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
+
+    dc_close                            # FFI only
+      -> can be dropped
+    dc_context_unref
+
+    for ios share-extension:
+    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
+        0: message was sent
+        1: message failed to go out, is queued as a job to be retried later
+        2: message permanently failed?
+
+EVENT handling:
+    start a callback thread and call get_next_event() which is BLOCKING
+    it's fine to start this callback thread later, it will see all events.
+    Note that the core infinitely fills the internal queue if you never drain it.
+
+    FFI-get_next_event() returns NULL if the context is unrefed already?
+
+   sidenote: how python's callback thread does it currently:
+         CB-thread runs this while loop:
+         while not QUITFLAG:
+             ev = context.get_next_event( )
+             ...
+         So in order to shutdown properly one has to set QUITFLAG
+         before calling dc_stop_jobs() and dc_context_unref
+
+    event API:
+        get_data1_int
+        get_data2_int
+        get_data3_str
+
+
+- userdata likely only used for the callbacks, likely can be dropped, needs verification
+
+
+- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
+

CHANGELOG.md

@@ -1,792 +1,4 @@
-# Changelog
-
-## 1.67.0
-
-### API changes
-- `dc_get_securejoin_qr_svg(chat_id)` added #2815
-- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
-
-
-## 1.66.0
-
-### API changes
-- `dc_contact_get_last_seen()` added #2823
-- python: `Contact.last_seen` added #2823
-- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
-
-### Changes
-- python: add cutil.from_optional_dc_charpointer() #2824
-- refactorings #2807 #2822 #2825
-
-
-## 1.65.0
-
-### Changes
-- python: add mypy support and some type hints #2809
-
-### Fixes
-- do not disable ephemeral timer when downloading a message partially #2811
-- apply existing ephemeral timer also to partially downloaded messages;
-  after full download, the ephemeral timer starts over #2811
-- replace user-visible error on verification failure with warning;
-  the error is logged to the corresponding chat anyway #2808
-
-
-## 1.64.0
-
-### Fixes
-- add 'waiting for being added to the group' only for group-joins,
-  not for setup-contact #2797
-- prioritize In-Reply-To: and References: headers over group IDs when assigning
-  messages to chats to fix incorrect assignment of Delta Chat replies to
-  classic email threads #2795
-
-
-## 1.63.0
-
-### API changes
-- `dc_get_last_error()` added #2788
-
-### Changes
-- Optimize Autocrypt gossip #2743
-
-### Fixes
-- fix permanently hiding of one-to-one chats after secure-join #2791
-
-
-## 1.62.0
-
-### API Changes
-- `dc_join_securejoin()` now always returns immediately;
-  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
-  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
-- introduce multi-device-sync-messages;
-  as older cores display them as files in self-chat,
-  they are currently only sent if config option `send_sync_msgs` is set #2669
-- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
-
-### Changes
-- use system DNS instead of google for MX queries #2780
-- improve error logging #2758
-- improve tests #2764 #2781
-- improve ci #2770
-- refactorings #2677 #2728 #2740 #2729 #2766 #2778
-
-### Fixes
-- add Let's Encrypt certificate to core as it may be missing older devices #2752
-- prioritize certificate setting from user over the one from provider-db #2749
-- fix "QR process failed" error #2725
-- do not update quota in endless loop #2726
-
-
-## 1.61.0
-
-### API Changes
-- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
-  and `download_limit` config option #2631 #2696
-- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
-- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
-  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
-- emit warnings and errors from account manager with account-id 0 #2712
-
-### Changes
-- notify about incoming contact requests #2690
-- messages are marked as read on first read receipt #2699
-- quota warning reappears after import, rewarning at 95% #2702
-- lock strict TLS if certificate checks are automatic #2711
-- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
-- `Accounts` is not cloneable anymore #2654 #2658
-- update chat/contact data only when there was no newer update #2642
-- better detection of mailing list names #2665 #2685
-- log all decisions when applying ephemeral timer to chats #2679
-- connectivity view now translatable #2694 #2723
-- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
-- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
-- update provider database #2618
-
-### Fixes
-- ephemeral timer rollback protection #2693 #2709
-- recreate configured folders if they are deleted #2691
-- ignore MDNs sent to self #2674
-- recognize NDNs that put headers into "message/global-headers" part #2598
-- avoid `dc_get_contacts()` returning duplicate contact ids #2591
-- do not leak group names on forwarding messages #2719
-- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
-- fix pkg-config file #2660
-- fix "QR process failed" error #2725
-
-
-## 1.60.0
-
-### Added
-- add device message to warn about QUOTA #2621
-- add SOCKS5 support #2474 #2620
-
-### Changes
-- don't emit multiple events with the same import/export progress number #2639
-- reduce message length limit to 5000 chars #2615
-
-### Fixes
-- keep event emitter from closing when there are no accounts #2636
-
-
-## 1.59.0
-
-### Added
-- add quota information to `dc_get_connectivity_html()`
-
-### Changes
-- refactorings #2592 #2570 #2581
-- add 'device chat about' to now existing status #2613
-- update provider database #2608
-
-### Fixes
-- provider database supports socket=PLAIN and dotless domains now #2604 #2608
-- add migrated accounts to events emitter #2607
-- fix forwarding quote-only mails #2600
-- do not set WantsMdn param for outgoing messages #2603
-- set timestamps for system messages #2593
-- do not treat gmail labels as folders #2587
-- avoid timing problems in `dc_maybe_network_lost()` #2551
-- only set smtp to "connected" if the last message was actually sent #2541
-
-
-## 1.58.0
-
-### Fixes
-- move WAL file together with database
-  and avoid using data if the database was not closed correctly before #2583
-
-
-## 1.57.0
-
-### API Changes
-
-- breaking change: removed deaddrop chat #2514 #2563
-
-  Contact request chats are not merged into a single virtual
-  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
-  same way as other chats, but sending of messages to them is not
-  allowed and MDNs are not sent automatically until the chat is
-  "accepted" by the user.
-
-  New API:
-  - `dc_chat_is_contact_request()`: returns true if chat is a contact
-    request.  In this case an option to accept the chat via
-    `dc_accept_chat()` should be shown in the UI.
-  - `dc_accept_chat()`: unblock the chat or accept contact request
-  - `dc_block_chat()`: block the chat, currently works only for mailing
-    lists.
-
-  Removed API:
-  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_decide_on_contact_request()`: this call requires a message ID
-    from deaddrop chat as input. As deaddrop chat is removed, this
-    call can't be used anymore.
-  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
-    only difference between these calls was in handling of deaddrop
-    chat
-  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
-
-- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
-  Instead, there is a new api `dc_get_connectivity()`
-  and `dc_get_connectivity_html()`;
-  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
-
-- breaking change: removed `dc_accounts_import_account()`
-  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
-  on its context
-
-- update account api, 2 new methods:
-  `int dc_all_work_done (dc_context_t* context);`
-  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
-
-- add api to check if a message was `Auto-Submitted`
-  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
-  python: `Message.is_bot()`
-
-- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
-  now returns `NULL` if there is no selected account
-
-- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
-  connectivity loss on its own (eg. iOS) #2550
-
-### Added
-- use Auto-Submitted: auto-generated header to identify bots #2502
-- allow sending stickers via repl tool
-- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
-- withdraw/revive own qr-codes #2512
-- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
-
-### Changes
-- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
-- update documentation #2548 #2561 #2569
-- breaking: `Accounts::create` does not also create an default account anymore #2500
-- remove "forwarded" from stickers, as the primary way of getting stickers
-  is by asking a bot and then forwarding them currently #2526
-- mimeparser: use mailparse to parse RFC 2231 filenames #2543
-- allow email addresses without dot in the domain part #2112
-- allow installing lib and include under different prefixes #2558
-- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
-- improve tests #2487 #2491 #2497
-- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
-- improve ci #2494
-- update provider-database #2565
-
-### Removed
-- remove `dc_accounts_import_account()` api #2521
-- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
-
-### Fixes
-- allow stickers with gif-images #2481
-- fix database migration #2486
-- do not count hidden messages in get_msg_cnt(). #2493
-- improve drafts detection #2489
-- fix panic when removing last, selected account from account manager #2500
-- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
-- avoid hiding outgoing classic emails #2505
-- fixes for message timestamps #2517
-- do not process names, avatars, location XMLs, message signature etc.
-  for duplicate messages #2513
-- fix `can_send` for users not in group #2479
-- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
-- fix which chats messages are assigned to #2465
-- fix: don't create chats when MDNs are received #2578
-
-
-## 1.56.0
-
-- fix downscaling images #2469
-
-- fix outgoing messages popping up in selfchat #2456
-
-- securejoin: display error reason if there is any #2470
-
-- do not allow deleting contacts with ongoing chats #2458
-
-- fix: ignore drafts folder when scanning #2454
-
-- fix: scan folders also when inbox is not watched #2446
-
-- more robust In-Reply-To parsing #2182
-
-- update dependencies #2441 #2438 #2439 #2440 #2447 #2448 #2449 #2452 #2453 #2460 #2464 #2466
-
-- update provider-database #2471
-
-- refactorings #2459 #2457
-
-- improve tests and ci #2445 #2450 #2451
-
-
-## 1.55.0
-
-- fix panic when receiving some HTML messages #2434
-
-- fix downloading some messages multiple times #2430
-
-- fix formatting of read receipt texts #2431
-
-- simplify SQL error handling #2415
-
-- explicit rust API for creating chats with blocked status #2282
-
-- debloat the binary by using less AsRef arguments #2425
-
-
-## 1.54.0
-
-- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
-
-- global search performance improvement #2364 #2365 #2366
-
-- improve SQLite performance with `PRAGMA synchronous=normal` #2382
-
-- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
-
-- python: list `requests` as a requirement #2390
-
-- fix creation of many delete jobs when being offline #2372
-
-- synchronize status between devices #2386
-
-- deaddrop (contact requests) chat improvements #2373
-
-- add "Forwarded:" to notification and chatlist summaries #2310
-
-- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
-
-- improve tests #2360 #2362 #2370 #2377 #2387
-
-- cleanup #2359 #2361 #2374 #2376 #2379 #2388
-
-
-## 1.53.0
-
-- fix sqlx performance regression #2355 2356
-
-- add a `ci_scripts/coverage.sh` #2333 #2334
-
-- refactorings and tests #2348 #2349 #2350
-
-- improve python bindings #2332 #2326
-
-
-## 1.52.0
-
-- database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340
-
-- add alias support: UIs should check for `dc_msg_get_override_sender_name()`
-  also in single-chats now and display divergent names and avatars #2297
-
-- parse blockquote-tags for better quote detection #2313
-
-- ignore unknown classical emails from spam folder #2311
-
-- support "Mixed Up” encryption repairing #2321
-
-- fix single chat search #2344
-
-- fix nightly clippy and rustc errors #2341
-
-- update dependencies #2350
-
-- improve ci #2342
-
-- improve python bindings #2332 #2326
-
-
-## 1.51.0
-
-- breaking change: You have to call `dc_stop_io()`/`dc_start_io()`
-  before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`:
-  fix race condition and db corruption
-  when a message was received during backup #2253
-
-- save subject for messages: new api `dc_msg_get_subject()`,
-  when quoting, use the subject of the quoted message as the new subject,
-  instead of the last subject in the chat #2274 #2283
-
-- new apis to get full or html message,
-  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151 #2264 #2279
-
-- new chat type and apis for the new mailing list support,
-  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
-  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
-  #2241 #2243 #2258 #2259 #2261 #2267 #2270 #2272 #2290
-
-- new api `dc_decide_on_contact_request()`,
-  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
-
-- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
-
-- new api `dc_get_chat_encrinfo()` #2186
-
-- new api `dc_contact_get_status()`, returning the recent footer #2218 #2307
-
-- improve contact name update rules,
-  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
-
-- new api for bots: `dc_msg_set_html()` #2153
-
-- new api for bots: `dc_msg_set_override_sender_name()` #2231
-
-- api removed: `dc_is_io_running()` #2139
-
-- api removed: `dc_contact_get_first_name()` #2165 #2171
-
-- improve compatibility with providers changing the Message-ID
-  (as Outlook.com) #2250 #2265
-
-- correctly show emails that were sent to an alias and then bounced 
-
-- implement Consistent Color Generation (XEP-0392),
-  that results in contact colors be be changed #2228 #2229 #2239
-
-- fetch recent existing messages
-  and create corresponding chats after configure #2106
-
-- improve e-mail compatibility
-  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
-
-- better support videochat-services not supporting random rooms #2191
-
-- export backups as .tar files #2023
-
-- scale avatars based on media_quality, fix avatar rotation #2063
-
-- compare ephemeral timer to parent message to deal with reordering better #2100
-
-- better ephemeral system messages #2183
-
-- read quotes out of html messages #2104
-
-- prepend subject to messages with attachments, if needed #2111
-
-- run housekeeping at least once a day #2114
-
-- resolve MX domain only once per OAuth2 provider #2122
-
-- configure provider based on MX record #2123 #2134
-
-- make transient bad destination address error permanent
-  after n tries #2126 #2202
-
-- enable strict TLS for known providers by default #2121
-
-- improve and harden secure join #2154 #2161 #2251
-
-- update `dc_get_info()` to return more information #2156
-
-- prefer In-Reply-To/References
-  over group-id stored in Message-ID #2164 #2172 #2173
-
-- apply gossiped encryption preference to new peerstates #2174
-
-- fix: do not return quoted messages from the trash chat #2221
-
-- fix: allow emojis for location markers #2177
-
-- fix encoding of Chat-Group-Name-Changed messages that could even lead to
-  messages not being delivered #2141
-
-- fix error when no temporary directory is available #1929
-
-- fix marking read receipts as seen #2117
-
-- fix read-notification for mixed-case addresses #2103
-
-- fix decoding of attachment filenames #2080 #2094 #2102
-
-- fix downloading ranges of message #2061
-
-- fix parsing quoted encoded words in From: header #2193 #2204
-
-- fix import/export race condition #2250
-
-- fix: exclude muted chats from notified-list #2269 #2275
-
-- fix: update uid_next if the server rewind it #2288
-
-- fix: return error on fingerprint mismatch on qr-scan #2295
-
-- fix ci #2217 #2226 #2244 #2245 #2249 #2277 #2286
-
-- try harder on backup opening #2148
-
-- trash messages more thoroughly #2273
-
-- nicer logging #2284
-
-- add CMakeLists.txt #2260
-
-- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
-
-- improve python bindings #2113 #2115 #2133 #2214
-
-- improve documentation #2143 #2160 #2175 #2146
-
-- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
-  #2200 #2230 #2262 #2203
-
-- update provider-database #2299
-
-
-## 1.50.0
-
-- do not fetch emails in between inbox_watch disabled and enabled again #2087
-
-- fix: do not fetch from INBOX if inbox_watch is disabled #2085
-
-- fix: do not use STARTTLS when PLAIN connection is requested
-  and do not allow downgrade if STARTTLS is not available #2071
-
-
-## 1.49.0
-
-- add timestamps to image and video filenames #2068
-
-- forbid quoting messages from another context #2069
-
-- fix: preserve quotes in messages with attachments #2070
-
-
-## 1.48.0
-
-- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
-  #2035 #2042
-
-- skip fetch existing messages/contacts if config-option `bot` set #2017
-
-- always log why a message is sorted to trash #2045
-
-- display a quote if top posting is detected #2047
-
-- add ephemeral task cancellation to `dc_stop_io()`;
-  before, there was no way to quickly terminate pending ephemeral tasks #2051
-
-- when saved-messages chat is deleted,
-  a device-message about recreation is added #2050
-
-- use `max_smtp_rcpt_to` from provider-db,
-  sending messages to many recipients in configurable chunks #2056
-
-- fix handling of empty autoconfigure files #2027
-
-- fix adding saved messages to wrong chats on multi-device #2034 #2039
-
-- fix hang on android4.4 and other systems
-  by adding a workaround to executer-blocking-handling bug #2040
-
-- fix secret key export/import roundtrip #2048
-
-- fix mistakenly unarchived chats #2057
-
-- fix outdated-reminder test that fails only 7 days a year,
-  including halloween :) #2059
-
-- improve python bindings #2021 #2036 #2038
-
-- update provider-database #2037
-
-
-## 1.47.0
-
-- breaking change: `dc_update_device_chats()` removed;
-  this is now done automatically during configure
-  unless the new config-option `bot` is set #1957
-
-- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
-  and remove `dc_marknoticed_all_chats()` #1942 #1981
-
-- breaking change: remove unused starring options #1965
-
-- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
-  `dc_chat_is_protected()`; also single-chats may be protected now, this may
-  happen over the wire even if the UI do not offer an option for that #1968
-
-- breaking change: split quotes off message text,
-  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
-
-- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
-  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
-
-- require quorum to enable encryption #1946
-
-- speed up and clean up account creation #1912 #1927 #1960 #1961
-
-- configure now collects recent contacts and fetches last messages
-  unless disabled by `fetch_existing` config-option #1913 #2003
-  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
-
-- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
-  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
-
-- add `dc_set_chat_protection()`; the `protect` parameter in
-  `dc_create_group_chat()` will be removed in an upcoming release;
-  up to then, UIs using the "verified group" paradigm
-  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
-
-- remove unneeded `DC_STR_COUNT` #1991
-
-- mark all failed messages as failed when receiving an NDN #1993
-
-- check some easy cases for bad system clock and outdated app #1901
-
-- fix import temporary directory usage #1929
-
-- fix forcing encryption for reset peers #1998
-
-- fix: do not allow to save drafts in non-writeable chats #1997
-
-- fix: do not show HTML if there is no content and there is an attachment #1988
-
-- fix recovering offline/lost connections, fixes background receive bug #1983
-
-- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
-
-- fix whitespace for summaries #1938
-
-- fix: improve sentbox name guessing #1941
-
-- fix: avoid manual poll impl for accounts events #1944
-
-- fix encoding newlines in param as a preparation for storing quotes #1945
-
-- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
-
-- fix ci #1928 #1931 #1932 #1933 #1934 #1943
-
-- update provider-database #1940 #2005 #2006
-
-- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
-
-
-## 1.46.0
-
-- breaking change: `dc_configure()` report errors in
-  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
-  #1886 #1905
-
-- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
-  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
-
-- parse multiple servers in Mozilla autoconfig #1860
-
-- try multiple servers for each protocol #1871
-
-- do IMAP and SMTP configuration in parallel #1891
-
-- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
-
-- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
-
-- do not reset peerstate on encrypted messages,
-  ignore reordered autocrypt headers #1885 #1890
-
-- always sort message replies after parent message #1852
-
-- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
-
-- improve mimetype guessing for PDF and many other formats #1857 #1861
-
-- improve accepting invalid html #1851
-
-- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
-
-- tweak HELO command #1908
-
-- make `dc_accounts_get_all()` return accounts sorted #1909
-
-- fix KML coordinates precision used for location streaming #1872
-
-- fix cancelling import/export #1855
-
-
-## 1.45.0
-
-- add `dc_accounts_t` account manager object and related api functions #1784
-
-- add capability to import backups as .tar files,
-  which will become the default in a subsequent release #1749
-
-- try various server domains on configuration #1780 #1838
-
-- recognize .tgs files as stickers #1826
-
-- remove X-Mailer debug header #1819
-
-- improve guessing message types from extension #1818
-
-- fix showing unprotected subjects in encrypted messages #1822
-
-- fix threading in interaction with non-delta-clients #1843
-
-- fix handling if encryption degrades #1829
-
-- fix webrtc-servers names set by the user #1831
-
-- update provider database #1828
-
-- update async-imap to fix Oauth2 #1837
-
-- optimize jpeg assets with trimage #1840
-
-- add tests and documentations #1809 #1820
-
-
-## 1.44.0
-
-- fix peerstate issues #1800 #1805
-
-- fix a crash related to muted chats #1803
-
-- fix incorrect dimensions sometimes reported for images #1806
-
-- fixed `dc_chat_get_remaining_mute_duration` function #1807
-
-- handle empty tags (e.g. `<br/>`) in HTML mails #1810
-
-- always translate the message about disappearing messages timer change #1813
-
-- improve footer detection in plain text email #1812
-
-- update device chat icon to fix warnings in iOS logs #1802
-
-- fix deletion of multiple messages #1795
-
-
-## 1.43.0
-
-- improve using own jitsi-servers #1785
-
-- fix smtp-timeout tweaks for larger mails #1797
-
-- more bug fixes and updates #1794 #1792 #1789 #1787
-
-
-## 1.42.0
-
-- new qr-code type `DC_QR_WEBRTC` #1779
-
-- new `dc_chatlist_get_summary2()` api #1771
-
-- tweak smtp-timeout for larger mails #1782
-
-- optimize read-receipts #1765
-
-- Allow http scheme for DCACCOUNT URLs #1770
-
-- improve tests #1769
-
-- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
-
-
-## 1.41.0
-
-- new apis to initiate video chats #1718 #1735
-
-- new apis `dc_msg_get_ephemeral_timer()`
-  and `dc_msg_get_ephemeral_timestamp()`
-
-- new api `dc_chatlist_get_summary2()` #1771
-
-- improve IMAP handling #1703 #1704
-
-- improve ephemeral messages #1696 #1705
-
-- mark location-messages as auto-generated #1715
-
-- multi-device avatar-sync #1716 #1717
-
-- improve python bindings #1732 #1733 #1738 #1769
-
-- Allow http scheme for DCACCOUNT urls #1770
-
-- more fixes #1702 #1706 #1707 #1710 #1719 #1721
-  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
-
-- refactorings #1712 #1714 #1757
-
-- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
-
-
-## 1.40.0
-
-- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
-
-- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
-
-- improve idle #1690 #1688
-
-- fix message processing issues by sequential processing #1694
-
-- refactorings #1670 #1673
-
+# Changelog 
 
 ## 1.39.0
 
@@ -1371,3 +583,4 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

CMakeLists.txt

@@ -1,42 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-include(GNUInstallDirs)
-
-find_program(CARGO cargo)
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,13 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.67.0"
+version = "1.39.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
-resolver = "2"
-
-[profile.dev]
-debug = 0
 
 [profile.release]
 lto = true
@@ -15,77 +11,68 @@ lto = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
-backtrace = "0.3"
-base64 = "0.13"
-bitflags = "1.3"
-byteorder = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+libc = "0.2.51"
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-kamadak-exif = "0.5"
+sha2 = "0.9.0"
+rand = "0.7.0"
+smallvec = "1.0.0"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+num-derive = "0.3.0"
+num-traits = "0.2.6"
+async-smtp = { version = "0.3" }
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
-once_cell = "1.8.0"
+async-imap = "0.3.1"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.19"
-rand = "0.7"
-regex = "1.5"
-rusqlite = "0.26"
-rust-hsluv = "0.1"
-rustyline = { version = "9.0", optional = true }
-sanitize-filename = "0.3"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9"
-sha2 = "0.9"
-smallvec = "1"
-stop-token = "0.6"
-strum = "0.23"
-strum_macros = "0.23"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "3.2.1"
-textwrap = "0.14.2"
+serde_json = "1.0"
+chrono = "0.4.6"
+indexmap = "1.3.0"
+kamadak-exif = "0.5"
+lazy_static = "1.4.0"
+regex = "1.1.6"
+strum = "0.18.0"
+strum_macros = "0.18.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.8.0"
+image-meta = "0.1.0"
+quick-xml = "0.18.1"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.12.1"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
+thiserror = "1.0.14"
+anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+async-std-resolver = "0.19.5"
+sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std", "sqlite", "macros"] }
+libsqlite3-sys = { version = "0.18", features = ["bundled", "min_sqlite_version_3_7_16"] }
+
+pretty_env_logger = { version = "0.4.0", optional = true }
+log = { version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
-futures-lite = "1.12"
-log = "0.4"
-pretty_env_logger = "0.4"
-proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.11"
+log = "0.4.8"
 
 [workspace]
 members = [
@@ -104,21 +91,9 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
-[[bench]]
-name = "create_account"
-harness = false
-
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
 [features]
-default = ["vendored"]
+default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]

README.md

@@ -2,7 +2,7 @@
 
 > Deltachat-core written in Rust 
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
 ## Installing Rust and Cargo
 
@@ -17,7 +17,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -79,16 +79,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -105,8 +95,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
-SMTP tracing in addition to info messages.
+
 
 ### Expensive tests
 
@@ -121,6 +110,11 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
@@ -128,7 +122,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
+- [Go](https://github.com/hugot/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

assets/icon-broadcast.svg

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   enable-background="new 0 0 128 128"
-   viewBox="0 0 60 60"
-   version="1.1"
-   id="svg878"
-   sodipodi:docname="icon-broadcast.svg"
-   width="60"
-   height="60"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
-  <metadata
-     id="metadata884">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs882" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1329"
-     inkscape:window-height="847"
-     id="namedview880"
-     showgrid="false"
-     inkscape:zoom="5.21875"
-     inkscape:cx="36.598802"
-     inkscape:cy="32.191617"
-     inkscape:window-x="111"
-     inkscape:window-y="205"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg878"
-     inkscape:document-rotation="0" />
-  <radialGradient
-     id="c"
-     cx="65.25"
-     cy="89"
-     r="26.440001"
-     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
-     gradientUnits="userSpaceOnUse">
-    <stop
-       stop-color="#FFC107"
-       offset="0"
-       id="stop833" />
-    <stop
-       stop-color="#FFBD06"
-       offset=".3502"
-       id="stop835" />
-    <stop
-       stop-color="#FFB104"
-       offset=".6938"
-       id="stop837" />
-    <stop
-       stop-color="#FFA000"
-       offset="1"
-       id="stop839" />
-  </radialGradient>
-  <radialGradient
-     id="b"
-     cx="52.5"
-     cy="19.75"
-     r="92.975998"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)">
-    <stop
-       stop-color="#EF5350"
-       offset="0"
-       id="stop848" />
-    <stop
-       stop-color="#EB4F4C"
-       offset=".246"
-       id="stop850" />
-    <stop
-       stop-color="#E04341"
-       offset=".4878"
-       id="stop852" />
-    <stop
-       stop-color="#CD302F"
-       offset=".7272"
-       id="stop854" />
-    <stop
-       stop-color="#C62828"
-       offset=".8004"
-       id="stop856" />
-    <stop
-       stop-color="#C62828"
-       offset="1"
-       id="stop858" />
-  </radialGradient>
-  <radialGradient
-     id="a"
-     cx="16.979"
-     cy="92"
-     r="24.165001"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)"
-     xlink:href="#b">
-    <stop
-       stop-color="#E0E0E0"
-       offset="0"
-       id="stop863" />
-    <stop
-       stop-color="#CFCFCF"
-       offset=".3112"
-       id="stop865" />
-    <stop
-       stop-color="#A4A4A4"
-       offset=".9228"
-       id="stop867" />
-    <stop
-       stop-color="#9E9E9E"
-       offset="1"
-       id="stop869" />
-  </radialGradient>
-  <rect
-     y="0"
-     x="0"
-     height="60"
-     width="60"
-     id="rect1420"
-     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
-  <path
-     id="path872"
-     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
-     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
-</svg>

assets/qrcode_logo_footer.svg

@@ -1,10 +0,0 @@
-<text
-     xml:space="preserve"
-     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     x="42.325161"
-     y="23.32255"
-     id="text72398">get.delta.chat</text>
-  <path
-     id="path84310"
-     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -1,39 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, &book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/create_account.rs

@@ -1,26 +0,0 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::accounts::Accounts;
-use tempfile::tempdir;
-
-async fn create_accounts(n: u32) {
-    let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
-
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-
-    for expected_id in 2..n {
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, expected_id);
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,29 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -1,12 +1,11 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Continuous Integration, run through CircleCI and an own build machine.
 
 ## Description of scripts 
 
-- `../.github/workflows` contains jobs run by GitHub Actions.
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -27,8 +26,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -42,10 +41,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd scripts  # where all CI things are 
+    cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

ci_scripts/ci_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install -U pip setuptools
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*
+
+popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust
+# Install Rust nightly
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -1,9 +1,12 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,17 +20,16 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
-
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -41,5 +43,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash scripts/run-python-test.sh 
+    bash ci_scripts/run-python-test.sh 
 _HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -4,7 +4,6 @@
 # and tox/pytest. 
 
 set -e -x
-shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -ex
-shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,19 +15,17 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
-export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.7
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
+ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -40,8 +38,9 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
+unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.67.0"
+version = "1.39.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,12 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1"
-num-traits = "0.2"
+human-panic = "1.0.1"
+num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1"
-anyhow = "1"
-thiserror = "1"
-rand = "0.7"
+async-std = "1.6.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -236,6 +236,12 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all
@@ -583,7 +589,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,16 +4,4 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
-	border-radius: 6px;
 }
-
-code {
-	background-color: #e0e0e0;
-	padding-left: .5em;
-	padding-right: .5em;
-	border-radius: 6px;
-}
-
-li {
-	margin-bottom: .5em;
-}

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/build.rs

@@ -19,17 +19,15 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
+        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/dc_array.rs

@@ -1,56 +1,46 @@
-use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
-use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
-    MsgIds(Vec<MsgId>),
-    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub(crate) fn get_id(&self, index: usize) -> u32 {
+    pub fn new(capacity: usize) -> Self {
+        dc_array_t::Uint(Vec::with_capacity(capacity))
+    }
+
+    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
+    pub fn new_locations(capacity: usize) -> Self {
+        dc_array_t::Locations(Vec::with_capacity(capacity))
+    }
+
+    pub fn add_id(&mut self, item: u32) {
+        if let Self::Uint(array) = self {
+            array.push(item);
+        } else {
+            panic!("Attempt to add id to array of other type");
+        }
+    }
+
+    pub fn add_location(&mut self, location: Location) {
+        if let Self::Locations(array) = self {
+            array.push(location)
+        } else {
+            panic!("Attempt to add a location to array of other type");
+        }
+    }
+
+    pub fn get_id(&self, index: usize) -> u32 {
         match self {
-            Self::MsgIds(array) => array[index].to_u32(),
-            Self::Chat(array) => match array[index] {
-                ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
-                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
-            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index],
+            Self::Uint(array) => array[index] as u32,
         }
     }
 
-    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(array) => array.get(index).and_then(|item| match item {
-                ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
-                ChatItem::DayMarker { timestamp } => Some(*timestamp),
-            }),
-            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(_) => None,
-            Self::Locations(array) => array
-                .get(index)
-                .and_then(|location| location.marker.as_deref()),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_location(&self, index: usize) -> &Location {
+    pub fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -58,18 +48,55 @@ impl dc_array_t {
         }
     }
 
-    /// Returns the number of elements in the array.
-    pub(crate) fn len(&self) -> usize {
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Locations(array) => array.is_empty(),
+            Self::Uint(array) => array.is_empty(),
+        }
+    }
+
+    /// Returns the number of elements in the array.
+    pub fn len(&self) -> usize {
         match self {
-            Self::MsgIds(array) => array.len(),
-            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
-        (0..self.len()).find(|i| self.get_id(*i) == needle)
+    pub fn clear(&mut self) {
+        match self {
+            Self::Locations(array) => array.clear(),
+            Self::Uint(array) => array.clear(),
+        }
+    }
+
+    pub fn search_id(&self, needle: u32) -> Option<usize> {
+        if let Self::Uint(array) = self {
+            for (i, &u) in array.iter().enumerate() {
+                if u == needle {
+                    return Some(i);
+                }
+            }
+            None
+        } else {
+            panic!("Attempt to search for id in array of other type");
+        }
+    }
+
+    pub fn sort_ids(&mut self) {
+        if let dc_array_t::Uint(v) = self {
+            v.sort();
+        } else {
+            panic!("Attempt to sort array of something other than uints");
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const u32 {
+        if let dc_array_t::Uint(v) = self {
+            v.as_ptr()
+        } else {
+            panic!("Attempt to convert array of something other than uints to raw");
+        }
     }
 }
 
@@ -79,18 +106,6 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
-impl From<Vec<MsgId>> for dc_array_t {
-    fn from(array: Vec<MsgId>) -> Self {
-        dc_array_t::MsgIds(array)
-    }
-}
-
-impl From<Vec<ChatItem>> for dc_array_t {
-    fn from(array: Vec<ChatItem>) -> Self {
-        dc_array_t::Chat(array)
-    }
-}
-
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -103,11 +118,12 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let arr: dc_array_t = Vec::<u32>::new().into();
-        assert!(arr.len() == 0);
+        let mut arr = dc_array_t::new(7);
+        assert!(arr.is_empty());
 
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
 
         assert_eq!(arr.len(), 1000);
 
@@ -115,15 +131,31 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        assert_eq!(arr.search_id(10), Some(8));
-        assert_eq!(arr.search_id(1), None);
+        arr.clear();
+
+        assert!(arr.is_empty());
+
+        arr.add_id(13);
+        arr.add_id(7);
+        arr.add_id(666);
+        arr.add_id(0);
+        arr.add_id(5000);
+
+        arr.sort_ids();
+
+        assert_eq!(arr.get_id(0), 0);
+        assert_eq!(arr.get_id(1), 7);
+        assert_eq!(arr.get_id(2), 13);
+        assert_eq!(arr.get_id(3), 666);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        let mut arr = dc_array_t::new(7);
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lot.rs

@@ -1,245 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use crate::message::MessageState;
-use crate::qr::Qr;
-use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Debug)]
-pub enum Lot {
-    Summary(Summary),
-    Qr(Qr),
-    Error(String),
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
-            },
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::FprOk { .. } => None,
-                Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
-                Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-            },
-            Self::Error(err) => Some(err),
-        }
-    }
-
-    pub fn get_text2(&self) -> Option<Cow<str>> {
-        match self {
-            Self::Summary(summary) => Some(summary.truncated_text(160)),
-            Self::Qr(_) => None,
-            Self::Error(_) => None,
-        }
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => Meaning::None,
-                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
-                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
-                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
-            },
-            Self::Qr(_qr) => Meaning::None,
-            Self::Error(_err) => Meaning::None,
-        }
-    }
-
-    pub fn get_state(&self) -> LotState {
-        match self {
-            Self::Summary(summary) => summary.state.into(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
-                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
-                Qr::FprOk { .. } => LotState::QrFprOk,
-                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
-                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
-                Qr::Account { .. } => LotState::QrAccount,
-                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Addr { .. } => LotState::QrAddr,
-                Qr::Url { .. } => LotState::QrUrl,
-                Qr::Text { .. } => LotState::QrText,
-                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
-                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
-                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
-                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
-            },
-            Self::Error(_err) => LotState::QrError,
-        }
-    }
-
-    pub fn get_id(&self) -> u32 {
-        match self {
-            Self::Summary(_) => Default::default(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
-                Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
-                Qr::FprWithoutAddr { .. } => Default::default(),
-                Qr::Account { .. } => Default::default(),
-                Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
-                Qr::Url { .. } => Default::default(),
-                Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
-            },
-            Self::Error(_) => Default::default(),
-        }
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        match self {
-            Self::Summary(summary) => summary.timestamp,
-            Self::Qr(_) => Default::default(),
-            Self::Error(_) => Default::default(),
-        }
-    }
-}
-
-#[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// text1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        Lot::Summary(summary)
-    }
-}
-
-impl From<Qr> for Lot {
-    fn from(qr: Qr) -> Self {
-        Lot::Qr(qr)
-    }
-}
-
-// Make it easy to convert errors into the final `Lot`.
-impl From<Error> for Lot {
-    fn from(error: Error) -> Self {
-        Lot::Error(error.to_string())
-    }
-}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -102,9 +105,8 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         })
     }
 
@@ -120,9 +122,8 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         }),
         None => Err(CStringError::NotUnicode),
     }
@@ -167,20 +168,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +184,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name
@@ -250,7 +239,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] slice can not outlive
+/// Because this returns a reference the [Path] silce can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1"
-quote = "1"
+syn = "1.0.13"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -8,40 +8,33 @@ use quote::quote;
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 
-#[proc_macro_derive(ToSql)]
-pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(Sqlx)]
+pub fn sqlx_derive(input: TokenStream) -> TokenStream {
     let ast: syn::DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
     let gen = quote! {
-        impl rusqlite::types::ToSql for #name {
-            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-                let num = *self as i64;
-                let value = rusqlite::types::Value::Integer(num);
-                let output = rusqlite::types::ToSqlOutput::Owned(value);
-                std::result::Result::Ok(output)
-            }
-        }
-    };
-    gen.into()
-}
-
-#[proc_macro_derive(FromSql)]
-pub fn from_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::FromSql for #name {
-            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-                let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
+        impl<'q> sqlx::encode::Encode<'q, sqlx::sqlite::Sqlite> for #name {
+            fn encode_by_ref(&self, buf: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
+                num_traits::ToPrimitive::to_i32(self).expect("invalid type").encode(buf)
             }
         }
+
+
+        impl<'de> sqlx::decode::Decode<'de, sqlx::sqlite::Sqlite> for #name {
+            fn decode(value: sqlx::sqlite::SqliteValueRef) -> std::result::Result<Self, Box<dyn std::error::Error + 'static + Send + Sync>> {
+                let raw: i32 = sqlx::decode::Decode::decode(value)?;
+
+                Ok(num_traits::FromPrimitive::from_i32(raw).unwrap_or_default())
+            }
+        }
+
+        impl sqlx::types::Type<sqlx::sqlite::Sqlite> for #name {
+            fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
+                <i32 as sqlx::types::Type<_>>::type_info()
+            }
+        }
+
     };
     gen.into()
 }

draft/group-sync.rst

@@ -5,95 +5,69 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-1. Alice and Bob are in a two-member group
+- Alice and Bob are in a two-member group
 
-2. Then Alice adds Carol, while concurrently Bob adds Doris
+- Alice adds Carol, concurrently Bob adds Doris
 
-Right now, the group has inconsistent memberships:
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
 
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
 
 
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
 
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
 
 
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
 
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
 
-For brevity let's abbreviate "group membership modification" as **GMM**.
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
 
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
+Notes:
 
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
+- mechanism works for both encrypted and unencrypted add/del messages
 
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
 
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
+- multiple member-added/removed messages can be attached in a single
+  correction message
 
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
 
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
 
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
 
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
   b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -109,3 +83,44 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

draft/msgwork_new_imap_jobs.rst

@@ -1,66 +0,0 @@
-
-simplify/streamline mark-seen/delete/move/send-mdn job handling
----------------------------------------------------------------
-
-Idea: Introduce a new "msgwork" sql table that looks very
-much like the jobs table but has a primary key "msgid"
-and no job id and no foreign-id anymore. This opens up
-bulk-processing by looking at the whole table and combining
-flag-setting to reduce imap-roundtrips and select-folder calls.
-
-Concretely, these IMAP jobs:
-
-    DeleteMsgOnImap
-    MarkseenMsgOnImap
-    MoveMsg
-
-Would be replaced by a few per-message columns in the new msgwork table:
-
-- needs_mark_seen: (bool) message shall be marked as seen on imap
-- needs_to_move: (bool) message should be moved to mvbox_folder
-- deletion_time: (target_time or 0) message shall be deleted at specified time
-- needs_send_mdn: (bool) MDN shall be sent
-
-The various places that currently add the (replaced) jobs
-would now add/modify the respective message record in the message-work table.
-
-Looking at a single message-work entry conceptually looks like this::
-
-    if msg.server_uid==0:
-        return RetryLater  # nothing can be done without server_uid
-
-    if msg.deletion_time > current_time:
-        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
-        clear(needs_deletion)
-        clear(mark_seen)
-
-    if needs_mark_seen:
-        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
-        clear(needs_mark_seen)
-
-    if needs_send_mdn:
-        schedule_smtp_send_mdn(msg)
-        clear(needs_send_mdn)
-
-    if any_flag_set():
-        retrylater
-    # remove msgwork entry from table
-
-
-Notes/Questions:
-
-- it's unclear how much we need per-message retry-time tracking/backoff
-
-- drafting bulk processing algo is useful before
-  going for the implementation, i.e. including select_folder calls etc.
-
-- maybe it's better to not have bools for the flags but
-
-  0 (no change)
-  1 (set the imap flag)
-  2 (clear the imap flag)
-
-  and design such that we can cover all imap flags.
-
-- It might not be neccessary to keep needs_send_mdn state in this table
-  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/main.rs

@@ -19,50 +19,50 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::Event;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
-fn receive_event(event: EventType) {
+fn receive_event(event: Event) {
     let yellow = Color::Yellow.normal();
     match event {
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        EventType::SmtpConnected(msg) => {
+        Event::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        EventType::ImapConnected(msg) => {
+        Event::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        EventType::SmtpMessageSent(msg) => {
+        Event::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
             warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        Event::Error(msg) => {
             error!("{}", msg);
         }
-        EventType::ErrorSelfNotInGroup(msg) => {
+        Event::ErrorNetwork(msg) => {
+            error!("[NETWORK] msg={}", msg);
+        }
+        Event::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        EventType::MsgsChanged { chat_id, msg_id } => {
+        Event::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,44 +71,34 @@ fn receive_event(event: EventType) {
                 ))
             );
         }
-        EventType::ContactsChanged(_) => {
+        Event::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        EventType::LocationChanged(contact) => {
+        Event::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        EventType::ConfigureProgress { progress, comment } => {
-            if let Some(comment) = comment {
-                info!(
-                    "{}",
-                    yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
-                    ))
-                );
-            } else {
-                info!(
-                    "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-                );
-            }
+        Event::ConfigureProgress(progress) => {
+            info!(
+                "{}",
+                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+            );
         }
-        EventType::ImexProgress(progress) => {
+        Event::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        EventType::ImexFileWritten(file) => {
+        Event::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        EventType::ChatModified(chat) => {
+        Event::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -156,7 +146,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
     "set",
     "get",
@@ -164,19 +154,18 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 35] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "createchatbymsg",
     "creategroup",
-    "createbroadcast",
-    "createprotected",
+    "createverified",
     "addmember",
     "removemember",
     "groupname",
@@ -189,49 +178,36 @@ const CHAT_COMMANDS: [&str; 35] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
-    "sendsyncmsg",
-    "videochat",
     "draft",
     "listmedia",
     "archive",
     "unarchive",
     "pin",
     "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
-    "download",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 10] = [
     "getqr",
-    "getqrsvg",
     "getbadqr",
     "checkqr",
-    "joinqr",
     "event",
     "fileinfo",
     "clear",
@@ -242,9 +218,7 @@ const MISC_COMMANDS: [&str; 12] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -266,10 +240,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -290,19 +265,18 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event.typ);
+            receive_event(event);
         }
     });
 
@@ -315,7 +289,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
+    let (reader_s, reader_r) = async_std::sync::channel(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -324,21 +298,21 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
 
         loop {
             let p = "> ";
-            let readline = rl.readline(p);
+            let readline = rl.readline(&p);
 
             match readline {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    async_std::task::block_on(reader_s.send(line));
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -397,9 +371,9 @@ async fn handle_cmd(
             ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -415,39 +389,27 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
-            if !qr.is_empty() {
-                if arg0 == "getbadqr" && qr.len() > 40 {
-                    qr.replace_range(12..22, "0000000000")
-                }
-                println!("{}", qr);
-                let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
-            }
-        }
-        "getqrsvg" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            match get_securejoin_qr_svg(&ctx, group).await {
-                Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get QR code svg: {}", err);
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
+                if !qr.is_empty() {
+                    if arg0 == "getbadqr" && qr.len() > 40 {
+                        qr.replace_range(12..22, "0000000000")
+                    }
+                    println!("{}", qr);
+                    let output = Command::new("qrencode")
+                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                        .output()
+                        .expect("failed to execute process");
+                    io::stdout().write_all(&output.stdout).unwrap();
+                    io::stderr().write_all(&output.stderr).unwrap();
                 }
             }
         }
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                dc_join_securejoin(&ctx, arg1).await;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -1,25 +1,25 @@
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::Event;
 
-fn cb(event: EventType) {
+fn cb(event: Event) {
     match event {
-        EventType::ConfigureProgress { progress, .. } => {
+        Event::ConfigureProgress(progress) => {
             log::info!("progress: {}", progress);
         }
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             log::info!("{}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
+    let ctx = Context::new("FakeOs".into(), dbfile.into())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event.typ);
+            cb(event);
         }
     });
 
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

proptest-regressions/dc_tools.txt

@@ -7,5 +7,3 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
-cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
-cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/CHANGELOG

@@ -1,29 +1,6 @@
-1.51.0
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
-1.44.0
-------
-
-- fix Chat.get_mute_duration()
-
-1.40.1
+0.900.0 (DRAFT)
 ---------------
 
-- emit "ac_member_removed" event (with 'actor' being the removed contact) 
-  for when a user leaves a group.
-
-- fix create_contact(addr) when addr is the self-contact. 
-
-
-1.40.0
----------------
-
-- uses latest 1.40+ Delta Chat core
-
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -33,7 +10,6 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
-
 0.800.0
 -------
 

python/README.rst

@@ -7,14 +7,76 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -41,79 +103,6 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Running tests
-=============
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-This will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _livetests:
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
-
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
-
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
-
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
-
-Each test run creates new accounts.
-
-
-.. _sourceinstall:
-
-Installing bindings from source (Updated: July 2020)
-=========================================================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   python install_python_bindings.py
-
-The core compilation and bindings building might take a while,
-depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
 Code examples
 =============
 
@@ -143,7 +132,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -152,9 +141,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/doc/_templates/globaltoc.html

@@ -9,7 +9,8 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
+  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/group_tracking.py

@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, actor, message):
+    def ac_member_added(self, chat, contact, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, actor, message):
+    def ac_member_removed(self, chat, contact, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_added {}*
+    """.format(contact3.addr))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -17,11 +17,8 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-
     if target == 'release':
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
-
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/mypy.ini

@@ -1,19 +0,0 @@
-[mypy]
-
-[mypy-deltachat.capi.*]
-ignore_missing_imports = True
-
-[mypy-pluggy.*]
-ignore_missing_imports = True
-
-[mypy-cffi.*]
-ignore_missing_imports = True
-
-[mypy-imapclient.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-_pytest.*]
-ignore_missing_imports = True

python/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,11 +8,17 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
+        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
+        use_scm_version = {
+            "root": "..",
+            "relative_to": __file__,
+            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
+        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
-        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name in dir(const):
+        for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[getattr(const, name)] = name
+                _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -77,7 +77,6 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/_build.py

@@ -9,6 +9,8 @@ import subprocess
 import tempfile
 import textwrap
 import types
+from os.path import abspath
+from os.path import dirname as dn
 
 import cffi
 
@@ -48,7 +50,6 @@ def system_build_flags():
     flags.objs = []
     flags.incs = []
     flags.extra_link_args = []
-    return flags
 
 
 def extract_functions(flags):
@@ -144,13 +145,9 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
-                | DC_IMEX
-                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -168,8 +165,11 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()

python/src/deltachat/account.py

@@ -8,14 +8,13 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
@@ -29,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -59,11 +58,11 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self) -> None:
+    def disable_logging(self):
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self) -> None:
+    def enable_logging(self):
         """ re-enable logging. """
         self._logging = True
 
@@ -74,12 +73,12 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name: str) -> None:
+    def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self) -> Dict[str, str]:
+    def get_info(self):
         """ return dictionary of built config parameters. """
         lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
         d = {}
@@ -90,35 +89,19 @@ class Account(object):
             d[key.lower()] = value
         return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
-    def set_stock_translation(self, id: int, string: str) -> None:
+    def set_stock_translation(self, id, string):
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        bytestring = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name: str, value: Optional[str]) -> None:
+    def set_config(self, name, value):
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -126,16 +109,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
+        name = name.encode("utf8")
+        if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            valuebytes = value.encode("utf8")
+            value = value.encode("utf8")
         else:
-            valuebytes = ffi.NULL
-        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
+            value = ffi.NULL
+        lib.dc_set_config(self._dc_context, name, value)
 
-    def get_config(self, name: str) -> str:
+    def get_config(self, name):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -144,12 +127,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, namebytes)
+        name = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, name)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
+    def _preconfigure_keypair(self, addr, public, secret):
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -161,7 +144,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs: Dict[str, Any]) -> None:
+    def update_config(self, kwargs):
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -171,7 +154,7 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self) -> bool:
+    def is_configured(self):
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -179,7 +162,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def set_avatar(self, img_path: Optional[str]) -> None:
+    def set_avatar(self, img_path):
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -191,18 +174,31 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self) -> None:
+    def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir) -> Optional[str]:
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
+    def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        return from_optional_dc_charpointer(res)
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
-    def get_blobdir(self) -> str:
+    def get_blobdir(self):
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -210,17 +206,17 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self) -> Contact:
+    def get_self_contact(self):
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
-        """create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        Calling this method will always result in the same
+        Calling this method will always resulut in the same
         underlying contact id.  If there already is a Contact
         with that e-mail address, it is unblocked and its display
         `name` is updated if specified.
@@ -229,19 +225,6 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
-
-    def get_contact(self, obj) -> Optional[Contact]:
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -257,9 +240,16 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return (name, addr)
+        return self._create_contact(addr, name)
 
-    def delete_contact(self, contact: Contact) -> bool:
+    def _create_contact(self, addr, name):
+        addr = as_dc_charpointer(addr)
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
+        return Contact(self, contact_id)
+
+    def delete_contact(self, contact):
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -270,40 +260,23 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
+    def get_contact_by_addr(self, email):
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
-        return None
 
-    def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
         :param contact_id: integer id of this contact.
-        :returns: :class:`deltachat.contact.Contact` instance.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self) -> List[Contact]:
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
-    def get_contacts(
-        self,
-        query: Optional[str] = None,
-        with_self: bool = False,
-        only_verified: bool = False,
-    ) -> List[Contact]:
-        """get a (filtered) list of contacts.
+    def get_contacts(self, query=None, with_self=False, only_verified=False):
+        """ get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -323,7 +296,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self) -> Generator[Message, None, None]:
+    def get_fresh_messages(self):
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -331,17 +304,15 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj) -> Chat:
+    def create_chat(self, obj):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(
-        self,
-        name: str,
-        contacts: Optional[List[Contact]] = None,
-        verified: bool = False,
-    ) -> Chat:
-        """create a new group chat object.
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+
+    def create_group_chat(self, name, contacts=None, verified=False):
+        """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -357,7 +328,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self) -> List[Chat]:
+    def get_chats(self):
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -374,17 +345,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self) -> Chat:
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -396,18 +367,19 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
+    def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            arr.append(getattr(msg, "id", msg))
+            msg = getattr(msg, "id", msg)
+            arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
+    def forward_messages(self, messages, chat):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -417,7 +389,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages):
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -431,23 +403,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account has to be stopped; call stop_io() if necessary.
+        Note that the account does not have to be started.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -457,7 +429,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
+        self._import(path, imex_cmd=2)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -465,28 +437,27 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-
-    def initiate_key_transfer(self) -> str:
+    def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self) -> str:
+    def get_setup_contact_qr(self):
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -536,9 +507,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -579,15 +548,6 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self) -> str:
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 
@@ -598,9 +558,6 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
-        If you are using this from a test, you may want to call
-        wait_all_initial_fetches() afterwards.
-
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
@@ -610,34 +567,12 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def maybe_network(self):
-        """This function should be called when there is a hint
-        that the network is available again,
-        e.g. as a response to system event reporting network availability.
-        The library will try to send pending messages out immediately.
-
-        Moreover, to have a reliable state
-        when the app comes to foreground with network available,
-        it may be reasonable to call the function also at that moment.
-
-        It is okay to call the function unconditionally when there is
-        network available, however, calling the function
-        _without_ having network may interfere with the backoff algorithm
-        and will led to let the jobs fail faster, with fewer retries
-        and may avoid messages being sent out.
-
-        Finally, if the context was created by the dc_accounts_t account manager
-        (currently not implemented in the Python bindings),
-        use dc_accounts_maybe_network() instead of this function
-        """
-        lib.dc_maybe_network(self._dc_context)
-
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
+    def configure(self):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert self.is_configured() == reconfigure
+        assert not self.is_configured()
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -645,19 +580,25 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self) -> None:
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
+
+    def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self) -> None:
+    def stop_io(self):
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        self.log("dc_stop_io (stop core IO scheduler)")
-        lib.dc_stop_io(self._dc_context)
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
 
-    def shutdown(self) -> None:
+    def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:
@@ -666,24 +607,12 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-
-        # if _dc_context is unref'ed the event thread should quickly
-        # receive the termination signal. However, some python code might
-        # still hold a reference and so we use a secondary signal
-        # to make sure the even thread terminates if it receives any new
-        # event, indepedently from waiting for the core to send NULL to
-        # get_next_event().
-        self._event_thread.mark_shutdown()
+        # the dc_context_unref triggers get_next_event to return ffi.NULL
+        # which in turns makes the event thread finish execution
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        try:
-            self._event_thread.wait(timeout=2)
-        except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
-
-        if self._event_thread.is_alive():
-            self.log("WARN: event thread did not terminate yet, ignoring.")
+        self._event_thread.wait()
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -3,13 +3,12 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
-from typing import Optional
 
 
 class Chat(object):
@@ -18,20 +17,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
+    def __init__(self, account, id):
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other):
         return not (self == other)
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -41,7 +40,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self) -> None:
+    def delete(self):
         """Delete this chat and all its messages.
 
         Note:
@@ -51,37 +50,32 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self) -> None:
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self) -> None:
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self) -> bool:
+    def is_group(self):
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
 
-    def is_muted(self) -> bool:
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
+    def is_muted(self):
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -91,38 +85,30 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self) -> bool:
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+    def is_verified(self):
+        """ return True if this chat is a verified group.
 
-        :returns: True if the chat is writable, False otherwise
+        :returns: True if chat is verified, False otherwise.
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return lib.dc_chat_is_verified(self._dc_chat)
 
-    def is_protected(self) -> bool:
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
-
-    def get_name(self) -> Optional[str]:
+    def get_name(self):
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name: str) -> bool:
+    def set_name(self, name):
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: True on success, False otherwise
+        :returns: None
         """
         name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
-    def mute(self, duration: Optional[int] = None) -> None:
+    def mute(self, duration=None):
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -136,7 +122,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self) -> None:
+    def unmute(self):
         """ unmutes the chat
 
         :returns: None
@@ -145,45 +131,22 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self) -> int:
+    def get_mute_duration(self):
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
-    def get_ephemeral_timer(self) -> int:
-        """ get ephemeral timer.
-
-        :returns: ephemeral timer value in seconds
-        """
-        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
-
-    def set_ephemeral_timer(self, timer: int) -> bool:
-        """ set ephemeral timer.
-
-        :param: timer value in seconds
-
-        :returns: True on success, False otherwise
-        """
-        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
-
-    def get_type(self) -> int:
+    def get_type(self):
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self) -> Optional[str]:
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
-    def get_join_qr(self) -> Optional[str]:
+    def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -195,7 +158,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg: Message) -> Message:
+    def send_msg(self, msg):
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -263,19 +226,17 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """ create a new prepared message.
 
         :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
+        :returns: :class:`deltachat.message.Message` instance.
         """
         msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -389,7 +350,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.get_contact(obj)
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -513,24 +474,18 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
-                ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
-        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -1,13 +1,7 @@
-from typing import Any, List
-
 from .capi import lib
 
 
-def __getattr__(name: str) -> Any:
+for name in dir(lib):
     if name.startswith("DC_"):
-        return getattr(lib, name)
-    return globals()[name]
-
-
-def __dir__() -> List[str]:
-    return sorted(name for name in dir(lib) if name.startswith("DC_"))
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -1,12 +1,10 @@
 """ Contact object. """
 
-from datetime import date, datetime, timezone
-from typing import Optional
-
-from . import const, props
-from .capi import ffi, lib
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
 from .chat import Chat
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer
+from . import const
 
 
 class Contact(object):
@@ -37,60 +35,35 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self) -> str:
+    def addr(self):
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self) -> str:
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
-    @props.with_doc
-    def last_seen(self) -> date:
-        """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
-    def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, block)
-
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self) -> Optional[str]:
+    def get_profile_image(self):
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        return from_optional_dc_charpointer(dc_res)
-
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
 
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.

python/src/deltachat/cutil.py

@@ -1,9 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
-
-T = TypeVar('T')
+from datetime import datetime
 
 
 def as_dc_charpointer(obj):
@@ -14,28 +11,20 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
+def iter_array(dc_array_t, constructor):
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj) -> str:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    raise ValueError
-
-
-def from_optional_dc_charpointer(obj) -> Optional[str]:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    return None
+def from_dc_charpointer(obj):
+    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:
-    def __init__(self, dc_lot) -> None:
+    def __init__(self, dc_lot):
         self._dc_lot = dc_lot
 
-    def id(self) -> int:
+    def id(self):
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -54,4 +43,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)

python/src/deltachat/direct_imap.py

@@ -9,9 +9,7 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
-import imaplib
 import deltachat
-from deltachat import const, Account
 
 
 SEEN = b'\\Seen'
@@ -26,30 +24,12 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.delete_folder(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl
@@ -62,38 +42,25 @@ def dc_account_after_shutdown(account):
 
 
 class DirectImap:
-    def __init__(self, account: Account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
         self.connect()
 
     def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
+        ssl_context = ssl.create_default_context()
 
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -108,12 +75,6 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
-    def create_folder(self, foldername):
-        try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -147,15 +108,6 @@ class DirectImap:
 
     def get_all_messages(self):
         assert not self._idling
-
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
         return self.conn.fetch(ALL, [FLAGS])
 
     def get_unread_messages(self):
@@ -173,6 +125,21 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()
@@ -192,13 +159,9 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
             for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
-                    continue
-
+                body_bytes = data[b'BODY[HEADER]']
                 flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
@@ -229,7 +192,6 @@ class DirectImap:
             raise TimeoutError
         if terminate:
             self.idle_done()
-        self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
     def idle_wait_for_seen(self):
@@ -249,18 +211,3 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
-
-    def append(self, folder, msg):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
-        """
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
-
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]

python/src/deltachat/events.py

@@ -1,7 +1,6 @@
 import threading
 import time
 import re
-import os
 from queue import Queue, Empty
 
 import deltachat
@@ -9,11 +8,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_optional_dc_charpointer
+from .cutil import from_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name: str, data1, data2):
+    def __init__(self, name, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,13 +28,13 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
+    def ac_process_ffi_event(self, ffi_event):
         self.account.log(str(ffi_event))
 
     @account_hookimpl
@@ -49,15 +48,6 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-
-        if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
-            if message.startswith("DC_EVENT_WARNING"):
-                s = WARN + s + ENDC
-            if message.startswith("DC_EVENT_ERROR"):
-                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -69,7 +59,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent):
+    def ac_process_ffi_event(self, ffi_event):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -96,48 +86,13 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex: str) -> FFIEvent:
-        rex = re.compile(regex)
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.search(ev.data2):
+            if rex.match(ev.data2):
                 return ev
 
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -156,15 +111,6 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_all_initial_fetches(self):
-        """Has to be called after start_io() to wait for fetch_existing_msgs to run
-        so that new messages are not mistaken for old ones:
-        - ac1 and ac2 are created
-        - ac1 sends a message to ac2
-        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("Done fetching existing messages")
-
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
@@ -176,7 +122,6 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
-        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -190,11 +135,10 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
-        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -203,17 +147,14 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self) -> None:
-        self._marked_for_shutdown = True
-
-    def wait(self, timeout=None) -> None:
+    def wait(self):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join(timeout=timeout)
+        self.join()
 
-    def run(self) -> None:
+    def run(self):
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -223,19 +164,17 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while not self._marked_for_shutdown:
+        while 1:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
-            if self._marked_for_shutdown:
-                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
@@ -251,7 +190,7 @@ class EventThread(threading.Thread):
                 if self.account._dc_context is not None:
                     raise
 
-    def _map_ffi_event(self, ffi_event: FFIEvent):
+    def _map_ffi_event(self, ffi_event):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are generally not allowed to block/last long as this
+    Hooks are not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,6 +31,10 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -43,7 +47,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):
@@ -51,37 +55,19 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
-
-        :param message: Message that was just delivered.
-        """
+        """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
-
-        :param chat: Chat which was modified.
-        """
+        """ Chat was created or modified regarding membership, avatar, title. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
-
-        :param chat: Chat where contact was added.
-        :param contact: Contact that was added.
-        :param actor: Who added the contact (None if it was our self-addr)
-        :param message: The original system message that reports the addition.
-        """
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
-
-        :param chat: Chat where contact was removed.
-        :param contact: Contact that was removed.
-        :param actor: Who removed the contact (None if it was our self-addr)
-        :param message: The original system message that reports the removal.
-        """
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
 
 
 class Global:

python/src/deltachat/message.py

@@ -1,13 +1,11 @@
 """ The Message object. """
 
 import os
-import re
 from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import datetime
 
 
 class Message(object):
@@ -22,8 +20,8 @@ class Message(object):
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        self.id = lib.dc_msg_get_id(dc_msg)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
@@ -47,13 +45,9 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param: view_type is "text", "audio", "video", "file"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
@@ -62,21 +56,19 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a contact request
+        If the message is a deaddrop contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.chat.accept()
-        return self.chat
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
-
-    @props.with_doc
-    def text(self) -> str:
+    def text(self):
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -84,23 +76,6 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self) -> str:
-        """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -114,13 +89,12 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self) -> str:
+    def basename(self):
         """basename of the attachment if it exists, otherwise empty string. """
-        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self) -> str:
+    def filemime(self):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -132,7 +106,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self) -> str:
+    def get_setupcodebegin(self):
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -140,15 +114,11 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self) -> str:
+    def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -172,7 +142,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def time_received(self):
@@ -182,48 +152,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @props.with_doc
-    def ephemeral_timer(self):
-        """Ephemeral timer in seconds
-
-        :returns: timer in seconds or None if there is no timer
-        """
-        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
-        if timer:
-            return timer
-
-    @props.with_doc
-    def ephemeral_timestamp(self):
-        """UTC time when the message will be deleted.
-
-        :returns: naive datetime.datetime() object or None if the timer is not started.
-        """
-        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
-        if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @property
-    def quoted_text(self) -> Optional[str]:
-        """Text inside the quote
-
-        :returns: Quoted text"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
-
-    @property
-    def quote(self):
-        """Quote getter
-
-        :returns: Quoted message, if found in the database"""
-        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
-        if msg:
-            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
-
-    @quote.setter
-    def quote(self, quoted_message):
-        """Quote setter"""
-        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+            return datetime.utcfromtimestamp(ts)
 
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
@@ -241,11 +170,6 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
-    @property
-    def error(self) -> Optional[str]:
-        """Error message"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
@@ -256,20 +180,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self) -> Optional[str]:
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_optional_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -382,10 +292,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -406,22 +312,21 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    const.DC_MSG_TEXT: 'text',
+    const.DC_MSG_IMAGE: 'image',
+    const.DC_MSG_GIF: 'gif',
+    const.DC_MSG_AUDIO: 'audio',
+    const.DC_MSG_VIDEO: 'video',
+    const.DC_MSG_FILE: 'file'
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
-    if code is not None:
-        return code
+    for code, value in _view_type_mapping.items():
+        if value == view_type_name:
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
 
 
 #
@@ -431,43 +336,20 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if not res:
-            return
-        action, affected, actor = res
-        affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
-        return "ac_member_" + res[0], d
-
-
-def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
-    if m:
-        text = m.group(1)
-    text = text.rstrip(".")
-    return text.strip()
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
 
 
 def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
-
-    returns a (action, affected, actor) triple """
-
     # Member Me (x@y) removed by a@b.
-    # Member x@y added by a@b
-    # Member With space (tmp1@x.org) removed by tmp2@x.org.
-    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-    # Group left by some one (tmp1@x.org).
-    # Group left by tmp1@x.org.
+    # Member x@y removed by a@b
     text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
-    if m:
-        affected, action, actor = m.groups()
-        return action, extract_addr(affected), extract_addr(actor)
-    if text.startswith("group left by "):
-        addr = extract_addr(text[13:])
-        if addr:
-            return "removed", addr, addr
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr) -> None:
+    def __init__(self, account, addr):
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self) -> str:
+    def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self) -> str:
+    def get_before_login_hints(self):
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
-    def status(self) -> int:
+    def status(self):
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -9,7 +9,6 @@ import fnmatch
 import time
 import weakref
 import tempfile
-from typing import List, Dict, Callable
 
 import pytest
 import requests
@@ -33,10 +32,6 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    parser.addoption(
-        "--strict-tls", action="store_true",
-        help="Never accept invalid TLS certificates for test accounts",
-    )
 
 
 def pytest_configure(config):
@@ -127,7 +122,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn) -> None:
+    def __init__(self, fn):
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -138,34 +133,32 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index: int):
+    def get(self, index):
         return self.configlist[index]
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    configlist: List[Dict[str, str]]
-
-    def __init__(self, url: str) -> None:
+    def __init__(self, url):
         self.configlist = []
         self.url = url
 
-    def get(self, index: int):
+    def get(self, index):
         try:
             return self.configlist[index]
         except IndexError:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
             return config
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
@@ -182,7 +175,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self) -> None:
+        def __init__(self):
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -213,10 +206,7 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        _finalizers: List[Callable[[], None]]
-        _accounts: List[Account]
-
-        def __init__(self) -> None:
+        def __init__(self):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
@@ -241,13 +231,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
-                logger = FFIEventLogger(ac)
-                logger.init_time = self.init_time
-                ac.add_account_plugin(logger)
+                ac.add_account_plugin(FFIEventLogger(ac))
             self._accounts.append(ac)
             return ac
 
@@ -257,7 +244,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -292,15 +282,16 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            if pytestconfig.getoption("--strict-tls"):
-                # Enable strict certificate checks for online accounts
-                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -318,36 +309,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io([ac1])
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
+            self.wait_configure_and_start_io()
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
-
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -355,35 +341,19 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            if hasattr(account, "direct_imap"):
-                # Attach the existing direct_imap. If we did not do this, a new one would be created and
-                # delete existing messages (see dc_account_extra_configure(configure))
-                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
-            started_accounts = []
-            for acc in accounts:
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
-            for acc in started_accounts:
-                acc._evtracker.wait_all_initial_fetches()
-
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
@@ -420,16 +390,16 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
+                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        def get_accepted_chat(self, ac1: Account, ac2: Account):
+        def get_accepted_chat(self, ac1, ac2):
             ac2.create_chat(ac1)
             return ac1.create_chat(ac2)
 
@@ -457,9 +427,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
 
 class BotProcess:
-    stdout_queue: queue.Queue
-
-    def __init__(self, popen, bot_cfg) -> None:
+    def __init__(self, popen, bot_cfg):
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -467,10 +435,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(True)
+        t.setDaemon(1)
         t.start()
 
-    def _run_stdout_thread(self) -> None:
+    def _run_stdout_thread(self):
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -482,10 +450,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self) -> None:
+    def kill(self):
         self.popen.kill()
 
-    def wait(self, timeout=30) -> None:
+    def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -517,16 +485,13 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg: str) -> None:
+        def sec(self, msg):
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg: str) -> None:
+        def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg: str) -> None:
-            print("  " + msg)
-
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,16 +20,6 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
-
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/auditwheels.py

@@ -1,17 +1,15 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                 "--plat", "manylinux2014_x86_64"])

python/tests/test_increation.py

@@ -6,6 +6,8 @@ import shutil
 import pytest
 from filecmp import cmp
 
+from deltachat import const
+
 
 def wait_msg_delivered(account, msg_list):
     """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
@@ -100,10 +102,14 @@ class TestOnlineInCreation:
         ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tox.ini

@@ -1,19 +1,19 @@
 [tox]
-isolated_build = true
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py37
     lint
-    mypy
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
+    DCC_PY_LIVECONFIG
     DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
@@ -44,15 +44,6 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
-[testenv:mypy]
-deps =
-    mypy
-    typing
-    types-setuptools
-    types-requests
-commands =
-    mypy --no-incremental src/
-
 [testenv:doc]
 changedir=doc
 deps =

run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   scripts/run-integration-tests.sh -e py35
+#   ./run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   scripts/run-integration-tests.sh -e py35 -- -x
+#   ./run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,6 +23,9 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
+    export DCC_PY_LIVECONFIG=liveconfig
+fi
 tox "$@"
 ret=$?
 popd

rust-toolchain

@@ -1 +1 @@
-1.54.0
+1.43.1

scripts/concourse/README.md

@@ -1,21 +0,0 @@
-# Concourse CI pipeline
-
-`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
-that builds C documentation, Python documentation, Python wheels for `x86_64`
-and `aarch64` and Python source packages, and uploads them.
-
-To setup the pipeline run
-```
-fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
-```
-where `secret.yml` contains the following secrets:
-```
-c.delta.chat:
-  private_key: |
-    -----BEGIN RSA PRIVATE KEY-----
-    ...
-    -----END RSA PRIVATE KEY-----
-devpi:
-  login: dc
-  password: ...
-```

scripts/concourse/docs_wheels.yml

@@ -1,232 +0,0 @@
-resources:
-- name: deltachat-core-rust
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-
-- name: deltachat-core-rust-release
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-    tag_filter: "py-*"
-
-jobs:
-- name: doxygen
-  plan:
-  - get: deltachat-core-rust
-    trigger: true
-
-  # Build Doxygen documentation
-  - task: build-doxygen
-    config:
-      inputs:
-        - name: deltachat-core-rust
-      outputs:
-        - name: c-docs
-      image_resource:
-        source:
-          repository: hrektts/doxygen
-        type: registry-image
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
-          cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
-
-  - task: upload-c-docs
-    config:
-      inputs:
-        - name: c-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
-- name: python-x86_64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-docs
-          path: ./python/doc/_build/
-        # Source packages
-        - name: py-dist
-          path: ./python/.docker-tox/dist/
-        # Binary wheels
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload python docs to py.delta.chat
-  - task: upload-py-docs
-    config:
-      inputs:
-        - name: py-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
-  # Upload x86_64 wheels and source packages
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-        - name: py-dist
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*
-          devpi upload py-dist/*
-
-- name: python-aarch64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload aarch64 wheels
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*

scripts/coverage.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-set -eu
-
-if ! command -v grcov >/dev/null; then
-    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
-    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
-    exit 1
-fi
-
-# Allow `-Z` flags without using nightly Rust.
-export RUSTC_BOOTSTRAP=1
-
-# We are using `-Zprofile` instead of source-based coverage [1]
-# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
-#
-# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
-# [2] https://github.com/mozilla/grcov/issues/595
-
-export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
-export RUSTDOCFLAGS="-Cpanic=abort"
-cargo clean
-cargo build
-cargo test
-
-grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment 
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/manual_remote_tests.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -xe
-
-JOB=${1:?need to specify 'rust' or 'python'}
-BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
-REPONAME="$(basename $(git rev-parse --show-toplevel))"
-
-time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"