more type fixes

.circleci/config.yml

@@ -0,0 +1,230 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
+jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: pyenv versions
+      - run: pyenv global 3.5.2
+      - run: ls -laR workspace
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy 
+
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      # - cargo_fetch
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
+
+      - build_doxygen:
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      # Linux Desktop 64bit
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "monthly"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 50

.github/mergeable.yml

@@ -1,26 +0,0 @@
-version: 2
-mergeable:
-  - when: pull_request.*
-    name: "Changelog check"
-    validate:
-      - do: or
-        validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/**'
-                required: ['CHANGELOG.md']
-    fail:
-      - do: checks
-        status: 'action_required' 
-        payload:
-          title: Changlog might need an update
-          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -8,24 +8,19 @@ on:
       - staging
       - trying
 
-env:
-  RUSTFLAGS: -Dwarnings
-
 jobs:
 
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.43.1
           override: true
       - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           command: fmt
@@ -34,64 +29,25 @@ jobs:
   run_clippy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.43.1
             components: clippy
             override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches
 
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.60.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.60.0
-            python: false # Python bindings compilation on Windows is not supported.
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly, 1.43.1]
 
-          # Minimum Supported Rust Version = 1.56.0
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.56.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -101,43 +57,33 @@ jobs:
         toolchain: ${{ matrix.rust }}
         override: true
 
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
 
     - name: check
       uses: actions-rs/cargo@v1
       with:
-        command: check
-        args: --all --bins --examples --tests --features repl --benches
+        command:  check
+        args: --workspace --all --bins --examples --tests
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3
+        args: --workspace
+          

.github/workflows/dependabot.yml

@@ -1,21 +0,0 @@
-name: Dependabot auto-approve
-on: pull_request
-
-permissions:
-  pull-requests: write
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Approve a PR
-        run: gh pr review --approve "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.github/workflows/upload-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on rs.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat --no-deps
-    - name: Upload to rs.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/rs/"
-

.github/workflows/upload-ffi-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on cffi.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat_ffi --no-deps
-    - name: Upload to cffi.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/cffi/"
-

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -26,10 +25,3 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
-
-coverage/
-.DS_Store
-.vscode/launch.json
-python/accounts.txt
-python/all-testaccounts.txt
-tmp/

ASYNC-API-TODO.txt

@@ -0,0 +1,54 @@
+
+Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
+
+- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
+
+APIs:
+    dc_context_new                 # opens the database
+    dc_open                             # FFI only
+      -> drop it and move parameters to dc_context_new()
+
+    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
+    dc_imex NEVER goes through the job system
+    dc_imex import_backup needs to ensure dc_stop_jobs()
+
+    dc_start_io                # start smtp/imap and job handling subsystems
+    dc_stop_io                 # stop smtp/imap and job handling subsystems
+    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
+
+    dc_close                            # FFI only
+      -> can be dropped
+    dc_context_unref
+
+    for ios share-extension:
+    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
+        0: message was sent
+        1: message failed to go out, is queued as a job to be retried later
+        2: message permanently failed?
+
+EVENT handling:
+    start a callback thread and call get_next_event() which is BLOCKING
+    it's fine to start this callback thread later, it will see all events.
+    Note that the core infinitely fills the internal queue if you never drain it.
+
+    FFI-get_next_event() returns NULL if the context is unrefed already?
+
+   sidenote: how python's callback thread does it currently:
+         CB-thread runs this while loop:
+         while not QUITFLAG:
+             ev = context.get_next_event( )
+             ...
+         So in order to shutdown properly one has to set QUITFLAG
+         before calling dc_stop_jobs() and dc_context_unref
+
+    event API:
+        get_data1_int
+        get_data2_int
+        get_data3_str
+
+
+- userdata likely only used for the callbacks, likely can be dropped, needs verification
+
+
+- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
+

CMakeLists.txt

@@ -1,50 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-include(GNUInstallDirs)
-
-find_program(CARGO cargo)
-
-if(APPLE)
-       set(DYNAMIC_EXT "dylib")
-elseif(UNIX)
-       set(DYNAMIC_EXT "so")
-else()
-       set(DYNAMIC_EXT "dll")
-endif()
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
-	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,13 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.80.0"
+version = "1.39.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2021"
+edition = "2018"
 license = "MPL-2.0"
-rust-version = "1.56"
-
-[profile.dev]
-debug = 0
 
 [profile.release]
 lto = true
@@ -15,76 +11,68 @@ lto = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
-async-std-resolver = "0.21"
-async-std = { version = "1" }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
-backtrace = "0.3"
-base64 = "0.13"
-bitflags = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+libc = "0.2.51"
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-kamadak-exif = "0.5"
+sha2 = "0.9.0"
+rand = "0.7.0"
+smallvec = "1.0.0"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+num-derive = "0.3.0"
+num-traits = "0.2.6"
+async-smtp = { version = "0.3" }
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
-log = {version = "0.4.16", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
-once_cell = "1.10.0"
+async-imap = "0.3.1"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.20"
-rand = "0.7"
-regex = "1.5"
-rusqlite = { version = "0.27", features = ["sqlcipher"] }
-rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.10"
-sha2 = "0.10"
-smallvec = "1"
-strum = "0.24"
-strum_macros = "0.24"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "4.3.3"
-textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
+serde_json = "1.0"
+chrono = "0.4.6"
+indexmap = "1.3.0"
+kamadak-exif = "0.5"
+lazy_static = "1.4.0"
+regex = "1.1.6"
+strum = "0.18.0"
+strum_macros = "0.18.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.8.0"
+image-meta = "0.1.0"
+quick-xml = "0.18.1"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.12.1"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
+thiserror = "1.0.14"
+anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+async-std-resolver = "0.19.5"
+sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std", "sqlite", "macros"] }
+libsqlite3-sys = { version = "0.18", features = ["bundled", "min_sqlite_version_3_7_16"] }
+
+pretty_env_logger = { version = "0.4.0", optional = true }
+log = { version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = { version = "0.3.4", features = ["async_std"] }
-futures-lite = "1.12"
-log = "0.4"
-pretty_env_logger = "0.4"
-proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.11"
+log = "0.4.8"
 
 [workspace]
 members = [
@@ -103,33 +91,9 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
-[[bench]]
-name = "create_account"
-harness = false
-
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
-[[bench]]
-name = "receive_emails"
-harness = false
-
-[[bench]]
-name = "get_chat_msgs"
-harness = false
-
-[[bench]]
-name = "get_chatlist"
-harness = false
-
 [features]
-default = ["vendored"]
+default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]

README.md

@@ -2,7 +2,7 @@
 
 > Deltachat-core written in Rust 
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
 ## Installing Rust and Cargo
 
@@ -12,14 +12,12 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
-> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
-
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -81,16 +79,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -107,8 +95,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
-SMTP tracing in addition to info messages.
+
 
 ### Expensive tests
 
@@ -123,15 +110,20 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
 
-- **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
-- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
-- **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

assets/icon-broadcast.svg

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   enable-background="new 0 0 128 128"
-   viewBox="0 0 60 60"
-   version="1.1"
-   id="svg878"
-   sodipodi:docname="icon-broadcast.svg"
-   width="60"
-   height="60"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
-  <metadata
-     id="metadata884">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs882" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1329"
-     inkscape:window-height="847"
-     id="namedview880"
-     showgrid="false"
-     inkscape:zoom="5.21875"
-     inkscape:cx="36.598802"
-     inkscape:cy="32.191617"
-     inkscape:window-x="111"
-     inkscape:window-y="205"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg878"
-     inkscape:document-rotation="0" />
-  <radialGradient
-     id="c"
-     cx="65.25"
-     cy="89"
-     r="26.440001"
-     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
-     gradientUnits="userSpaceOnUse">
-    <stop
-       stop-color="#FFC107"
-       offset="0"
-       id="stop833" />
-    <stop
-       stop-color="#FFBD06"
-       offset=".3502"
-       id="stop835" />
-    <stop
-       stop-color="#FFB104"
-       offset=".6938"
-       id="stop837" />
-    <stop
-       stop-color="#FFA000"
-       offset="1"
-       id="stop839" />
-  </radialGradient>
-  <radialGradient
-     id="b"
-     cx="52.5"
-     cy="19.75"
-     r="92.975998"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)">
-    <stop
-       stop-color="#EF5350"
-       offset="0"
-       id="stop848" />
-    <stop
-       stop-color="#EB4F4C"
-       offset=".246"
-       id="stop850" />
-    <stop
-       stop-color="#E04341"
-       offset=".4878"
-       id="stop852" />
-    <stop
-       stop-color="#CD302F"
-       offset=".7272"
-       id="stop854" />
-    <stop
-       stop-color="#C62828"
-       offset=".8004"
-       id="stop856" />
-    <stop
-       stop-color="#C62828"
-       offset="1"
-       id="stop858" />
-  </radialGradient>
-  <radialGradient
-     id="a"
-     cx="16.979"
-     cy="92"
-     r="24.165001"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)"
-     xlink:href="#b">
-    <stop
-       stop-color="#E0E0E0"
-       offset="0"
-       id="stop863" />
-    <stop
-       stop-color="#CFCFCF"
-       offset=".3112"
-       id="stop865" />
-    <stop
-       stop-color="#A4A4A4"
-       offset=".9228"
-       id="stop867" />
-    <stop
-       stop-color="#9E9E9E"
-       offset="1"
-       id="stop869" />
-  </radialGradient>
-  <rect
-     y="0"
-     x="0"
-     height="60"
-     width="60"
-     id="rect1420"
-     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
-  <path
-     id="path872"
-     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
-     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
-</svg>

assets/icon-webxdc.svg

@@ -1,181 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="80mm"
-   height="297mm"
-   viewBox="0 0 80 297"
-   version="1.1"
-   id="svg71"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   sodipodi:docname="icon-webxdc.svg"
-   inkscape:export-filename="C:\Users\user\OneDrive - BFW-Leipzig\Documents\LogoDC\finalohnerand.png"
-   inkscape:export-xdpi="96"
-   inkscape:export-ydpi="96">
-  <metadata
-     id="metadata856">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <sodipodi:namedview
-     id="namedview73"
-     pagecolor="#767676"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="true"
-     inkscape:document-units="mm"
-     showgrid="false"
-     showborder="false"
-     inkscape:snap-bbox="true"
-     inkscape:bbox-paths="true"
-     inkscape:bbox-nodes="true"
-     inkscape:snap-bbox-edge-midpoints="true"
-     inkscape:snap-bbox-midpoints="true"
-     inkscape:object-paths="true"
-     inkscape:snap-intersection-paths="true"
-     inkscape:zoom="1.4142136"
-     inkscape:cx="-90.271136"
-     inkscape:cy="-1233.1209"
-     inkscape:window-width="1864"
-     inkscape:window-height="1027"
-     inkscape:window-x="56"
-     inkscape:window-y="25"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="layer1"
-     inkscape:snap-global="false"
-     showguides="false"
-     inkscape:guide-bbox="true"
-     inkscape:document-rotation="0"
-     units="px">
-    <sodipodi:guide
-       position="-154.76097,641.11689"
-       orientation="0,-1"
-       id="guide21118" />
-    <sodipodi:guide
-       position="-60.286487,633.36619"
-       orientation="0,-1"
-       id="guide21120" />
-  </sodipodi:namedview>
-  <defs
-     id="defs68">
-    <linearGradient
-       id="linearGradient4375">
-      <stop
-         style="stop-color:#364e59;stop-opacity:1;"
-         offset="0"
-         id="stop4377" />
-      <stop
-         style="stop-color:#364e59;stop-opacity:0;"
-         offset="1"
-         id="stop4379" />
-    </linearGradient>
-  </defs>
-  <g
-     inkscape:label="Ebene 1"
-     inkscape:groupmode="layer"
-     id="layer1">
-    <rect
-       style="fill:#1a1a1a;stroke:#000000;stroke-width:0.167903"
-       id="rect880"
-       width="79.8321"
-       height="79.8321"
-       x="-64.03286"
-       y="-375.9097"
-       ry="0" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path3799-2"
-       d="m -24.089585,-372.59579 c -19.986026,0.24336 -36.196903,16.666 -36.196903,36.67011 0,20.00409 16.210877,36.03233 36.196903,35.78912 19.0024236,-0.076 14.5340713,-10.6146 35.538854,-0.85693 -11.50627538,-17.97454 0.390097,-20.36737 0.658079,-35.81316 0,-20.00411 -16.2108788,-36.03235 -36.196911,-35.78914 z"
-       style="fill:#364e59;fill-opacity:1;stroke:none;stroke-width:1.93355"
-       sodipodi:nodetypes="sscccs" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -54.193871,-325.26419 Z"
-       id="path3846" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.397951,-326.67773 Z"
-       id="path3848" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="m -49.397951,-326.67773 v 0 0"
-       id="path3850" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.01;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
-       d="m -51.35133,-325.0334 -7.964067,5.98895 z"
-       id="path3965" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path11037"
-       d="m -24.089585,-372.19891 c -19.986026,0.24156 -36.196903,16.54352 -36.196903,36.40062 0,7.86524 2.543315,15.1113 6.857155,20.97971 6.577146,8.94734 11.123515,9.77363 11.123515,9.77363 1.343237,1.78324 10.270932,4.3223 10.270932,4.3223 l 16.791727,-70.86654 -0.468369,-0.33457 c 0.458597,0.26445 0.428277,-0.27515 -8.378035,-0.27515 z"
-       style="fill:#7cc5cc;fill-opacity:1;stroke:none;stroke-width:1.92643"
-       sodipodi:nodetypes="sssccccss" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.944239,-310.69957 Z"
-       id="path13674" />
-    <g
-       id="g15178"
-       transform="matrix(0.79975737,0,0,0.79975737,53.088959,-63.716396)">
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072"
-         width="29.897917"
-         height="6.8791666"
-         x="-334.4964"
-         y="-154.51025"
-         transform="rotate(45)" />
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072-5"
-         width="29.897917"
-         height="6.8791666"
-         x="147.63107"
-         y="-334.4964"
-         transform="rotate(-45)"
-         inkscape:transform-center-x="-0.74835017"
-         inkscape:transform-center-y="0.37417525" />
-    </g>
-    <g
-       id="g22468"
-       transform="translate(3.3033974)">
-      <g
-         id="g15178-0"
-         transform="matrix(-0.79975737,0,0,0.79975737,-103.11028,-63.716404)"
-         style="fill:#7cc5cc;fill-opacity:1">
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-2"
-           width="29.897917"
-           height="6.8791666"
-           x="-334.4964"
-           y="-154.51025"
-           transform="rotate(45)" />
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-5-5"
-           width="29.897917"
-           height="6.8791666"
-           x="147.63107"
-           y="-334.4964"
-           transform="rotate(-45)"
-           inkscape:transform-center-x="-0.74835017"
-           inkscape:transform-center-y="0.37417525" />
-      </g>
-    </g>
-  </g>
-</svg>

assets/qrcode_logo_footer.svg

@@ -1,10 +0,0 @@
-<text
-     xml:space="preserve"
-     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     x="42.325161"
-     y="23.32255"
-     id="text72398">get.delta.chat</text>
-  <path
-     id="path84310"
-     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -1,37 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, &book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/create_account.rs

@@ -1,26 +0,0 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::accounts::Accounts;
-use tempfile::tempdir;
-
-async fn create_accounts(n: u32) {
-    let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
-
-    let mut accounts = Accounts::new(p.clone()).await.unwrap();
-
-    for expected_id in 2..n {
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, expected_id);
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/get_chat_msgs.rs

@@ -1,40 +0,0 @@
-use async_std::path::Path;
-
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chat::{self, ChatId};
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    for c in chats.iter().take(10) {
-        black_box(chat::get_chat_msgs(&context, *c, 0, None).await.ok());
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
-            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
-            let len = chatlist.len();
-            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
-        });
-
-        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/get_chatlist.rs

@@ -1,27 +0,0 @@
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_list_benchmark(context: &Context) {
-    Chatlist::try_load(context, 0, None, None).await.unwrap();
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
-        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_list_benchmark(black_box(&context)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/receive_emails.rs

@@ -1,84 +0,0 @@
-use async_std::{path::PathBuf, task::block_on};
-use criterion::{
-    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
-    Criterion,
-};
-use deltachat::{
-    config::Config,
-    context::Context,
-    dc_receive_imf::dc_receive_imf,
-    imex::{imex, ImexMode},
-};
-use tempfile::tempdir;
-
-async fn recv_all_emails(context: Context) -> Context {
-    for i in 0..100 {
-        let imf_raw = format!(
-            "Subject: Benchmark
-Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
-Date: Sat, 07 Dec 2019 19:00:27 +0000
-To: alice@example.com
-From: sender@testrun.org
-Chat-Version: 1.0
-Chat-Disposition-Notification-To: sender@testrun.org
-Chat-User-Avatar: 0
-In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
-MIME-Version: 1.0
-
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Hello {i}",
-            i = i,
-            i_dec = i - 1,
-        );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
-            .await
-            .unwrap();
-    }
-    context
-}
-
-async fn create_context() -> Context {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    let backup: PathBuf = std::env::current_dir()
-        .unwrap()
-        .join("delta-chat-backup.tar")
-        .into();
-    if backup.exists().await {
-        println!("Importing backup");
-        imex(&context, ImexMode::ImportBackup, &backup, None)
-            .await
-            .unwrap();
-    }
-
-    let addr = "alice@example.com";
-    context.set_config(Config::Addr, Some(addr)).await.unwrap();
-    context
-        .set_config(Config::ConfiguredAddr, Some(addr))
-        .await
-        .unwrap();
-    context
-        .set_config(Config::Configured, Some("1"))
-        .await
-        .unwrap();
-    context
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    let mut group = c.benchmark_group("Receive messages");
-    group.bench_function("Receive 100 simple text msgs", |b| {
-        b.to_async(AsyncStdExecutor).iter_batched(
-            || block_on(create_context()),
-            |context| recv_all_emails(black_box(context)),
-            BatchSize::LargeInput,
-        );
-    });
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,29 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -1,12 +1,11 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Continuous Integration, run through CircleCI and an own build machine.
 
 ## Description of scripts 
 
-- `../.github/workflows` contains jobs run by GitHub Actions.
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -27,8 +26,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -42,10 +41,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd scripts  # where all CI things are 
+    cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

ci_scripts/ci_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install -U pip setuptools
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*
+
+popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2014_aarch64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust
+# Install Rust nightly
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -1,9 +1,12 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,17 +20,16 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
-
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -41,5 +43,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash scripts/run-python-test.sh 
+    bash ci_scripts/run-python-test.sh 
 _HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -4,7 +4,6 @@
 # and tox/pytest. 
 
 set -e -x
-shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -ex
-shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,19 +15,17 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
-export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.7
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
+ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -40,8 +38,9 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
+unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.80.0"
+version = "1.39.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,12 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1"
-num-traits = "0.2"
+human-panic = "1.0.1"
+num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1"
-anyhow = "1"
-thiserror = "1"
-rand = "0.7"
+async-std = "1.6.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -236,6 +236,12 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all
@@ -583,7 +589,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,16 +4,4 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
-	border-radius: 6px;
 }
-
-code {
-	background-color: #e0e0e0;
-	padding-left: .5em;
-	padding-right: .5em;
-	border-radius: 6px;
-}
-
-li {
-	margin-bottom: .5em;
-}

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/README.md

@@ -1,9 +1,5 @@
 # Delta Chat C Interface
 
-## Installation
-
-see `Installing libdeltachat system wide` in [../README.md](../README.md)
-
 ## Documentation
 
 To generate the C Interface documentation,

deltachat-ffi/build.rs

@@ -19,17 +19,15 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
+        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/dc_array.rs

@@ -1,56 +1,46 @@
-use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
-use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
-    MsgIds(Vec<MsgId>),
-    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub(crate) fn get_id(&self, index: usize) -> u32 {
+    pub fn new(capacity: usize) -> Self {
+        dc_array_t::Uint(Vec::with_capacity(capacity))
+    }
+
+    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
+    pub fn new_locations(capacity: usize) -> Self {
+        dc_array_t::Locations(Vec::with_capacity(capacity))
+    }
+
+    pub fn add_id(&mut self, item: u32) {
+        if let Self::Uint(array) = self {
+            array.push(item);
+        } else {
+            panic!("Attempt to add id to array of other type");
+        }
+    }
+
+    pub fn add_location(&mut self, location: Location) {
+        if let Self::Locations(array) = self {
+            array.push(location)
+        } else {
+            panic!("Attempt to add a location to array of other type");
+        }
+    }
+
+    pub fn get_id(&self, index: usize) -> u32 {
         match self {
-            Self::MsgIds(array) => array[index].to_u32(),
-            Self::Chat(array) => match array[index] {
-                ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
-                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
-            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index],
+            Self::Uint(array) => array[index] as u32,
         }
     }
 
-    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(array) => array.get(index).and_then(|item| match item {
-                ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
-                ChatItem::DayMarker { timestamp } => Some(*timestamp),
-            }),
-            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(_) => None,
-            Self::Locations(array) => array
-                .get(index)
-                .and_then(|location| location.marker.as_deref()),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_location(&self, index: usize) -> &Location {
+    pub fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -58,18 +48,55 @@ impl dc_array_t {
         }
     }
 
-    /// Returns the number of elements in the array.
-    pub(crate) fn len(&self) -> usize {
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Locations(array) => array.is_empty(),
+            Self::Uint(array) => array.is_empty(),
+        }
+    }
+
+    /// Returns the number of elements in the array.
+    pub fn len(&self) -> usize {
         match self {
-            Self::MsgIds(array) => array.len(),
-            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
-        (0..self.len()).find(|i| self.get_id(*i) == needle)
+    pub fn clear(&mut self) {
+        match self {
+            Self::Locations(array) => array.clear(),
+            Self::Uint(array) => array.clear(),
+        }
+    }
+
+    pub fn search_id(&self, needle: u32) -> Option<usize> {
+        if let Self::Uint(array) = self {
+            for (i, &u) in array.iter().enumerate() {
+                if u == needle {
+                    return Some(i);
+                }
+            }
+            None
+        } else {
+            panic!("Attempt to search for id in array of other type");
+        }
+    }
+
+    pub fn sort_ids(&mut self) {
+        if let dc_array_t::Uint(v) = self {
+            v.sort();
+        } else {
+            panic!("Attempt to sort array of something other than uints");
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const u32 {
+        if let dc_array_t::Uint(v) = self {
+            v.as_ptr()
+        } else {
+            panic!("Attempt to convert array of something other than uints to raw");
+        }
     }
 }
 
@@ -79,18 +106,6 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
-impl From<Vec<MsgId>> for dc_array_t {
-    fn from(array: Vec<MsgId>) -> Self {
-        dc_array_t::MsgIds(array)
-    }
-}
-
-impl From<Vec<ChatItem>> for dc_array_t {
-    fn from(array: Vec<ChatItem>) -> Self {
-        dc_array_t::Chat(array)
-    }
-}
-
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -103,11 +118,12 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let arr: dc_array_t = Vec::<u32>::new().into();
-        assert!(arr.len() == 0);
+        let mut arr = dc_array_t::new(7);
+        assert!(arr.is_empty());
 
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
 
         assert_eq!(arr.len(), 1000);
 
@@ -115,15 +131,31 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        assert_eq!(arr.search_id(10), Some(8));
-        assert_eq!(arr.search_id(1), None);
+        arr.clear();
+
+        assert!(arr.is_empty());
+
+        arr.add_id(13);
+        arr.add_id(7);
+        arr.add_id(666);
+        arr.add_id(0);
+        arr.add_id(5000);
+
+        arr.sort_ids();
+
+        assert_eq!(arr.get_id(0), 0);
+        assert_eq!(arr.get_id(1), 7);
+        assert_eq!(arr.get_id(2), 13);
+        assert_eq!(arr.get_id(3), 666);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        let mut arr = dc_array_t::new(7);
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/lot.rs

@@ -1,245 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use crate::message::MessageState;
-use crate::qr::Qr;
-use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Debug)]
-pub enum Lot {
-    Summary(Summary),
-    Qr(Qr),
-    Error(String),
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
-            },
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::FprOk { .. } => None,
-                Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
-                Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-            },
-            Self::Error(err) => Some(err),
-        }
-    }
-
-    pub fn get_text2(&self) -> Option<Cow<str>> {
-        match self {
-            Self::Summary(summary) => Some(summary.truncated_text(160)),
-            Self::Qr(_) => None,
-            Self::Error(_) => None,
-        }
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => Meaning::None,
-                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
-                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
-                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
-            },
-            Self::Qr(_qr) => Meaning::None,
-            Self::Error(_err) => Meaning::None,
-        }
-    }
-
-    pub fn get_state(&self) -> LotState {
-        match self {
-            Self::Summary(summary) => summary.state.into(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
-                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
-                Qr::FprOk { .. } => LotState::QrFprOk,
-                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
-                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
-                Qr::Account { .. } => LotState::QrAccount,
-                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Addr { .. } => LotState::QrAddr,
-                Qr::Url { .. } => LotState::QrUrl,
-                Qr::Text { .. } => LotState::QrText,
-                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
-                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
-                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
-                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
-            },
-            Self::Error(_err) => LotState::QrError,
-        }
-    }
-
-    pub fn get_id(&self) -> u32 {
-        match self {
-            Self::Summary(_) => Default::default(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => contact_id.to_u32(),
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
-                Qr::FprWithoutAddr { .. } => Default::default(),
-                Qr::Account { .. } => Default::default(),
-                Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
-                Qr::Url { .. } => Default::default(),
-                Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
-            },
-            Self::Error(_) => Default::default(),
-        }
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        match self {
-            Self::Summary(summary) => summary.timestamp,
-            Self::Qr(_) => Default::default(),
-            Self::Error(_) => Default::default(),
-        }
-    }
-}
-
-#[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// text1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        Lot::Summary(summary)
-    }
-}
-
-impl From<Qr> for Lot {
-    fn from(qr: Qr) -> Self {
-        Lot::Qr(qr)
-    }
-}
-
-// Make it easy to convert errors into the final `Lot`.
-impl From<Error> for Lot {
-    fn from(error: Error) -> Self {
-        Lot::Error(error.to_string())
-    }
-}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -102,9 +105,8 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         })
     }
 
@@ -120,9 +122,8 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         }),
         None => Err(CStringError::NotUnicode),
     }
@@ -167,20 +168,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +184,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name
@@ -250,7 +239,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] slice can not outlive
+/// Because this returns a reference the [Path] silce can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1"
-quote = "1"
+syn = "1.0.13"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -8,40 +8,33 @@ use quote::quote;
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 
-#[proc_macro_derive(ToSql)]
-pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(Sqlx)]
+pub fn sqlx_derive(input: TokenStream) -> TokenStream {
     let ast: syn::DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
     let gen = quote! {
-        impl rusqlite::types::ToSql for #name {
-            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-                let num = *self as i64;
-                let value = rusqlite::types::Value::Integer(num);
-                let output = rusqlite::types::ToSqlOutput::Owned(value);
-                std::result::Result::Ok(output)
-            }
-        }
-    };
-    gen.into()
-}
-
-#[proc_macro_derive(FromSql)]
-pub fn from_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::FromSql for #name {
-            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-                let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
+        impl<'q> sqlx::encode::Encode<'q, sqlx::sqlite::Sqlite> for #name {
+            fn encode_by_ref(&self, buf: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
+                num_traits::ToPrimitive::to_i32(self).expect("invalid type").encode(buf)
             }
         }
+
+
+        impl<'de> sqlx::decode::Decode<'de, sqlx::sqlite::Sqlite> for #name {
+            fn decode(value: sqlx::sqlite::SqliteValueRef) -> std::result::Result<Self, Box<dyn std::error::Error + 'static + Send + Sync>> {
+                let raw: i32 = sqlx::decode::Decode::decode(value)?;
+
+                Ok(num_traits::FromPrimitive::from_i32(raw).unwrap_or_default())
+            }
+        }
+
+        impl sqlx::types::Type<sqlx::sqlite::Sqlite> for #name {
+            fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
+                <i32 as sqlx::types::Type<_>>::type_info()
+            }
+        }
+
     };
     gen.into()
 }

draft/aeap-mvp.rst

@@ -1,33 +0,0 @@
-AEAP MVP
-========
-
-Changes to the UIs
-------------------
-
-- The secondary self addresses (see below) are shown in the UI, but not editable.
-
-- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
-
-Changes in the core
--------------------
-
-- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
-
-- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
-
-  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
-
-  - The key stays the same.
-
-- No changes for 1:1 chats, there simply is a new one
-
-- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
-  
-  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
-
-  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
-
-Other
------
-
-- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/group-sync.rst

@@ -5,95 +5,69 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-1. Alice and Bob are in a two-member group
+- Alice and Bob are in a two-member group
 
-2. Then Alice adds Carol, while concurrently Bob adds Doris
+- Alice adds Carol, concurrently Bob adds Doris
 
-Right now, the group has inconsistent memberships:
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
 
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
 
 
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
 
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
 
 
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
 
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
 
-For brevity let's abbreviate "group membership modification" as **GMM**.
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
 
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
+Notes:
 
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
+- mechanism works for both encrypted and unencrypted add/del messages
 
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
 
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
+- multiple member-added/removed messages can be attached in a single
+  correction message
 
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
 
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
 
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
 
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
   b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -109,3 +83,44 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

draft/msgwork_new_imap_jobs.rst

@@ -1,66 +0,0 @@
-
-simplify/streamline mark-seen/delete/move/send-mdn job handling
----------------------------------------------------------------
-
-Idea: Introduce a new "msgwork" sql table that looks very
-much like the jobs table but has a primary key "msgid"
-and no job id and no foreign-id anymore. This opens up
-bulk-processing by looking at the whole table and combining
-flag-setting to reduce imap-roundtrips and select-folder calls.
-
-Concretely, these IMAP jobs:
-
-    DeleteMsgOnImap
-    MarkseenMsgOnImap
-    MoveMsg
-
-Would be replaced by a few per-message columns in the new msgwork table:
-
-- needs_mark_seen: (bool) message shall be marked as seen on imap
-- needs_to_move: (bool) message should be moved to mvbox_folder
-- deletion_time: (target_time or 0) message shall be deleted at specified time
-- needs_send_mdn: (bool) MDN shall be sent
-
-The various places that currently add the (replaced) jobs
-would now add/modify the respective message record in the message-work table.
-
-Looking at a single message-work entry conceptually looks like this::
-
-    if msg.server_uid==0:
-        return RetryLater  # nothing can be done without server_uid
-
-    if msg.deletion_time > current_time:
-        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
-        clear(needs_deletion)
-        clear(mark_seen)
-
-    if needs_mark_seen:
-        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
-        clear(needs_mark_seen)
-
-    if needs_send_mdn:
-        schedule_smtp_send_mdn(msg)
-        clear(needs_send_mdn)
-
-    if any_flag_set():
-        retrylater
-    # remove msgwork entry from table
-
-
-Notes/Questions:
-
-- it's unclear how much we need per-message retry-time tracking/backoff
-
-- drafting bulk processing algo is useful before
-  going for the implementation, i.e. including select_folder calls etc.
-
-- maybe it's better to not have bools for the flags but
-
-  0 (no change)
-  1 (set the imap flag)
-  2 (clear the imap flag)
-
-  and design such that we can cover all imap flags.
-
-- It might not be neccessary to keep needs_send_mdn state in this table
-  if this can be decided rather when we succeed with mark_seen/mark_delete.

draft/webxdc-dev-reference.md

@@ -1,185 +0,0 @@
-# Webxdc Developer Reference
-
-## Webxdc File Format
-
-- a **Webxdc app** is a **ZIP-file** with the extension `.xdc`
-- the ZIP-file must use the default compression methods as of RFC 1950,
-  this is "Deflate" or "Store"
-- the ZIP-file must contain at least the file `index.html`
-- if the Webxdc app is started, `index.html` is opened in a restricted webview
-  that allow accessing resources only from the ZIP-file
-
-
-## Webxdc API
-
-There are some additional APIs available once `webxdc.js` is included
-(the file will be provided by the concrete implementations,
-no need to add `webxdc.js` to your ZIP-file):
-
-```html
-<script src="webxdc.js"></script>
-```
-
-### sendUpdate()
-
-```js
-window.webxdc.sendUpdate(update, descr);
-```
-
-Webxdc apps are usually shared in a chat and run independently on each peer.
-To get a shared state, the peers use `sendUpdate()` to send updates to each other.
-
-- `update`: an object with the following properties:  
-    - `update.payload`: any javascript primitive, array or object.
-    - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
-       use this option sparingly to not spam the chat.
-    - `update.summary`: optional, short text, shown beside app icon;
-       it is recommended to use some aggregated value,  eg. "8 votes", "Highscore: 123"
-
-- `descr`: short, human-readable description what this update is about.
-  this is shown eg. as a fallback text in an email program.
-
-All peers, including the sending one,
-will receive the update by the callback given to `setUpdateListener()`.
-
-There are situations where the user cannot send messages to a chat,
-eg. if the webxdc instance comes as a contact request or if the user has left a group.
-In these cases, you can still call `sendUpdate()`,
-however, the update won't be sent to other peers
-and you won't get the update by `setUpdateListener()`.
-
-
-### setUpdateListener()
-
-```js
-let promise = window.webxdc.setUpdateListener((update) => {}, serial);
-```
-
-With `setUpdateListener()` you define a callback that receives the updates
-sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
-The `serial` specifies the last serial that you know about (defaults to 0). 
-The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
-
-Each `update` which is passed to the callback comes with the following properties: 
-
-- `update.payload`: equals the payload given to `sendUpdate()`
-
-- `update.serial`: the serial number of this update.
-  Serials are larger `0` and newer serials have higher numbers.
-  There may be gaps in the serials
-  and it is not guaranteed that the next serial is exactly incremented by one.
-
-- `update.max_serial`: the maximum serial currently known.
-  If `max_serial` equals `serial` this update is the last update (until new network messages arrive).
-
-- `update.info`: optional, short, informational message (see `send_update`) 
-
-- `update.summary`: optional, short text, shown beside app icon (see `send_update`)
-
-
-### selfAddr
-
-```js
-window.webxdc.selfAddr
-```
-
-Property with the peer's own address.
-This is esp. useful if you want to differ between different peers -
-just send the address along with the payload,
-and, if needed, compare the payload addresses against selfAddr() later on.
-
-
-### selfName
-
-```js
-window.webxdc.selfName
-```
-
-Property with the peer's own name.
-This is name chosen by the user in their settings,
-if there is nothing set, that defaults to the peer's address.
-
-
-## manifest.toml
-
-If the ZIP-file contains a `manifest.toml` in its root directory,
-some basic information are read and used from there.
-
-the `manifest.toml` has the following format
-
-```toml
-name = "My App Name"
-```
-
-- **name** - The name of the app.
-  If no name is set or if there is no manifest, the filename is used as the app name.
-
-
-## App Icon
-
-If the ZIP-root contains an `icon.png` or `icon.jpg`,
-these files are used as the icon for the app.
-The icon should be a square at reasonable width/height;
-round corners etc. will be added by the implementations as needed.
-If no icon is set, a default icon will be used.
-
-
-## Webxdc Examples
-
-The following example shows an input field and  every input is show on all peers.
-
-```html
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8"/>
-    <script src="webxdc.js"></script>
-  </head>
-  <body>
-    <input id="input" type="text"/>
-    <a href="" onclick="sendMsg(); return false;">Send</a>
-    <p id="output"></p>
-    <script>
-    
-      function sendMsg() {
-        msg = document.getElementById("input").value;
-        window.webxdc.sendUpdate({payload: msg}, 'Someone typed "'+msg+'".');
-      }
-    
-      function receiveUpdate(update) {
-        document.getElementById('output').innerHTML += update.payload + "<br>";
-      }
-    
-      window.webxdc.setUpdateListener(receiveUpdate, 0);
-    </script>
-  </body>
-</html>
-```
-
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
-You can also use that repository as a template for your own app -
-just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/main.rs

@@ -19,50 +19,50 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::Event;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
-fn receive_event(event: EventType) {
+fn receive_event(event: Event) {
     let yellow = Color::Yellow.normal();
     match event {
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        EventType::SmtpConnected(msg) => {
+        Event::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        EventType::ImapConnected(msg) => {
+        Event::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        EventType::SmtpMessageSent(msg) => {
+        Event::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
             warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        Event::Error(msg) => {
             error!("{}", msg);
         }
-        EventType::ErrorSelfNotInGroup(msg) => {
+        Event::ErrorNetwork(msg) => {
+            error!("[NETWORK] msg={}", msg);
+        }
+        Event::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        EventType::MsgsChanged { chat_id, msg_id } => {
+        Event::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,44 +71,34 @@ fn receive_event(event: EventType) {
                 ))
             );
         }
-        EventType::ContactsChanged(_) => {
+        Event::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        EventType::LocationChanged(contact) => {
+        Event::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        EventType::ConfigureProgress { progress, comment } => {
-            if let Some(comment) = comment {
-                info!(
-                    "{}",
-                    yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
-                    ))
-                );
-            } else {
-                info!(
-                    "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-                );
-            }
+        Event::ConfigureProgress(progress) => {
+            info!(
+                "{}",
+                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+            );
         }
-        EventType::ImexProgress(progress) => {
+        Event::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        EventType::ImexFileWritten(file) => {
+        Event::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        EventType::ChatModified(chat) => {
+        Event::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -156,7 +146,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
     "set",
     "get",
@@ -164,19 +154,18 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 36] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "createchatbymsg",
     "creategroup",
-    "createbroadcast",
-    "createprotected",
+    "createverified",
     "addmember",
     "removemember",
     "groupname",
@@ -189,51 +178,37 @@ const CHAT_COMMANDS: [&str; 36] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
-    "sendsyncmsg",
-    "sendupdate",
-    "videochat",
     "draft",
     "listmedia",
     "archive",
     "unarchive",
     "pin",
     "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
-    "resend",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
-    "download",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
-const MISC_COMMANDS: [&str; 11] = [
+const MISC_COMMANDS: [&str; 10] = [
     "getqr",
-    "getqrsvg",
     "getbadqr",
     "checkqr",
-    "joinqr",
+    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -243,9 +218,7 @@ const MISC_COMMANDS: [&str; 11] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -267,10 +240,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -291,19 +265,18 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event.typ);
+            receive_event(event);
         }
     });
 
@@ -316,7 +289,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
+    let (reader_s, reader_r) = async_std::sync::channel(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -325,21 +298,21 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
 
         loop {
             let p = "> ";
-            let readline = rl.readline(p);
+            let readline = rl.readline(&p);
 
             match readline {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    async_std::task::block_on(reader_s.send(line));
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -398,9 +371,9 @@ async fn handle_cmd(
             ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -416,39 +389,27 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
-            if !qr.is_empty() {
-                if arg0 == "getbadqr" && qr.len() > 40 {
-                    qr.replace_range(12..22, "0000000000")
-                }
-                println!("{}", qr);
-                let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
-            }
-        }
-        "getqrsvg" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            match get_securejoin_qr_svg(&ctx, group).await {
-                Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get QR code svg: {}", err);
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
+                if !qr.is_empty() {
+                    if arg0 == "getbadqr" && qr.len() > 40 {
+                        qr.replace_range(12..22, "0000000000")
+                    }
+                    println!("{}", qr);
+                    let output = Command::new("qrencode")
+                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                        .output()
+                        .expect("failed to execute process");
+                    io::stdout().write_all(&output.stdout).unwrap();
+                    io::stderr().write_all(&output.stderr).unwrap();
                 }
             }
         }
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                dc_join_securejoin(&ctx, arg1).await;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -1,25 +1,25 @@
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::Event;
 
-fn cb(event: EventType) {
+fn cb(event: Event) {
     match event {
-        EventType::ConfigureProgress { progress, .. } => {
+        Event::ConfigureProgress(progress) => {
             log::info!("progress: {}", progress);
         }
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             log::info!("{}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new("FakeOs".into(), dbfile.into())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event.typ);
+            cb(event);
         }
     });
 
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

proptest-regressions/dc_tools.txt

@@ -7,5 +7,3 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
-cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
-cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/CHANGELOG

@@ -1,29 +1,6 @@
-1.51.0
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
-1.44.0
-------
-
-- fix Chat.get_mute_duration()
-
-1.40.1
+0.900.0 (DRAFT)
 ---------------
 
-- emit "ac_member_removed" event (with 'actor' being the removed contact) 
-  for when a user leaves a group.
-
-- fix create_contact(addr) when addr is the self-contact. 
-
-
-1.40.0
----------------
-
-- uses latest 1.40+ Delta Chat core
-
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -33,7 +10,6 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
-
 0.800.0
 -------
 

python/README.rst

@@ -7,14 +7,76 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -41,79 +103,6 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Running tests
-=============
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-This will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _livetests:
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
-
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
-
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
-
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
-
-Each test run creates new accounts.
-
-
-.. _sourceinstall:
-
-Installing bindings from source (Updated: July 2020)
-=========================================================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   python install_python_bindings.py
-
-The core compilation and bindings building might take a while,
-depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
 Code examples
 =============
 
@@ -143,7 +132,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -152,9 +141,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/doc/_templates/globaltoc.html

@@ -9,7 +9,8 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
+  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/group_tracking.py

@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, actor, message):
+    def ac_member_added(self, chat, contact, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, actor, message):
+    def ac_member_removed(self, chat, contact, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1, = acfactory.get_online_accounts(1)
+    ac1 = acfactory.get_one_online_account()
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -42,7 +42,7 @@ def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
     botproc = acfactory.run_bot_process(group_tracking, ffi=False)
 
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
         *ac_configure_completed*
@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_added {}*
+    """.format(contact3.addr))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -17,14 +17,11 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-
     if target == 'release':
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
-
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -1,26 +0,0 @@
-[mypy]
-
-[mypy-py.*]
-ignore_missing_imports = True
-
-[mypy-deltachat.capi.*]
-ignore_missing_imports = True
-
-[mypy-pluggy.*]
-ignore_missing_imports = True
-
-[mypy-cffi.*]
-ignore_missing_imports = True
-
-[mypy-imapclient.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-_pytest.*]
-ignore_missing_imports = True
-
-[mypy-imap_tools.*]
-ignore_missing_imports = True
-

python/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,14 +8,17 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
+        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
+        use_scm_version = {
+            "root": "..",
+            "relative_to": __file__,
+            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
+        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
-        setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
-        ],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ import sys
 
 from . import capi, const, hookspec # noqa
 from .capi import ffi  # noqa
-from .account import Account, get_core_info  # noqa
+from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa
@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name in dir(const):
+        for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[getattr(const, name)] = name
+                _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -75,8 +75,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/_build.py

@@ -8,9 +8,11 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import types
+from os.path import abspath
+from os.path import dirname as dn
 
 import cffi
-import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,31 +21,35 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = {}
+    flags = types.SimpleNamespace()
     if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
 
 
 def extract_functions(flags):
@@ -64,7 +70,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -100,7 +106,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
+                       include_dirs=flags.incs,
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -139,13 +145,9 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
-                | DC_IMEX
-                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -163,8 +165,11 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()
@@ -178,7 +183,10 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        **flags,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/account.py

@@ -8,43 +8,19 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
     """ Account is missing `addr` and `mail_pw` config values. """
 
 
-def get_core_info():
-    """ get some system info. """
-    from tempfile import NamedTemporaryFile
-
-    with NamedTemporaryFile() as path:
-        path.close()
-        return get_dc_info_as_dict(ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
-            lib.dc_context_unref,
-        ))
-
-
-def get_dc_info_as_dict(dc_context):
-    lines = from_dc_charpointer(lib.dc_get_info(dc_context))
-    info_dict = {}
-    for line in lines.split("\n"):
-        if not line.strip():
-            continue
-        key, value = line.split("=", 1)
-        info_dict[key.lower()] = value
-    return info_dict
-
-
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
@@ -52,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -82,17 +58,14 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self) -> None:
+    def disable_logging(self):
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self) -> None:
+    def enable_logging(self):
         """ re-enable logging. """
         self._logging = True
 
-    def __repr__(self):
-        return "<Account path={}>".format(self.db_path)
-
     # def __del__(self):
     #    self.shutdown()
 
@@ -100,44 +73,35 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name: str) -> None:
+    def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self) -> Dict[str, str]:
+    def get_info(self):
         """ return dictionary of built config parameters. """
-        return get_dc_info_as_dict(self._dc_context)
+        lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
+        d = {}
+        for line in lines.split("\n"):
+            if not line.strip():
+                continue
+            key, value = line.split("=", 1)
+            d[key.lower()] = value
+        return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
-    def set_stock_translation(self, id: int, string: str) -> None:
+    def set_stock_translation(self, id, string):
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        bytestring = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name: str, value: Optional[str]) -> None:
+    def set_config(self, name, value):
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -145,18 +109,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
+        name = name.encode("utf8")
+        if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
-        if isinstance(value, (int, bool)):
-            value = str(int(value))
         if value is not None:
-            valuebytes = value.encode("utf8")
+            value = value.encode("utf8")
         else:
-            valuebytes = ffi.NULL
-        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
+            value = ffi.NULL
+        lib.dc_set_config(self._dc_context, name, value)
 
-    def get_config(self, name: str) -> str:
+    def get_config(self, name):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -165,12 +127,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, namebytes)
+        name = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, name)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
+    def _preconfigure_keypair(self, addr, public, secret):
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -182,7 +144,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs: Dict[str, Any]) -> None:
+    def update_config(self, kwargs):
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -190,9 +152,9 @@ class Account(object):
         :returns: None
         """
         for key, value in kwargs.items():
-            self.set_config(key, value)
+            self.set_config(key, str(value))
 
-    def is_configured(self) -> bool:
+    def is_configured(self):
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -200,13 +162,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def is_open(self) -> bool:
-        """Determine if account is open
-
-        :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
-
-    def set_avatar(self, img_path: Optional[str]) -> None:
+    def set_avatar(self, img_path):
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -218,18 +174,31 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self) -> None:
+    def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir) -> Optional[str]:
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
+    def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        return from_optional_dc_charpointer(res)
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
-    def get_blobdir(self) -> str:
+    def get_blobdir(self):
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -237,17 +206,17 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self) -> Contact:
+    def get_self_contact(self):
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
-        """create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        Calling this method will always result in the same
+        Calling this method will always resulut in the same
         underlying contact id.  If there already is a Contact
         with that e-mail address, it is unblocked and its display
         `name` is updated if specified.
@@ -256,19 +225,6 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
-
-    def get_contact(self, obj) -> Optional[Contact]:
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -284,9 +240,16 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return (name, addr)
+        return self._create_contact(addr, name)
 
-    def delete_contact(self, contact: Contact) -> bool:
+    def _create_contact(self, addr, name):
+        addr = as_dc_charpointer(addr)
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
+        return Contact(self, contact_id)
+
+    def delete_contact(self, contact):
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -297,40 +260,23 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
+    def get_contact_by_addr(self, email):
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
-        return None
 
-    def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
         :param contact_id: integer id of this contact.
-        :returns: :class:`deltachat.contact.Contact` instance.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self) -> List[Contact]:
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
-    def get_contacts(
-        self,
-        query: Optional[str] = None,
-        with_self: bool = False,
-        only_verified: bool = False,
-    ) -> List[Contact]:
-        """get a (filtered) list of contacts.
+    def get_contacts(self, query=None, with_self=False, only_verified=False):
+        """ get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -350,7 +296,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self) -> Generator[Message, None, None]:
+    def get_fresh_messages(self):
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -358,17 +304,15 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj) -> Chat:
+    def create_chat(self, obj):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(
-        self,
-        name: str,
-        contacts: Optional[List[Contact]] = None,
-        verified: bool = False,
-    ) -> Chat:
-        """create a new group chat object.
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+
+    def create_group_chat(self, name, contacts=None, verified=False):
+        """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -384,7 +328,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self) -> List[Chat]:
+    def get_chats(self):
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -401,17 +345,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self) -> Chat:
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -423,21 +367,19 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
+    def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            if isinstance(msg, Message):
-                arr.append(msg.id)
-            else:
-                arr.append(msg)
+            msg = getattr(msg, "id", msg)
+            arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
+    def forward_messages(self, messages, chat):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -447,7 +389,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages):
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -461,23 +403,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account has to be stopped; call stop_io() if necessary.
+        Note that the account does not have to be started.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -487,7 +429,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
+        self._import(path, imex_cmd=2)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -495,28 +437,27 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-
-    def initiate_key_transfer(self) -> str:
+    def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self) -> str:
+    def get_setup_contact_qr(self):
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -566,9 +507,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -590,8 +529,6 @@ class Account(object):
         """ add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
-        if name and self._pm.has_plugin(name=name):
-            self._pm.unregister(name=name)
         self._pm.register(plugin, name=name)
         self._pm.check_pending()
         return plugin
@@ -611,15 +548,6 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self) -> str:
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 
@@ -630,9 +558,6 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
-        If you are using this from a test, you may want to call
-        wait_all_initial_fetches() afterwards.
-
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
@@ -642,34 +567,12 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def maybe_network(self):
-        """This function should be called when there is a hint
-        that the network is available again,
-        e.g. as a response to system event reporting network availability.
-        The library will try to send pending messages out immediately.
-
-        Moreover, to have a reliable state
-        when the app comes to foreground with network available,
-        it may be reasonable to call the function also at that moment.
-
-        It is okay to call the function unconditionally when there is
-        network available, however, calling the function
-        _without_ having network may interfere with the backoff algorithm
-        and will led to let the jobs fail faster, with fewer retries
-        and may avoid messages being sent out.
-
-        Finally, if the context was created by the dc_accounts_t account manager
-        (currently not implemented in the Python bindings),
-        use dc_accounts_maybe_network() instead of this function
-        """
-        lib.dc_maybe_network(self._dc_context)
-
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
+    def configure(self):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert self.is_configured() == reconfigure
+        assert not self.is_configured()
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -677,43 +580,40 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self) -> None:
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
+
+    def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self) -> None:
+    def stop_io(self):
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        self.log("dc_stop_io (stop core IO scheduler)")
-        lib.dc_stop_io(self._dc_context)
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
 
-    def shutdown(self) -> None:
+    def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:
             return
 
-        # mark the event thread for shutdown (latest on next incoming event)
-        self._event_thread.mark_shutdown()
-
-        # stop_io also causes an info event which will wake up
-        # the EventThread's inner loop and let it notice the shutdown marker.
         self.stop_io()
 
-        self.log("wait for event thread to finish")
-        try:
-            self._event_thread.wait(timeout=5)
-        except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
-
-        if self._event_thread.is_alive():
-            self.log("WARN: event thread did not terminate yet, ignoring.")
-
-        self.log("remove dc_context references, making the Account unusable")
+        self.log("remove dc_context references")
+        # the dc_context_unref triggers get_next_event to return ffi.NULL
+        # which in turns makes the event thread finish execution
         self._dc_context = None
 
+        self.log("wait for event thread to finish")
+        self._event_thread.wait()
+
         self._shutdown_event.set()
 
         hook = hookspec.Global._get_plugin_manager().hook

python/src/deltachat/chat.py

@@ -3,13 +3,12 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
-from typing import Optional
 
 
 class Chat(object):
@@ -18,20 +17,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
+    def __init__(self, account, id):
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other):
         return not (self == other)
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -41,7 +40,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self) -> None:
+    def delete(self):
         """Delete this chat and all its messages.
 
         Note:
@@ -51,37 +50,32 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self) -> None:
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self) -> None:
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self) -> bool:
+    def is_group(self):
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
 
-    def is_muted(self) -> bool:
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
+    def is_muted(self):
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -91,38 +85,30 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self) -> bool:
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+    def is_verified(self):
+        """ return True if this chat is a verified group.
 
-        :returns: True if the chat is writable, False otherwise
+        :returns: True if chat is verified, False otherwise.
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return lib.dc_chat_is_verified(self._dc_chat)
 
-    def is_protected(self) -> bool:
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
-
-    def get_name(self) -> Optional[str]:
+    def get_name(self):
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name: str) -> bool:
+    def set_name(self, name):
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: True on success, False otherwise
+        :returns: None
         """
         name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
-    def mute(self, duration: Optional[int] = None) -> None:
+    def mute(self, duration=None):
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -136,7 +122,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self) -> None:
+    def unmute(self):
         """ unmutes the chat
 
         :returns: None
@@ -145,45 +131,22 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self) -> int:
+    def get_mute_duration(self):
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
-    def get_ephemeral_timer(self) -> int:
-        """ get ephemeral timer.
-
-        :returns: ephemeral timer value in seconds
-        """
-        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
-
-    def set_ephemeral_timer(self, timer: int) -> bool:
-        """ set ephemeral timer.
-
-        :param: timer value in seconds
-
-        :returns: True on success, False otherwise
-        """
-        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
-
-    def get_type(self) -> int:
+    def get_type(self):
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self) -> Optional[str]:
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
-    def get_join_qr(self) -> Optional[str]:
+    def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -195,7 +158,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg: Message) -> Message:
+    def send_msg(self, msg):
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -263,19 +226,17 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """ create a new prepared message.
 
         :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
+        :returns: :class:`deltachat.message.Message` instance.
         """
         msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -389,7 +350,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.get_contact(obj)
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -513,24 +474,18 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
-                ),
-                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
-        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -1,13 +1,7 @@
-from typing import Any, List
-
 from .capi import lib
 
 
-def __getattr__(name: str) -> Any:
+for name in dir(lib):
     if name.startswith("DC_"):
-        return getattr(lib, name)
-    return globals()[name]
-
-
-def __dir__() -> List[str]:
-    return sorted(name for name in dir(lib) if name.startswith("DC_"))
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -1,12 +1,10 @@
 """ Contact object. """
 
-from datetime import date, datetime, timezone
-from typing import Optional
-
-from . import const, props
-from .capi import ffi, lib
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
 from .chat import Chat
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer
+from . import const
 
 
 class Contact(object):
@@ -37,60 +35,35 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self) -> str:
+    def addr(self):
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self) -> str:
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
-    @props.with_doc
-    def last_seen(self) -> date:
-        """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
-    def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, block)
-
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self) -> Optional[str]:
+    def get_profile_image(self):
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        return from_optional_dc_charpointer(dc_res)
-
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
 
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.

python/src/deltachat/cutil.py

@@ -1,9 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
-
-T = TypeVar('T')
+from datetime import datetime
 
 
 def as_dc_charpointer(obj):
@@ -14,28 +11,20 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
+def iter_array(dc_array_t, constructor):
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj) -> str:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    raise ValueError
-
-
-def from_optional_dc_charpointer(obj) -> Optional[str]:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    return None
+def from_dc_charpointer(obj):
+    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:
-    def __init__(self, dc_lot) -> None:
+    def __init__(self, dc_lot):
         self._dc_lot = dc_lot
 
-    def id(self) -> int:
+    def id(self):
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -54,4 +43,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)

python/src/deltachat/direct_imap.py

@@ -4,71 +4,82 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
+import email
 import ssl
 import pathlib
-from contextlib import contextmanager
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
-import imaplib
-from deltachat import const, Account
-from typing import List
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
 
 
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
 class DirectImap:
-    def __init__(self, account: Account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
         self.connect()
 
     def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
+        ssl_context = ssl.create_default_context()
 
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
         try:
             self.conn.logout()
-        except (OSError, imaplib.IMAP4.abort):
+        except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
-    def create_folder(self, foldername):
-        try:
-            self.conn.folder.create(foldername)
-        except errors.MailboxFolderCreateError as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
-    def select_folder(self, foldername: str) -> tuple:
+    def select_folder(self, foldername):
         assert not self._idling
-        return self.conn.folder.set(foldername)
+        return self.conn.select_folder(foldername)
 
-    def select_config_folder(self, config_name: str):
+    def select_config_folder(self, config_name):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -77,38 +88,58 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self) -> List[str]:
+    def list_folders(self):
         """ return list of all existing folder names"""
         assert not self._idling
-        return [folder.name for folder in self.conn.folder.list()]
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
 
-    def delete(self, uid_list: str, expunge=True):
+    def delete(self, range, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.set_flags(range, [DELETED])
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self) -> List[MailMessage]:
+    def get_all_messages(self):
         assert not self._idling
-        return [mail for mail in self.conn.fetch()]
+        return self.conn.fetch(ALL, [FLAGS])
 
-    def get_unread_messages(self) -> List[str]:
+    def get_unread_messages(self):
         assert not self._idling
-        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
+            res = self.conn.set_flags(messages, [SEEN])
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self) -> int:
+    def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()
@@ -128,84 +159,55 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            for msg in self.conn.fetch(mark_seen=False):
-                body = getattr(msg.obj, "text", None)
-                if not body:
-                    body = getattr(msg.obj, "html", None)
-                if not body:
-                    log("Message", msg.uid, "has empty body")
-                    continue
-
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(msg.uid))
-                fn.write_bytes(body)
-                log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
 
         print(stream.getvalue(), file=logfile)
 
-    @contextmanager
-    def idle(self):
-        """ return Idle ContextManager. """
-        idle_manager = IdleManager(self)
-        try:
-            yield idle_manager
-        finally:
-            idle_manager.done()
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
 
-    def append(self, folder: str, msg: str):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
-        """
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
-
-    def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]
-
-
-class IdleManager:
-    def __init__(self, direct_imap):
-        self.direct_imap = direct_imap
-        self.log = direct_imap.account.log
-        # fetch latest messages before starting idle so that it only
-        # returns messages that arrive anew
-        self.direct_imap.conn.fetch("1:*")
-        self.direct_imap.conn.idle.start()
-
-    def check(self, timeout=None) -> List[bytes]:
+    def idle_check(self, terminate=False):
         """ (blocking) wait for next idle message from server. """
-        self.log("imap-direct: calling idle_check")
-        res = self.direct_imap.conn.idle.poll(timeout=timeout)
-        self.log("imap-direct: idle_check returned {!r}".format(res))
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
         return res
 
-    def wait_for_new_message(self, timeout=None) -> bytes:
-        while 1:
-            for item in self.check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
-                    return item
-
-    def wait_for_seen(self, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
         """
         while 1:
-            for item in self.check(timeout=timeout):
-                if FETCH in item:
-                    self.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        return int(item.split(b' ')[1])
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
 
-    def done(self):
+    def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
-        res = self.direct_imap.conn.idle.stop()
-        return res
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -1,10 +1,6 @@
 import threading
-import sys
-import traceback
 import time
-import io
 import re
-import os
 from queue import Queue, Empty
 
 import deltachat
@@ -12,11 +8,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_optional_dc_charpointer
+from .cutil import from_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name: str, data1, data2):
+    def __init__(self, name, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -32,20 +28,18 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, init_time=None) -> None:
+    def __init__(self, account):
         self.account = account
-        self.logid = logid or self.account.get_config("displayname")
-        if init_time is None:
-            init_time = time.time()
-        self.init_time = init_time
+        self.logid = self.account.get_config("displayname")
+        self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
+    def ac_process_ffi_event(self, ffi_event):
         self.account.log(str(ffi_event))
 
     @account_hookimpl
     def ac_log_line(self, message):
-        t = threading.current_thread()
+        t = threading.currentThread()
         tname = getattr(t, "name", t)
         if tname == "MainThread":
             tname = "MAIN"
@@ -54,15 +48,6 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-
-        if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
-            if message.startswith("DC_EVENT_WARNING"):
-                s = WARN + s + ENDC
-            if message.startswith("DC_EVENT_ERROR"):
-                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -74,7 +59,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent):
+    def ac_process_ffi_event(self, ffi_event):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -101,48 +86,13 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex: str) -> FFIEvent:
-        rex = re.compile(regex)
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.search(ev.data2):
+            if rex.match(ev.data2):
                 return ev
 
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -161,15 +111,6 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_idle_inbox_ready(self):
-        """Has to be called after start_io() to wait for fetch_existing_msgs to run
-        so that new messages are not mistaken for old ones:
-        - ac1 and ac2 are created
-        - ac1 sends a message to ac2
-        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("INBOX: Idle entering")
-
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
@@ -181,7 +122,6 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
-        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -195,11 +135,10 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         super(EventThread, self).__init__(name="events")
-        self.daemon = True
-        self._marked_for_shutdown = False
+        self.setDaemon(True)
         self.start()
 
     @contextmanager
@@ -208,17 +147,14 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self) -> None:
-        self._marked_for_shutdown = True
-
-    def wait(self, timeout=None) -> None:
+    def wait(self):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join(timeout=timeout)
+        self.join()
 
-    def run(self) -> None:
+    def run(self):
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -228,9 +164,9 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while not self._marked_for_shutdown:
+        while 1:
             event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL or self._marked_for_shutdown:
+            if event == ffi.NULL:
                 break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
@@ -238,31 +174,23 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
             lib.dc_event_unref(event)
             ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            with self.swallow_and_log_exception("ac_process_ffi_event {}".format(ffi_event)):
+            try:
                 self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-            for name, kwargs in self._map_ffi_event(ffi_event):
-                hook = getattr(self.account._pm.hook, name)
-                info = "call {} kwargs={} failed".format(name, kwargs)
-                with self.swallow_and_log_exception(info):
+                for name, kwargs in self._map_ffi_event(ffi_event):
+                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
+                    hook = getattr(self.account._pm.hook, name)
                     hook(**kwargs)
+            except Exception:
+                if self.account._dc_context is not None:
+                    raise
 
-    @contextmanager
-    def swallow_and_log_exception(self, info):
-        try:
-            yield
-        except Exception as ex:
-            logfile = io.StringIO()
-            traceback.print_exception(*sys.exc_info(), file=logfile)
-            self.account.log("{}\nException {}\nTraceback:\n{}"
-                             .format(info, ex, logfile.getvalue()))
-
-    def _map_ffi_event(self, ffi_event: FFIEvent):
+    def _map_ffi_event(self, ffi_event):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are generally not allowed to block/last long as this
+    Hooks are not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,6 +31,10 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -43,7 +47,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):
@@ -51,37 +55,19 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
-
-        :param message: Message that was just delivered.
-        """
+        """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
-
-        :param chat: Chat which was modified.
-        """
+        """ Chat was created or modified regarding membership, avatar, title. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
-
-        :param chat: Chat where contact was added.
-        :param contact: Contact that was added.
-        :param actor: Who added the contact (None if it was our self-addr)
-        :param message: The original system message that reports the addition.
-        """
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
-
-        :param chat: Chat where contact was removed.
-        :param contact: Contact that was removed.
-        :param actor: Who removed the contact (None if it was our self-addr)
-        :param message: The original system message that reports the removal.
-        """
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
 
 
 class Global:

python/src/deltachat/message.py

@@ -1,13 +1,11 @@
 """ The Message object. """
 
 import os
-import re
 from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import datetime
 
 
 class Message(object):
@@ -22,8 +20,8 @@ class Message(object):
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        self.id = lib.dc_msg_get_id(dc_msg)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
@@ -47,13 +45,9 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param: view_type is "text", "audio", "video", "file"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
@@ -62,21 +56,19 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a contact request
+        If the message is a deaddrop contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.chat.accept()
-        return self.chat
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
-
-    @props.with_doc
-    def text(self) -> str:
+    def text(self):
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -84,23 +76,6 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self) -> str:
-        """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -114,13 +89,12 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self) -> str:
+    def basename(self):
         """basename of the attachment if it exists, otherwise empty string. """
-        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self) -> str:
+    def filemime(self):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -132,7 +106,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self) -> str:
+    def get_setupcodebegin(self):
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -140,15 +114,11 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self) -> str:
+    def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -172,7 +142,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def time_received(self):
@@ -182,52 +152,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @props.with_doc
-    def ephemeral_timer(self):
-        """Ephemeral timer in seconds
-
-        :returns: timer in seconds or None if there is no timer
-        """
-        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
-        if timer:
-            return timer
-
-    @props.with_doc
-    def ephemeral_timestamp(self):
-        """UTC time when the message will be deleted.
-
-        :returns: naive datetime.datetime() object or None if the timer is not started.
-        """
-        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
-        if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
-
-    @property
-    def quoted_text(self) -> Optional[str]:
-        """Text inside the quote
-
-        :returns: Quoted text"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
-
-    @property
-    def quote(self):
-        """Quote getter
-
-        :returns: Quoted message, if found in the database"""
-        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
-        if msg:
-            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
-
-    @quote.setter
-    def quote(self, quoted_message):
-        """Quote setter"""
-        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
-
-    def force_plaintext(self) -> None:
-        """Force the message to be sent in plain text."""
-        lib.dc_msg_force_plaintext(self._dc_msg)
+            return datetime.utcfromtimestamp(ts)
 
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
@@ -245,11 +170,6 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
-    @property
-    def error(self) -> Optional[str]:
-        """Error message"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
@@ -260,20 +180,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self) -> Optional[str]:
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_optional_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -386,10 +292,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -410,22 +312,21 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    const.DC_MSG_TEXT: 'text',
+    const.DC_MSG_IMAGE: 'image',
+    const.DC_MSG_GIF: 'gif',
+    const.DC_MSG_AUDIO: 'audio',
+    const.DC_MSG_VIDEO: 'video',
+    const.DC_MSG_FILE: 'file'
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
-    if code is not None:
-        return code
+    for code, value in _view_type_mapping.items():
+        if value == view_type_name:
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
 
 
 #
@@ -435,43 +336,20 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if not res:
-            return
-        action, affected, actor = res
-        affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
-        return "ac_member_" + res[0], d
-
-
-def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
-    if m:
-        text = m.group(1)
-    text = text.rstrip(".")
-    return text.strip()
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
 
 
 def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
-
-    returns a (action, affected, actor) triple """
-
     # Member Me (x@y) removed by a@b.
-    # Member x@y added by a@b
-    # Member With space (tmp1@x.org) removed by tmp2@x.org.
-    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-    # Group left by some one (tmp1@x.org).
-    # Group left by tmp1@x.org.
+    # Member x@y removed by a@b
     text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
-    if m:
-        affected, action, actor = m.groups()
-        return action, extract_addr(affected), extract_addr(actor)
-    if text.startswith("group left by "):
-        addr = extract_addr(text[13:])
-        if addr:
-            return "removed", addr, addr
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr) -> None:
+    def __init__(self, account, addr):
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self) -> str:
+    def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self) -> str:
+    def get_before_login_hints(self):
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
-    def status(self) -> int:
+    def status(self):
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -8,43 +8,30 @@ import threading
 import fnmatch
 import time
 import weakref
-from queue import Queue
-from typing import List, Callable
+import tempfile
 
 import pytest
 import requests
-import py
 
-from . import Account, const, account_hookimpl, get_core_info
+from . import Account, const
+from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
 
 def pytest_addoption(parser):
-    group = parser.getgroup("deltachat testplugin options")
-    group.addoption(
+    parser.addoption(
         "--liveconfig", action="store", default=None,
         help="a file with >=2 lines where each line "
              "contains NAME=VALUE config settings for one account"
     )
-    group.addoption(
+    parser.addoption(
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    group.addoption(
-        "--strict-tls", action="store_true",
-        help="Never accept invalid TLS certificates for test accounts",
-    )
-    group.addoption(
-        "--extra-info", action="store_true",
-        help="show more info on failures (imap server state, config)"
-    )
-    group.addoption(
-        "--debug-setup", action="store_true",
-        help="show events during configure and start io phases of online accounts"
-    )
 
 
 def pytest_configure(config):
@@ -99,12 +86,9 @@ def pytest_configure(config):
 
         @pytest.hookimpl(hookwrapper=True)
         def pytest_runtest_teardown(self, item):
-            logging = item.config.getoption("--extra-info")
-            if logging:
-                self.enable_logging(item)
+            self.enable_logging(item)
             yield
-            if logging:
-                self.disable_logging(item)
+            self.disable_logging(item)
 
     la = LoggingAspect()
     config.pluginmanager.register(la)
@@ -112,12 +96,20 @@ def pytest_configure(config):
 
 
 def pytest_report_header(config, startdir):
-    info = get_core_info()
-    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
-        info['deltachat_core_version'],
-        info['sqlite_version'],
-        info['journal_mode'],
-    )]
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+         info['journal_mode'],
+     )])
 
     cfg = config.option.liveconfig
     if cfg:
@@ -129,105 +121,61 @@ def pytest_report_header(config, startdir):
     return summary
 
 
-@pytest.fixture(scope="session")
-def testprocess(request):
-    return TestProcess(pytestconfig=request.config)
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
 
 
-class TestProcess:
-    """ A pytest session-scoped instance to help with managing "live" account configurations.
-    """
-    def __init__(self, pytestconfig):
-        self.pytestconfig = pytestconfig
-        self._addr2files = {}
-        self._configlist = []
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
 
-    def get_liveconfig_producer(self):
-        """ provide live account configs, cached on a per-test-process scope
-        so that test functions can re-use already known live configs.
-        Depending on the --liveconfig option this comes from
-        a HTTP provider or a file with a line specifying each accounts config.
-        """
-        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
-        if not liveconfig_opt:
-            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
-
-        if not liveconfig_opt.startswith("http"):
-            for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith('#'):
-                    d = {}
-                    for part in line.split():
-                        name, value = part.split("=")
-                        d[name] = value
-                    self._configlist.append(d)
-
-            yield from iter(self._configlist)
-        else:
-            MAX_LIVE_CREATED_ACCOUNTS = 10
-            for index in range(MAX_LIVE_CREATED_ACCOUNTS):
-                try:
-                    yield self._configlist[index]
-                except IndexError:
-                    res = requests.post(liveconfig_opt)
-                    if res.status_code != 200:
-                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
-                                    index, res.status_code, res.text))
-                    d = res.json()
-                    config = dict(addr=d["email"], mail_pw=d["password"])
-                    print("newtmpuser {}: addr={}".format(index, config["addr"]))
-                    self._configlist.append(config)
-                    yield config
-            pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS))
-
-    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
-        db_target_path = py.path.local(db_target_path)
-        assert not db_target_path.exists()
-
-        print("checking cache for", cache_addr)
+    def get(self, index):
         try:
-            filescache = self._addr2files[cache_addr]
-        except KeyError:
-            print("CACHE FAIL for", cache_addr)
-            return False
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
         else:
-            print("CACHE HIT for", cache_addr)
-            targetdir = db_target_path.dirpath()
-            write_dict_to_dir(filescache, targetdir)
-            return True
-
-    def cache_maybe_store_configured_db_files(self, acc):
-        addr = acc.get_config("addr")
-        assert acc.is_configured()
-        # don't overwrite existing entries
-        if addr not in self._addr2files:
-            print("storing cache for", addr)
-            basedir = py.path.local(acc.get_blobdir()).dirpath()
-            self._addr2files[addr] = create_dict_from_files_in_path(basedir)
-            return True
-
-
-def create_dict_from_files_in_path(path):
-    base = py.path.local(path)
-    cachedict = {}
-    for path in base.visit(fil=py.path.local.isfile):
-        cachedict[path.relto(base)] = path.read_binary()
-    return cachedict
-
-
-def write_dict_to_dir(dic, target_dir):
-    assert dic
-    target_dir = py.path.local(target_dir)
-    for relpath, content in dic.items():
-        path = target_dir.join(relpath)
-        if not path.dirpath().exists():
-            path.dirpath().ensure(dir=1)
-        path.write_binary(content)
+            return SessionLiveConfigFromFile(liveconfig_opt)
 
 
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self) -> None:
+        def __init__(self):
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -254,417 +202,243 @@ def data(request):
     return Data()
 
 
-class ACSetup:
-    """ accounts setup helper to deal with multiple configure-process
-    and io & imap initialization phases. From tests, use the higher level
-    public ACFactory methods instead of its private helper class.
-    """
-    CONFIGURING = "CONFIGURING"
-    CONFIGURED = "CONFIGURED"
-    IDLEREADY = "IDLEREADY"
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
-    def __init__(self, testprocess, init_time):
-        self._configured_events = Queue()
-        self._account2state = {}
-        self._imap_cleaned = set()
-        self.testprocess = testprocess
-        self.init_time = init_time
-
-    def log(self, *args):
-        print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
-
-    def add_configured(self, account):
-        """ add an already configured account. """
-        assert account.is_configured()
-        self._account2state[account] = self.CONFIGURED
-        self.log("added already configured account", account, account.get_config("addr"))
-        return
-
-    def start_configure(self, account, reconfigure=False):
-        """ add an account and start its configure process. """
-
-        if reconfigure:
-            assert account.is_configured()
-
-        class PendingTracker:
-            @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
-
-        account.add_account_plugin(PendingTracker(), name="pending_tracker")
-        self._account2state[account] = self.CONFIGURING
-        account.configure(reconfigure=reconfigure)
-        self.log("started {}configure on".format("re-" if reconfigure else ""), account)
-
-    def wait_one_configured(self, account):
-        """ wait until this account has successfully configured. """
-        if self._account2state[account] == self.CONFIGURING:
-            while 1:
-                acc = self._pop_config_success()
-                if acc == account:
-                    break
-            self.init_imap(acc)
-            self.init_logging(acc)
-            acc._evtracker.consume_events()
-
-    def wait_all_configured(self):
-        """ Wait for all unconfigured accounts to become finished. """
-        while self.CONFIGURING in self._account2state.values():
-            self._pop_config_success()
-
-    def bring_online(self):
-        """ Wait for all accounts to become ready to receive messages.
-
-        This will initialize logging, start IO and the direct_imap attribute
-        for each account which either is CONFIGURED already or which is CONFIGURING
-        and successfully completing the configuration process.
-        """
-        print("wait_all_configured finds accounts=", self._account2state)
-        for acc, state in self._account2state.items():
-            if state == self.CONFIGURED:
-                self._onconfigure_start_io(acc)
-                self._account2state[acc] = self.IDLEREADY
-
-        while self.CONFIGURING in self._account2state.values():
-            acc = self._pop_config_success()
-            self._onconfigure_start_io(acc)
-            self._account2state[acc] = self.IDLEREADY
-        print("finished, account2state", self._account2state)
-
-    def _pop_config_success(self):
-        acc, success = self._configured_events.get()
-        if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
-        self._account2state[acc] = self.CONFIGURED
-        return acc
-
-    def _onconfigure_start_io(self, acc):
-        self.init_imap(acc)
-        self.init_logging(acc)
-        acc.start_io()
-        print(acc._logid, "waiting for inbox IDLE to become ready")
-        acc._evtracker.wait_idle_inbox_ready()
-        acc._evtracker.consume_events()
-        acc.log("inbox IDLE ready")
-
-    def init_logging(self, acc):
-        """ idempotent function for initializing logging (will replace existing logger). """
-        logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
-        acc.add_account_plugin(logger, name="logger-" + acc._logid)
-
-    def init_imap(self, acc):
-        """ initialize direct_imap and cleanup server state. """
-        from deltachat.direct_imap import DirectImap
-
-        assert acc.is_configured()
-        if not hasattr(acc, "direct_imap"):
-            acc.direct_imap = DirectImap(acc)
-        addr = acc.get_config("addr")
-        if addr not in self._imap_cleaned:
-            imap = acc.direct_imap
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete("1:*", expunge=True)
-                else:
-                    imap.conn.folder.delete(folder)
-            acc.log("imap cleaned for addr {}".format(addr))
-            self._imap_cleaned.add(addr)
-
-
-class ACFactory:
-    _finalizers: List[Callable[[], None]]
-    _accounts: List[Account]
-
-    def __init__(self, request, testprocess, tmpdir, data) -> None:
-        self.init_time = time.time()
-        self.tmpdir = tmpdir
-        self.pytestconfig = request.config
-        self.data = data
-        self.testprocess = testprocess
-        self._liveconfig_producer = testprocess.get_liveconfig_producer()
-
-        self._finalizers = []
-        self._accounts = []
-        self._acsetup = ACSetup(testprocess, self.init_time)
-        self._preconfigured_keys = ["alice", "bob", "charlie",
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self._accounts = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-        self.set_logging_default(False)
-        request.addfinalizer(self.finalize)
+            self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
-    def log(self, *args):
-        print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args)
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
 
-    def finalize(self):
-        while self._finalizers:
-            fin = self._finalizers.pop()
-            fin()
-
-        while self._accounts:
-            acc = self._accounts.pop()
-            if acc is not None:
-                imap = getattr(acc, "direct_imap", None)
-                if imap is not None:
-                    imap.shutdown()
-                    del acc.direct_imap
+            while self._accounts:
+                acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
-    def get_next_liveconfig(self):
-        """ Base function to get functional online configurations
-        where we can make valid SMTP and IMAP connections with.
-        """
-        configdict = next(self._liveconfig_producer).copy()
-        if "e2ee_enabled" not in configdict:
-            configdict["e2ee_enabled"] = "1"
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path, logging=self._logging)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac))
+            self._accounts.append(ac)
+            return ac
+
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
 
-        if self.pytestconfig.getoption("--strict-tls"):
             # Enable strict certificate checks for online accounts
             configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
             configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
-        assert "addr" in configdict and "mail_pw" in configdict
-        return configdict
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            return ac, dict(configdict)
 
-    def _get_cached_account_copy(self, addr):
-        if addr in self.testprocess._addr2files:
-            return self._getaccount(addr)
-
-    def get_unconfigured_account(self):
-        return self._getaccount()
-
-    def _getaccount(self, try_cache_addr=None):
-        logid = "ac{}".format(len(self._accounts) + 1)
-
-        # we need to use fixed database basename for maybe_cache_* functions to work
-        path = self.tmpdir.mkdir(logid).join("dc.db")
-        if try_cache_addr:
-            self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
-        ac = Account(path.strpath, logging=self._logging)
-        ac._logid = logid  # later instantiated FFIEventLogger needs this
-        ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-        if self.pytestconfig.getoption("--debug-setup"):
-            self._acsetup.init_logging(ac)
-        self._accounts.append(ac)
-        return ac
-
-    def set_logging_default(self, logging):
-        self._logging = bool(logging)
-
-    def remove_preconfigured_keys(self):
-        self._preconfigured_keys = []
-
-    def _preconfigure_key(self, account, addr):
-        # Only set a preconfigured key if we haven't used it yet for another account.
-        try:
-            keyname = self._preconfigured_keys.pop(0)
-        except IndexError:
-            pass
-        else:
-            fname_pub = self.data.read_path("key/{name}-public.asc".format(name=keyname))
-            fname_sec = self.data.read_path("key/{name}-secret.asc".format(name=keyname))
-            if fname_pub and fname_sec:
-                account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                return True
-            else:
-                print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-    def get_pseudo_configured_account(self):
-        # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
-        acname = ac._logid
-        addr = "{}@offline.org".format(acname)
-        ac.update_config(dict(
-            addr=addr, displayname=acname, mail_pw="123",
-            configured_addr=addr, configured_mail_pw="123",
-            configured="1",
-        ))
-        self._preconfigure_key(ac, addr)
-        self._acsetup.init_logging(ac)
-        return ac
-
-    # XXX deprecate the next function?
-    def new_online_configuring_account(self, cache=False, **kwargs):
-        configdict = self.get_next_liveconfig()
-        configdict.update(kwargs)
-        return self._setup_online_configuring_account(configdict, cache=cache)
-
-    def get_online_second_device(self, ac1, **kwargs):
-        ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
-        if ac2 is None:
-            # some tests setup the primary account without causing caching.
-            configdict = kwargs.copy()
-            configdict["addr"] = ac1.get_config("addr")
-            configdict["mail_pw"] = ac1.get_config("mail_pw")
-            ac2 = self._setup_online_configuring_account(configdict, cache=False)
-        elif kwargs:
-            ac2.update_config(kwargs)
-            self._acsetup.add_configured(ac2)
-
-        self.bring_accounts_online()
-        return ac2
-
-    def get_online_multidevice_setup(self, copied=True):
-        """ Provide two accounts. The second uses the same credentials
-        and if copied is True, also the same database and blobs.
-        You can use copy=False to get a typical configuration where
-        a user unsuspectingly sets up a second device and expects it to
-        "just work" not knowing that an export/import is required.
-        """
-        ac1, = self.get_online_accounts(1)
-        if copied:
-            ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
-            self._acsetup.add_configured(ac2)
-        else:
-            config2 = dict(addr=ac1.get_config("addr"), mail_pw=ac1.get_config("mail_pw"))
-            ac2 = self._setup_online_configuring_account(config2, cache=False)
-        self.bring_accounts_online()
-        return ac1, ac2
-
-    def get_online_devnull_email(self):
-        return self.get_next_liveconfig()["addr"]
-
-    def _setup_online_configuring_account(self, configdict, cache=False):
-        ac = self._get_cached_account_copy(configdict["addr"]) if cache else None
-        if ac is not None:
-            # make sure we consume a preconfig key, as if we had created a fresh account
-            self._preconfigured_keys.pop(0)
-            self._acsetup.add_configured(ac)
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac._configtracker = ac.configure()
             return ac
-        ac = self.prepare_account_from_liveconfig(configdict)
-        self._acsetup.start_configure(ac)
-        return ac
 
-    def prepare_account_from_liveconfig(self, configdict):
-        ac = self.get_unconfigured_account()
-        assert "addr" in configdict and "mail_pw" in configdict, configdict
-        configdict.setdefault("bcc_self", False)
-        configdict.setdefault("mvbox_move", False)
-        configdict.setdefault("sentbox_watch", False)
-        ac.update_config(configdict)
-        self._preconfigure_key(ac, configdict["addr"])
-        return ac
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            self.wait_configure_and_start_io()
+            return ac1
 
-    def wait_configured(self, account):
-        """ Wait until the specified account has successfully completed configure. """
-        self._acsetup.wait_one_configured(account)
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            self.wait_configure_and_start_io()
+            return ac1, ac2
 
-    def bring_accounts_online(self):
-        print("bringing accounts online")
-        self._acsetup.bring_online()
-        print("all accounts online")
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io()
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
 
-    def get_online_configured_accounts(self, configlist):
-        accounts = [self.new_online_configuring_account(cache=True, **config)
-                    for config in configlist]
-        self._acsetup.wait_all_configured()
-        for acc in accounts:
-            self._acsetup.init_imap(acc)
-        return accounts
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            ac._configtracker = ac.configure()
+            return ac
 
-    def force_reconfigure(self, account):
-        self._acsetup.start_configure(account, reconfigure=True)
+        def wait_configure_and_start_io(self):
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
 
-    def get_online_accounts(self, num):
-        """ Return a list of configured and started Accounts.
+        def run_bot_process(self, module, ffi=True):
+            fn = module.__file__
 
-        This function creates plain online accounts and fill
-        a testprocess-scoped cache and re-use these plain accounts
-        on the next test function.
-        """
-        accounts = [self.new_online_configuring_account(cache=True) for i in range(num)]
-        self.bring_accounts_online()
-        for acc in accounts:
-            self.testprocess.cache_maybe_store_configured_db_files(acc)
-        return accounts
+            bot_ac, bot_cfg = self.get_online_config()
 
-    def run_bot_process(self, module, ffi=True):
-        fn = module.__file__
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
 
-        bot_cfg = self.get_next_liveconfig()
-        bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
+            args = [
+                sys.executable,
+                "-u",
+                fn,
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=0,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
 
-        # Forget ac as it will be opened by the bot subprocess
-        # but keep something in the list to not confuse account generation
-        self._accounts[self._accounts.index(bot_ac)] = None
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        args = [
-            sys.executable,
-            "-u",
-            fn,
-            "--email", bot_cfg["addr"],
-            "--password", bot_cfg["mail_pw"],
-            bot_ac.db_path,
-        ]
-        if ffi:
-            args.insert(-1, "--show-ffi")
-        print("$", " ".join(args))
-        popen = subprocess.Popen(
-            args=args,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-            bufsize=0,                 # line buffering
-            close_fds=True,            # close all FDs other than 0/1/2
-            universal_newlines=True    # give back text
-        )
-        bot = BotProcess(popen, addr=bot_cfg["addr"])
-        self._finalizers.append(bot.kill)
-        return bot
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
 
-    def dump_imap_summary(self, logfile):
-        for ac in self._accounts:
-            ac.dump_account_info(logfile=logfile)
-            imap = getattr(ac, "direct_imap", None)
-            if imap is not None:
-                imap.dump_imap_structures(self.tmpdir, logfile=logfile)
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
 
-    def get_accepted_chat(self, ac1: Account, ac2):
-        ac2.create_chat(ac1)
-        return ac1.create_chat(ac2)
-
-    def introduce_each_other(self, accounts, sending=True):
-        to_wait = []
-        for i, acc in enumerate(accounts):
-            for acc2 in accounts[i + 1:]:
-                chat = self.get_accepted_chat(acc, acc2)
-                if sending:
-                    chat.send_text("hi")
-                    to_wait.append(acc2)
-                    acc2.create_chat(acc).send_text("hi back")
-                    to_wait.append(acc)
-        for acc in to_wait:
-            acc.log("waiting for incoming message")
-            acc._evtracker.wait_next_incoming_message()
-
-
-@pytest.fixture
-def acfactory(request, tmpdir, testprocess, data):
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data)
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
     yield am
     if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testprocess.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
-    stdout_queue: queue.Queue
-
-    def __init__(self, popen, addr) -> None:
+    def __init__(self, popen, bot_cfg):
         self.popen = popen
-        self.addr = addr
+        self.addr = bot_cfg["addr"]
 
         # we read stdout as quickly as we can in a thread and make
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.daemon = True
+        t.setDaemon(1)
         t.start()
 
-    def _run_stdout_thread(self) -> None:
+    def _run_stdout_thread(self):
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -676,10 +450,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self) -> None:
+    def kill(self):
         self.popen.kill()
 
-    def wait(self, timeout=None) -> None:
+    def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -688,7 +462,7 @@ class BotProcess:
             print("+++FNMATCH:", next_pattern)
             ignored = []
             while 1:
-                line = self.stdout_queue.get()
+                line = self.stdout_queue.get(timeout=15)
                 if line is None:
                     if ignored:
                         print("BOT stdout terminated after these lines")
@@ -711,16 +485,13 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg: str) -> None:
+        def sec(self, msg):
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg: str) -> None:
+        def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg: str) -> None:
-            print("  " + msg)
-
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,16 +20,6 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
-
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []
@@ -90,11 +80,11 @@ class ConfigureTracker:
             if data1 is None or evdata == data1:
                 break
 
-    def wait_finish(self, timeout=None):
+    def wait_finish(self):
         """ wait until configure is completed.
 
         Raise Exception if Configure failed
         """
-        if not self._configure_events.get(timeout=timeout):
+        if not self._configure_events.get():
             content = "\n".join(map(str, self._ffi_events))
             raise ConfigureFailed(content)

python/tests/auditwheels.py

@@ -1,17 +1,15 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                 "--plat", "manylinux2014_x86_64"])

python/tests/bench_test_setup.py

@@ -1,20 +0,0 @@
-"""
-run with:
-
-     pytest -vv --durations=10 bench_empty.py
-
-to see timings of test setups.
-"""
-
-import pytest
-
-BENCH_NUM = 3
-
-
-class TestEmpty:
-    def test_prepare_setup_measurings(self, acfactory):
-        acfactory.get_online_accounts(BENCH_NUM)
-
-    @pytest.mark.parametrize("num", range(0, BENCH_NUM + 1))
-    def test_setup_online_accounts(self, acfactory, num):
-        acfactory.get_online_accounts(num)

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3)
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account

python/tests/test_0_complex_or_slow.py

@@ -1,462 +0,0 @@
-import sys
-import pytest
-
-
-class TestGroupStressTests:
-    def test_group_many_members_add_leave_remove(self, acfactory, lp):
-        accounts = acfactory.get_online_accounts(5)
-        acfactory.introduce_each_other(accounts)
-        ac1, ac5 = accounts.pop(), accounts.pop()
-
-        lp.sec("ac1: creating group chat with 3 other members")
-        chat = ac1.create_group_chat("title1", contacts=accounts)
-
-        lp.sec("ac1: send message to new group chat")
-        msg1 = chat.send_text("hello")
-        assert msg1.is_encrypted()
-        gossiped_timestamp = chat.get_summary()["gossiped_timestamp"]
-        assert gossiped_timestamp > 0
-
-        assert chat.num_contacts() == 3 + 1
-
-        lp.sec("ac2: checking that the chat arrived correctly")
-        ac2 = accounts[0]
-        msg2 = ac2._evtracker.wait_next_incoming_message()
-        assert msg2.text == "hello"
-        print("chat is", msg2.chat)
-        assert msg2.chat.num_contacts() == 4
-
-        lp.sec("ac3: checking that 'ac4' is a known contact")
-        ac3 = accounts[1]
-        msg3 = ac3._evtracker.wait_next_incoming_message()
-        assert msg3.text == "hello"
-        ac3_contacts = ac3.get_contacts()
-        assert len(ac3_contacts) == 4
-        ac4_contacts = ac3.get_contacts(query=accounts[2].get_config("addr"))
-        assert len(ac4_contacts) == 1
-
-        lp.sec("ac2: removing one contact")
-        to_remove = ac2.create_contact(accounts[-1])
-        msg2.chat.remove_contact(to_remove)
-
-        lp.sec("ac1: receiving system message about contact removal")
-        sysmsg = ac1._evtracker.wait_next_incoming_message()
-        assert to_remove.addr in sysmsg.text
-        assert sysmsg.chat.num_contacts() == 3
-
-        # Receiving message about removed contact does not reset gossip
-        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
-
-        lp.sec("ac1: sending another message to the chat")
-        chat.send_text("hello2")
-        msg = ac2._evtracker.wait_next_incoming_message()
-        assert msg.text == "hello2"
-        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
-
-        lp.sec("ac1: adding fifth member to the chat")
-        chat.add_contact(ac5)
-        # Adding contact to chat resets gossiped_timestamp
-        assert chat.get_summary()["gossiped_timestamp"] >= gossiped_timestamp
-
-        lp.sec("ac2: receiving system message about contact addition")
-        sysmsg = ac2._evtracker.wait_next_incoming_message()
-        assert ac5.get_config("configured_addr") in sysmsg.text
-        assert sysmsg.chat.num_contacts() == 4
-
-        lp.sec("ac5: waiting for message about addition to the chat")
-        sysmsg = ac5._evtracker.wait_next_incoming_message()
-        msg = sysmsg.chat.send_text("hello!")
-        # Message should be encrypted because keys of other members are gossiped
-        assert msg.is_encrypted()
-
-    def test_synchronize_member_list_on_group_rejoin(self, acfactory, lp):
-        """
-        Test that user recreates group member list when it joins the group again.
-        ac1 creates a group with two other accounts: ac2 and ac3
-        Then it removes ac2, removes ac3 and adds ac2 back.
-        ac2 did not see that ac3 is removed, so it should rebuild member list from scratch.
-        """
-        lp.sec("setting up accounts, accepted with each other")
-        accounts = acfactory.get_online_accounts(3)
-        acfactory.introduce_each_other(accounts)
-        ac1, ac2, ac3 = accounts
-
-        lp.sec("ac1: creating group chat with 2 other members")
-        chat = ac1.create_group_chat("title1", contacts=[ac2, ac3])
-        assert not chat.is_promoted()
-
-        lp.sec("ac1: send message to new group chat")
-        msg = chat.send_text("hello")
-        assert chat.is_promoted() and msg.is_encrypted()
-
-        assert chat.num_contacts() == 3
-
-        lp.sec("checking that the chat arrived correctly")
-        for ac in accounts[1:]:
-            msg = ac._evtracker.wait_next_incoming_message()
-            assert msg.text == "hello"
-            print("chat is", msg.chat)
-            assert msg.chat.num_contacts() == 3
-
-        lp.sec("ac1: removing ac2")
-        chat.remove_contact(ac2)
-
-        lp.sec("ac2: wait for a message about removal from the chat")
-        msg = ac2._evtracker.wait_next_incoming_message()
-
-        lp.sec("ac1: removing ac3")
-        chat.remove_contact(ac3)
-
-        lp.sec("ac1: adding ac2 back")
-        # Group is promoted, message is sent automatically
-        assert chat.is_promoted()
-        chat.add_contact(ac2)
-
-        lp.sec("ac2: check that ac3 is removed")
-        msg = ac2._evtracker.wait_next_incoming_message()
-
-        assert chat.num_contacts() == 2
-        assert msg.chat.num_contacts() == 2
-        acfactory.dump_imap_summary(sys.stdout)
-
-
-def test_qr_verified_group_and_chatting(acfactory, lp):
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
-    chat1 = ac1.create_group_chat("hello", verified=True)
-    assert chat1.is_protected()
-    qr = chat1.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    chat2 = ac2.qr_join_chat(qr)
-    assert chat2.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("ac2: read member added message")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.is_encrypted()
-    assert "added" in msg.text.lower()
-
-    lp.sec("ac1: send message")
-    msg_out = chat1.send_text("hello")
-    assert msg_out.is_encrypted()
-
-    lp.sec("ac2: read message and check it's verified chat")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "hello"
-    assert msg.chat.is_protected()
-    assert msg.is_encrypted()
-
-    lp.sec("ac2: send message and let ac1 read it")
-    chat2.send_text("world")
-    msg = ac1._evtracker.wait_next_incoming_message()
-    assert msg.text == "world"
-    assert msg.is_encrypted()
-
-    lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
-    qr = ac1.get_setup_contact_qr()
-
-    lp.sec("ac3: start QR-code based setup contact protocol")
-    ch = ac3.qr_setup_contact(qr)
-    assert ch.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("ac1: add ac3 to verified group")
-    chat1.add_contact(ac3)
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.is_encrypted()
-    assert msg.is_system_message()
-    assert not msg.error
-
-    lp.sec("ac2: send message and let ac3 read it")
-    chat2.send_text("hi")
-    # Skip system message about added member
-    ac3._evtracker.wait_next_incoming_message()
-    msg = ac3._evtracker.wait_next_incoming_message()
-    assert msg.text == "hi"
-    assert msg.is_encrypted()
-
-
-@pytest.mark.parametrize("mvbox_move", [False, True])
-def test_fetch_existing(acfactory, lp, mvbox_move):
-    """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
-    This way, we can already offer them some email addresses they can write to.
-
-    Also, the newest existing emails from each folder are fetched during onboarding.
-
-    Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
-
-    def assert_folders_configured(ac):
-        """There was a bug that scan_folders() set the configured folders to None under some circumstances.
-        So, check that they are still configured:"""
-        assert ac.get_config("configured_sentbox_folder") == "Sent"
-        if mvbox_move:
-            assert ac.get_config("configured_mvbox_folder")
-
-    ac1, ac2 = acfactory.get_online_configured_accounts([dict(mvbox_move=mvbox_move), dict()])
-    ac1.direct_imap.create_folder("Sent")
-    ac1.set_config("sentbox_watch", "1")
-
-    # We need to reconfigure to find the new "Sent" folder.
-    # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
-    # would also find the "Sent" folder, but it would be too late:
-    # The sentbox thread, started by `start_io()`, would have seen that there is no
-    # ConfiguredSentboxFolder and do nothing.
-    acfactory.force_reconfigure(ac1)
-    acfactory.bring_accounts_online()
-
-    assert_folders_configured(ac1)
-
-    assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
-    with ac1.direct_imap.idle() as idle1:
-        lp.sec("send out message with bcc to ourselves")
-        ac1.set_config("bcc_self", "1")
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-        chat.send_text("message text")
-        assert_folders_configured(ac1)
-
-        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
-        assert idle1.wait_for_seen()
-    assert_folders_configured(ac1)
-
-    lp.sec("create a cloned ac1 and fetch contact history during configure")
-    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
-    assert_folders_configured(ac1_clone)
-
-    lp.sec("check that ac2 contact was fetchted during configure")
-    ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
-    ac2_addr = ac2.get_config("addr")
-    assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
-    assert_folders_configured(ac1_clone)
-
-    lp.sec("check that messages changed events arrive for the correct message")
-    msg = ac1_clone._evtracker.wait_next_messages_changed()
-    assert msg.text == "message text"
-    assert_folders_configured(ac1)
-    assert_folders_configured(ac1_clone)
-
-
-def test_fetch_existing_msgs_group_and_single(acfactory, lp):
-    """There was a bug concerning fetch-existing-msgs:
-
-    A sent a message to you, adding you to a group. This created a contact request.
-    You wrote a message to A, creating a chat.
-    ...but the group stayed blocked.
-    So, after fetch-existing-msgs you have one contact request and one chat with the same person.
-
-    See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    lp.sec("receive a message")
-    ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
-    ac1._evtracker.wait_next_incoming_message()
-
-    lp.sec("send out message with bcc to ourselves")
-    with ac1.direct_imap.idle() as idle1:
-        ac1.set_config("bcc_self", "1")
-        ac1_ac2_chat = ac1.create_chat(ac2)
-        ac1_ac2_chat.send_text("outgoing, encrypted direct message, creating a chat")
-        # wait until the bcc_self message arrives
-        assert idle1.wait_for_seen()
-
-    lp.sec("Clone online account and let it fetch the existing messages")
-    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
-
-    chats = ac1_clone.get_chats()
-    assert len(chats) == 4  # two newly created chats + self-chat + device-chat
-    group_chat = [c for c in chats if c.get_name() == "group name"][0]
-    assert group_chat.is_group()
-    private_chat, = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
-    assert not private_chat.is_group()
-
-    group_messages = group_chat.get_messages()
-    assert len(group_messages) == 1
-    assert group_messages[0].text == "incoming, unencrypted group message"
-    private_messages = private_chat.get_messages()
-    assert len(private_messages) == 1
-    assert private_messages[0].text.startswith("outgoing, encrypted")
-
-
-def test_undecipherable_group(acfactory, lp):
-    """Test how group messages that cannot be decrypted are
-    handled.
-
-    Group name is encrypted and plaintext subject is set to "..." in
-    this case, so we should assign the messages to existing chat
-    instead of creating a new one. Since there is no existing group
-    chat, the messages should be assigned to 1-1 chat with the sender
-    of the message.
-    """
-
-    lp.sec("creating and configuring three accounts")
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    acfactory.introduce_each_other([ac1, ac2, ac3])
-
-    lp.sec("ac3 reinstalls DC and generates a new key")
-    ac3.stop_io()
-    acfactory.remove_preconfigured_keys()
-    ac4 = acfactory.get_online_second_device(ac3)
-    # Create contacts to make sure incoming messages are not treated as contact requests
-
-    lp.sec("ac1: creating group chat with 2 other members")
-    chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
-
-    lp.sec("ac1: send message to new group chat")
-    msg = chat.send_text("hello")
-
-    lp.sec("ac2: checking that the chat arrived correctly")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "hello"
-    assert msg.is_encrypted(), "Message is not encrypted"
-
-    # ac4 cannot decrypt the message.
-    # Error message should be assigned to the chat with ac1.
-    lp.sec("ac4: checking that message is assigned to the sender chat")
-    error_msg = ac4._evtracker.wait_next_incoming_message()
-    assert error_msg.error  # There is an error decrypting the message
-    assert error_msg.chat == chat41
-
-    lp.sec("ac2: sending a reply to the chat")
-    msg.chat.send_text("reply")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.text == "reply"
-    assert reply.is_encrypted(), "Reply is not encrypted"
-
-    lp.sec("ac4: checking that reply is assigned to ac2 chat")
-    error_reply = ac4._evtracker.wait_next_incoming_message()
-    assert error_reply.error  # There is an error decrypting the message
-    assert error_reply.chat == chat42
-
-    # Test that ac4 replies to error messages don't appear in the
-    # group chat on ac1 and ac2.
-    lp.sec("ac4: replying to ac1 and ac2")
-
-    # Otherwise reply becomes a contact request.
-    chat41.send_text("I can't decrypt your message, ac1!")
-    chat42.send_text("I can't decrypt your message, ac2!")
-
-    msg = ac1._evtracker.wait_next_incoming_message()
-    assert msg.error is None
-    assert msg.text == "I can't decrypt your message, ac1!"
-    assert msg.is_encrypted(), "Message is not encrypted"
-    assert msg.chat == ac1.create_chat(ac3)
-
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.error is None
-    assert msg.text == "I can't decrypt your message, ac2!"
-    assert msg.is_encrypted(), "Message is not encrypted"
-    assert msg.chat == ac2.create_chat(ac4)
-
-
-def test_ephemeral_timer(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    lp.sec("ac1: create chat with ac2")
-    chat1 = ac1.create_chat(ac2)
-    chat2 = ac2.create_chat(ac1)
-
-    lp.sec("ac1: set ephemeral timer to 60")
-    chat1.set_ephemeral_timer(60)
-
-    lp.sec("ac1: check that ephemeral timer is set for chat")
-    assert chat1.get_ephemeral_timer() == 60
-    chat1_summary = chat1.get_summary()
-    assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
-
-    lp.sec("ac2: receive system message about ephemeral timer modification")
-    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    system_message1 = ac2._evtracker.wait_next_incoming_message()
-    assert chat2.get_ephemeral_timer() == 60
-    assert system_message1.is_system_message()
-
-    # Disabled until markers are implemented
-    # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
-
-    lp.sec("ac2: send message to ac1")
-    sent_message = chat2.send_text("message")
-    assert sent_message.ephemeral_timer == 60
-    assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
-
-    # Timer is started immediately for sent messages
-    assert sent_message.ephemeral_timestamp is not None
-    assert "Expires: " in sent_message.get_message_info()
-
-    lp.sec("ac1: waiting for message from ac2")
-    text_message = ac1._evtracker.wait_next_incoming_message()
-    assert text_message.text == "message"
-    assert text_message.ephemeral_timer == 60
-    assert "Ephemeral timer: 60\n" in text_message.get_message_info()
-
-    # Timer should not start until message is displayed
-    assert text_message.ephemeral_timestamp is None
-    assert "Expires: " not in text_message.get_message_info()
-    text_message.mark_seen()
-    text_message = ac1.get_message_by_id(text_message.id)
-    assert text_message.ephemeral_timestamp is not None
-    assert "Expires: " in text_message.get_message_info()
-
-    lp.sec("ac2: set ephemeral timer to 0")
-    chat2.set_ephemeral_timer(0)
-    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-
-    lp.sec("ac1: receive system message about ephemeral timer modification")
-    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    system_message2 = ac1._evtracker.wait_next_incoming_message()
-    assert system_message2.ephemeral_timer is None
-    assert "Ephemeral timer: " not in system_message2.get_message_info()
-    assert chat1.get_ephemeral_timer() == 0
-
-
-def test_multidevice_sync_seen(acfactory, lp):
-    """Test that message marked as seen on one device is marked as seen on another."""
-    ac1, ac1_clone = acfactory.get_online_multidevice_setup()
-    ac2, = acfactory.get_online_accounts(1)
-
-    ac1.set_config("bcc_self", "1")
-    ac1_clone.set_config("bcc_self", "1")
-
-    ac1_chat = ac1.create_chat(ac2)
-    ac1_clone_chat = ac1_clone.create_chat(ac2)
-    ac2_chat = ac2.create_chat(ac1)
-
-    lp.sec("Send a message from ac2 to ac1 and check that it's 'fresh'")
-    ac2_chat.send_text("Hi")
-    ac1_message = ac1._evtracker.wait_next_incoming_message()
-    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
-    assert ac1_chat.count_fresh_messages() == 1
-    assert ac1_clone_chat.count_fresh_messages() == 1
-    assert ac1_message.is_in_fresh
-    assert ac1_clone_message.is_in_fresh
-
-    lp.sec("ac1 marks message as seen on the first device")
-    ac1.mark_seen_messages([ac1_message])
-    assert ac1_message.is_in_seen
-
-    lp.sec("ac1 clone detects that message is marked as seen")
-    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-    assert ev.data1 == ac1_clone_chat.id
-    assert ac1_clone_message.is_in_seen
-
-    lp.sec("Send an ephemeral message from ac2 to ac1")
-    ac2_chat.set_ephemeral_timer(60)
-    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    ac1._evtracker.wait_next_incoming_message()
-    ac1_clone._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    ac1_clone._evtracker.wait_next_incoming_message()
-
-    ac2_chat.send_text("Foobar")
-    ac1_message = ac1._evtracker.wait_next_incoming_message()
-    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
-    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
-    assert "Expires: " not in ac1_clone_message.get_message_info()
-    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
-    assert "Expires: " not in ac1_clone_message.get_message_info()
-
-    ac1.mark_seen_messages([ac1_message])
-    assert ac1_message.is_in_seen
-    assert "Expires: " in ac1_message.get_message_info()
-    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-    assert ev.data1 == ac1_clone_chat.id
-    assert ac1_clone_message.is_in_seen
-    # Test that the timer is started on the second device after synchronizing the seen status.
-    assert "Expires: " in ac1_clone_message.get_message_info()