parallelize tox-runs somewhat

.circleci/config.yml

@@ -0,0 +1,200 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
+jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - run:
+          name: Update submodules
+          command: git submodule update --init --recursive
+      - run:
+          name: Calculate dependencies
+          command: cargo generate-lockfile
+      - restore_cache:
+          keys:
+            - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo update
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo update
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
+  build_test_docs_wheel:
+    docker:
+      - image: deltachat/coredeps
+        environment:
+            TESTS: 1
+            DOCS: 1
+    working_directory: /mnt/crate
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: build docs, run tests and build wheels
+          command: ci_scripts/run-python.sh
+      - run:
+           name: copying docs and wheels to workspace
+           command: |
+             mkdir -p workspace/python
+             # cp -av docs workspace/c-docs
+             cp -av python/.docker-tox/wheelhouse workspace/
+             cp -av python/doc/_build/ workspace/py-docs
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: pyenv global 3.5.2
+      - run: ls -laR workspace
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy --all
+
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      - cargo_fetch
+      - build_test_docs_wheel:
+          requires:
+            - cargo_fetch
+      - upload_docs_wheels:
+          requires:
+            - build_test_docs_wheel
+      - rustfmt:
+          requires:
+            - cargo_fetch
+      - clippy:
+          requires:
+            - cargo_fetch
+
+      # Linux Desktop 64bit
+      - test_x86_64-unknown-linux-gnu:
+          requires:
+            - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -1,154 +0,0 @@
-name: Rust CI
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - staging
-      - trying
-
-jobs:
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: stable
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
-
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
-
-  build_and_test:
-    name: Build and test
-    strategy:
-      matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.54.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.54.0
-            python: false # Python bindings compilation on Windows is not supported.
-
-          # Minimum Supported Rust Version = 1.48.0
-          # This is the Debian "bullseye" release version of Rust.
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.48.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
-      with:
-        command: check
-        args: --all --bins --examples --tests --features repl
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,doc,py3

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -17,14 +16,9 @@ python/.tox
 *.egg-info
 __pycache__
 python/src/deltachat/capi*.so
-python/.venv/
 
 python/liveconfig*
 
 # ignore doxgen generated files
 deltachat-ffi/html
 deltachat-ffi/xml
-
-.rsynclist
-
-coverage/

CMakeLists.txt

@@ -1,42 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-
-find_program(CARGO cargo)
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-include(GNUInstallDirs)
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,89 +1,59 @@
 [package]
 name = "deltachat"
-version = "1.59.0"
+version = "1.0.0-alpha.4"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
 edition = "2018"
-license = "MPL-2.0"
+license = "MPL"
-
-[profile.dev]
-debug = 0
-
-[profile.release]
-lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
-
+libc = "0.2.51"
-ansi_term = { version = "0.12.1", optional = true }
+pgp = { version = "0.2", default-features = false }
-anyhow = "1.0.42"
+hex = "0.3.2"
-async-imap = { git = "https://github.com/async-email/async-imap" }
+sha2 = "0.8.0"
-async-native-tls = { version = "0.3.3" }
+rand = "0.6.5"
-async-smtp = { git = "https://github.com/async-email/async-smtp", rev="c8800625f7cf29f437143ac7e720ac2730a0962f" }
+phf = { git = "https://github.com/sfackler/rust-phf", rev = "0d00821", features = ["macros"] }
-async-std-resolver = "0.20.3"
+smallvec = "0.6.9"
-async-std = { version = "~1.9.0", features = ["unstable"] }
+reqwest = "0.9.15"
-async-tar = "0.3.0"
+num-derive = "0.2.5"
-async-trait = "0.1.50"
-backtrace = "0.3.59"
-base64 = "0.13"
-bitflags = "1.3.1"
-byteorder = "1.3.1"
-chrono = "0.4.6"
-dirs = { version = "3.0.2", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.1"
-futures = "0.3.16"
-hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7.0"
-itertools = "0.10.1"
-kamadak-exif = "0.5"
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.98"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13.5"
-native-tls = "0.2.3"
-num_cpus = "1.13.0"
-num-derive = "0.3.0"
 num-traits = "0.2.6"
-once_cell = "1.8.0"
+native-tls = "0.2.3"
+lettre = "0.9.0"
+imap = { git = "https://github.com/jonhoo/rust-imap", rev = "281d2eb8ab50dc656ceff2ae749ca5045f334e15" }
+mmime = { git = "https://github.com/dignifiedquire/mmime", rev = "bccd2c2" }
+base64 = "0.10"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7.0", default-features = false }
-pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.22.0"
-r2d2 = "0.8.9"
-r2d2_sqlite = "0.18.0"
-rand = "0.7.0"
-regex = "1.4.6"
-rusqlite = "0.25"
-rust-hsluv = "0.1.4"
-rustyline = { version = "8.2.0", optional = true }
-sanitize-filename = "0.3.0"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.7"
+serde_json = "1.0"
-sha2 = "0.9.5"
+chrono = "0.4.6"
-smallvec = "1.0.0"
+failure = "0.1.5"
-stop-token = "0.2.0"
+failure_derive = "0.1.5"
-strum = "0.21.0"
+# TODO: make optional
-strum_macros = "0.21.1"
+rustyline = "4.1.0"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+lazy_static = "1.4.0"
-thiserror = "1.0.26"
+regex = "1.1.6"
-toml = "0.5.6"
+rusqlite = { version = "0.20", features = ["bundled"] }
-url = "2.2.2"
+r2d2_sqlite = "0.12.0"
-uuid = { version = "0.8", features = ["serde", "v4"] }
+r2d2 = "0.8.5"
-humansize = "1.1.1"
+strum = "0.15.0"
+strum_macros = "0.15.0"
+thread-local-object = "0.1.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.8.0"
+image-meta = "0.1.0"
+quick-xml = "0.15.0"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+jetscii = "0.4.4"
+debug_stub_derive = "0.3.0"
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
-criterion = "0.3"
-futures-lite = "1.12.0"
-log = "0.4.11"
-pretty_assertions = "0.7.2"
-pretty_env_logger = "0.4.0"
-proptest = "1.0"
 tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.3.0"
+proptest = "0.9.4"
 
 [workspace]
 members = [
@@ -94,29 +64,14 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
-required-features = ["repl"]
 
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["repl"]
 
 
-[[bench]]
-name = "create_account"
-harness = false
-
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
 [features]
-default = ["vendored"]
+default = ["nightly", "ringbuf"]
-internals = []
+vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
 nightly = ["pgp/nightly"]
+ringbuf = ["pgp/ringbuf"]

LICENSE

@@ -2,6 +2,9 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
+NOTE that the files in the "libs" directory are copyrighted by third parties
+and come with their own respective licenses.
+
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -1,25 +1,26 @@
 # Delta Chat Rust
 
-> Deltachat-core written in Rust 
+> Project porting deltachat-core to rust
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
+
+Current commit on deltachat/deltachat-core: `12ef73c8e76185f9b78e844ea673025f56a959ab`.
 
 ## Installing Rust and Cargo
 
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
 
 ```
-$ curl https://sh.rustup.rs -sSf | sh
+curl https://sh.rustup.rs -sSf | sh
 ```
 
 ## Using the CLI client
 
-Compile and run Delta Chat Core command line utility, using `cargo`:
+Compile and run Delta Chat Core using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -79,16 +80,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -98,16 +89,6 @@ $ cargo test --all
 $ cargo build -p deltachat_ffi --release
 ```
 
-## Debugging environment variables 
-
-- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
-  printed
-
-- `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
-
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
-SMTP tracing in addition to info messages.
-
 ### Expensive tests
 
 Some tests are expensive and marked with `#[ignore]`, to run these
@@ -120,24 +101,9 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
+- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
-## Language bindings and frontend projects
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
-
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
-Language bindings are available for:
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
-
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-- [C](https://c.delta.chat)
-- [Node.js](https://www.npmjs.com/package/deltachat-node)
-- [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
-- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
-- **Java** and **Swift** (contained in the Android/iOS repos)
-
-The following "frontend" projects make use of the Rust-library
-or its language bindings:
-
-- [Android](https://github.com/deltachat/deltachat-android)
-- [iOS](https://github.com/deltachat/deltachat-ios)
-- [Desktop](https://github.com/deltachat/deltachat-desktop)
-- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
-- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
-- several **Bots**

Xargo.toml

@@ -0,0 +1,6 @@
+[dependencies.std]
+features = ["panic-unwind"]
+
+# if using `cargo test`
+[dependencies.test]
+stage = 1

appveyor.yml

@@ -0,0 +1,20 @@
+environment:
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init -yv --default-toolchain nightly-2019-07-10
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+  - rustc -vV
+  - cargo -vV
+  - cargo update
+
+build: false
+
+test_script:
+  - cargo test --release
+
+cache:
+  - target
+  - C:\Users\appveyor\.cargo\registry

assets/icon-device.svg

@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   inkscape:export-ydpi="409.60001"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-device.png"
-   version="1.0"
-   width="60"
-   height="60"
-   viewBox="0 0 45 45"
-   preserveAspectRatio="xMidYMid meet"
-   id="svg4344"
-   sodipodi:docname="icon-device.svg"
-   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
-  <defs
-     id="defs4348" />
-  <sodipodi:namedview
-     inkscape:snap-global="false"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     inkscape:document-rotation="0"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1600"
-     inkscape:window-height="1035"
-     id="namedview4346"
-     showgrid="false"
-     units="px"
-     inkscape:zoom="3.959798"
-     inkscape:cx="28.322498"
-     inkscape:cy="24.898474"
-     inkscape:window-x="45"
-     inkscape:window-y="23"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg4344" />
-  <metadata
-     id="metadata4336">
-Created by potrace 1.15, written by Peter Selinger 2001-2017
-<rdf:RDF>
-  <cc:Work
-     rdf:about="">
-    <dc:format>image/svg+xml</dc:format>
-    <dc:type
-       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-    <dc:title />
-  </cc:Work>
-</rdf:RDF>
-</metadata>
-  <rect
-     y="-4.4408921e-16"
-     x="0"
-     height="45"
-     width="45"
-     id="rect860"
-     style="opacity:1;fill:#76868b;fill-opacity:1;stroke-width:0.819271" />
-  <g
-     fill="#000000"
-     stroke="none"
-     style="fill:#ffffff;fill-opacity:1"
-     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
-     id="g4342">
-    <path
-       style="fill:#ffffff;fill-opacity:1"
-       d="m 8175,12765 c -703,-114 -1248,-608 -1387,-1258 -17,-82 -21,-136 -22,-277 0,-202 15,-307 70,-470 149,-446 499,-733 1009,-828 142,-26 465,-23 619,6 691,131 1201,609 1328,1244 31,158 31,417 0,565 -114,533 -482,889 -1038,1004 -133,27 -448,35 -579,14 z"
-       id="path4338"
-       inkscape:connector-curvature="0" />
-    <path
-       style="fill:#ffffff;fill-opacity:1"
-       d="m 7070,9203 c -212,-20 -275,-27 -397,-48 -691,-117 -1400,-444 -2038,-940 -182,-142 -328,-270 -585,-517 -595,-571 -911,-974 -927,-1181 -6,-76 11,-120 69,-184 75,-80 159,-108 245,-79 109,37 263,181 632,595 539,606 774,826 1035,969 135,75 231,105 341,106 82,1 94,-2 138,-27 116,-68 161,-209 122,-376 -9,-36 -349,-868 -757,-1850 -407,-982 -785,-1892 -838,-2021 -287,-694 -513,-1389 -615,-1889 -70,-342 -90,-683 -52,-874 88,-440 381,-703 882,-792 124,-23 401,-30 562,-16 783,69 1674,461 2561,1125 796,596 1492,1354 1607,1751 43,146 -33,308 -168,360 -61,23 -100,15 -173,-36 -105,-74 -202,-170 -539,-529 -515,-551 -762,-783 -982,-927 -251,-164 -437,-186 -543,-65 -56,64 -74,131 -67,247 13,179 91,434 249,815 135,324 1588,4102 1646,4280 106,325 151,561 159,826 9,281 -22,463 -112,652 -58,122 -114,199 -211,292 -245,233 -582,343 -1044,338 -91,-1 -181,-3 -200,-5 z"
-       id="path4340"
-       inkscape:connector-curvature="0" />
-  </g>
-</svg>

assets/icon-saved-messages.svg

@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   inkscape:export-ydpi="409.60001"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-filename="/home/kerle/test-icon.png"
-   version="1.0"
-   width="60"
-   height="60"
-   viewBox="0 0 45 45"
-   preserveAspectRatio="xMidYMid meet"
-   id="svg4344"
-   sodipodi:docname="icon-saved-messages.svg"
-   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
-  <defs
-     id="defs4348" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     inkscape:document-rotation="0"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1395"
-     inkscape:window-height="855"
-     id="namedview4346"
-     showgrid="false"
-     units="px"
-     inkscape:zoom="4"
-     inkscape:cx="29.308676"
-     inkscape:cy="49.03624"
-     inkscape:window-x="89"
-     inkscape:window-y="108"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg4344"
-     inkscape:lockguides="false" />
-  <metadata
-     id="metadata4336">
-Created by potrace 1.15, written by Peter Selinger 2001-2017
-<rdf:RDF>
-  <cc:Work
-     rdf:about="">
-    <dc:format>image/svg+xml</dc:format>
-    <dc:type
-       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-    <dc:title />
-  </cc:Work>
-</rdf:RDF>
-</metadata>
-  <rect
-     y="0"
-     x="0"
-     height="45"
-     width="45"
-     id="rect1420"
-     style="fill:#87aade;fill-opacity:1;stroke:none;stroke-width:0.968078" />
-  <path
-     id="rect846"
-     style="fill:#ffffff;stroke-width:0.58409804"
-     d="M 13.5,7.5 V 39 h 0.08654 L 22.533801,29.370239 31.482419,39 h 0.01758 V 7.5 Z m 9.004056,4.108698 1.879508,4.876388 5.039514,0.359779 -3.879358,3.363728 1.227764,5.095749 -4.276893,-2.796643 -4.280949,2.788618 1.237229,-5.093073 -3.873949,-3.371754 5.040866,-0.350417 z"
-     inkscape:connector-curvature="0" />
-</svg>

benches/contacts.rs

@@ -1,39 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, &book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/create_account.rs

@@ -1,26 +0,0 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::accounts::Accounts;
-use tempfile::tempdir;
-
-async fn create_accounts(n: u32) {
-    let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
-
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-
-    for expected_id in 2..n {
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, expected_id);
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,29 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -0,0 +1,52 @@
+
+# Continuous Integration Scripts for Delta Chat
+
+Continuous Integration is run through CircleCI
+but is largely independent of it. 
+
+
+## Generating docker containers for performing build step work 
+
+All tests, docs and wheel building is run in docker containers:
+
+- **coredeps/Dockerfile** specifies an image that contains all 
+  of Delta Chat's core dependencies as linkable libraries. 
+  It also serves to run python tests and build wheels 
+  (binary packages for Python). 
+
+- **doxygen/Dockerfile** specifies an image that contains
+  the doxygen tool which is used to generate C-docs. 
+
+To run tests locally you can pull existing images from "docker.io",
+the hub for sharing Docker images::
+
+    docker pull deltachat/coredeps
+    docker pull deltachat/doxygen 
+
+or you can build the docker images yourself locally
+to avoid the relatively large download:: 
+ 
+    cd ci_scripts  # where all CI things are 
+    docker build -t deltachat/coredeps docker-coredeps
+    docker build -t deltachat/doxygen docker-doxygen 
+
+## ci_run.sh (main entrypoint called by circle-ci)
+
+Once you have the docker images available 
+you can run python testing, documentation generation 
+and building binary wheels:: 
+
+    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
+    
+## ci_upload.sh (uploading artifacts on success) 
+
+- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- python wheels to `https://m.devpi.net/dc/<BRANCH>`
+  so that you install fully self-contained wheels like this:
+  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
+
+
+

ci_scripts/ci_upload.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+#DOXYDOCDIR=${1:?directory where doxygen docs to be found}
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+#rsync -avz \
+#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+#  "$DOXYDOCDIR/html/" \
+#  delta@py.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*.whl
+
+popd

ci_scripts/docker-coredeps/Dockerfile

@@ -1,21 +1,25 @@
-FROM quay.io/pypa/manylinux2014_aarch64
+FROM quay.io/pypa/manylinux1_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust
+# Install Rust nightly 
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
+
+# Install a recent Perl, needed to install OpenSSL
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+# Install OpenSSL
+ADD deps/build_openssl.sh /builder/build_openssl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
+

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.28.0
+PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION} 
+
+./Configure -de
+make
+make install

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x 
+
+# Install Rust 
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+#
+set -e -x
+
+# perform clean build of core and install 
+export TOXWORKDIR=.docker-tox
+
+# build core library
+
+cargo build --release -p deltachat_ffi
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+#
+# run python tests
+#
+
+if [ -n "$TESTS" ]; then 
+
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi

ci_scripts/run-python.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+
+    tox --workdir "$TOXWORKDIR" -e py37 
+    unset DCC_PY_LIVECONFIG
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/run-rust-test.sh

@@ -1,9 +1,8 @@
 #!/usr/bin/env bash
 
 set -ex
-shopt -s huponexit
 
-#export RUST_TEST_THREADS=1
+export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1
 export RUSTFLAGS='--deny warnings'
 export OPT="--target=$TARGET"
@@ -32,15 +31,12 @@ fi
 if [[ $NORUN == "1" ]]; then
     export CARGO_SUBCMD="build"
 else
-    export CARGO_SUBCMD="test --all"
+    export CARGO_SUBCMD="test"
     export OPT="${OPT} "
     export OPT_RELEASE="${OPT_RELEASE} "
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 
-# Run all the test configurations 
+# Run all the test configurations:
-# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
-unset RUSTC_WRAPPER
 $CARGO_CMD $CARGO_SUBCMD $OPT
-$CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED

contrib/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.59.0"
+version = "1.0.0-alpha.4"
 description = "Deltachat FFI"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
 edition = "2018"
 readme = "README.md"
-license = "MPL-2.0"
+license = "MIT OR Apache-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -16,17 +16,13 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-provider-overview = { git = "https://github.com/deltachat/provider-overview", rev = "366b41a7503973e4ffac3aa5173b419f2f03c211" }
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-serde_json = "1.0"
-async-std = "1.9.0"
-anyhow = "1.0.42"
-thiserror = "1.0.26"
-rand = "0.7.3"
 
 [features]
-default = ["vendored"]
+default = ["vendored", "nightly", "ringbuf"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
-
+ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/Doxyfile

@@ -236,6 +236,12 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/build.rs

@@ -19,17 +19,15 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
+libdir=${{prefix}}/lib
-includedir={includedir}
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/dc_array.rs

@@ -1,129 +0,0 @@
-use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
-use crate::location::Location;
-use crate::message::MsgId;
-
-/* * the structure behind dc_array_t */
-#[derive(Debug, Clone)]
-pub enum dc_array_t {
-    MsgIds(Vec<MsgId>),
-    Chat(Vec<ChatItem>),
-    Locations(Vec<Location>),
-    Uint(Vec<u32>),
-}
-
-impl dc_array_t {
-    pub(crate) fn get_id(&self, index: usize) -> u32 {
-        match self {
-            Self::MsgIds(array) => array[index].to_u32(),
-            Self::Chat(array) => match array[index] {
-                ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
-                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
-            },
-            Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index],
-        }
-    }
-
-    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(array) => array.get(index).and_then(|item| match item {
-                ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
-                ChatItem::DayMarker { timestamp } => Some(*timestamp),
-            }),
-            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(_) => None,
-            Self::Locations(array) => array
-                .get(index)
-                .and_then(|location| location.marker.as_deref()),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_location(&self, index: usize) -> &Location {
-        if let Self::Locations(array) = self {
-            &array[index]
-        } else {
-            panic!("Not an array of locations")
-        }
-    }
-
-    /// Returns the number of elements in the array.
-    pub(crate) fn len(&self) -> usize {
-        match self {
-            Self::MsgIds(array) => array.len(),
-            Self::Chat(array) => array.len(),
-            Self::Locations(array) => array.len(),
-            Self::Uint(array) => array.len(),
-        }
-    }
-
-    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
-        (0..self.len()).find(|i| self.get_id(*i) == needle)
-    }
-}
-
-impl From<Vec<u32>> for dc_array_t {
-    fn from(array: Vec<u32>) -> Self {
-        dc_array_t::Uint(array)
-    }
-}
-
-impl From<Vec<MsgId>> for dc_array_t {
-    fn from(array: Vec<MsgId>) -> Self {
-        dc_array_t::MsgIds(array)
-    }
-}
-
-impl From<Vec<ChatItem>> for dc_array_t {
-    fn from(array: Vec<ChatItem>) -> Self {
-        dc_array_t::Chat(array)
-    }
-}
-
-impl From<Vec<Location>> for dc_array_t {
-    fn from(array: Vec<Location>) -> Self {
-        dc_array_t::Locations(array)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_dc_array() {
-        let arr: dc_array_t = Vec::<u32>::new().into();
-        assert!(arr.len() == 0);
-
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
-
-        assert_eq!(arr.len(), 1000);
-
-        for i in 0..1000 {
-            assert_eq!(arr.get_id(i), (i + 2) as u32);
-        }
-
-        assert_eq!(arr.search_id(10), Some(8));
-        assert_eq!(arr.search_id(1), None);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_dc_array_out_of_bounds() {
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
-        arr.get_id(1000);
-    }
-}

deltachat-ffi/src/providers.rs

@@ -0,0 +1,92 @@
+extern crate deltachat_provider_overview;
+
+use std::ptr;
+
+use deltachat::dc_tools::{as_str, StrExt};
+use deltachat_provider_overview::StatusState;
+
+#[no_mangle]
+pub type dc_provider_t = deltachat_provider_overview::Provider;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_domain(
+    domain: *const libc::c_char,
+) -> *const dc_provider_t {
+    match deltachat_provider_overview::get_provider_info(as_str(domain)) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email(
+    email: *const libc::c_char,
+) -> *const dc_provider_t {
+    let domain = deltachat_provider_overview::get_domain_from_email(as_str(email));
+    match deltachat_provider_overview::get_provider_info(domain) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+macro_rules! null_guard {
+    ($context:tt) => {
+        if $context.is_null() {
+            return ptr::null_mut() as *mut libc::c_char;
+        }
+    };
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_overview_page(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    format!(
+        "{}/{}",
+        deltachat_provider_overview::PROVIDER_OVERVIEW_URL,
+        (*provider).overview_page
+    )
+    .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_name(provider: *const dc_provider_t) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).name.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_markdown(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).markdown.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status_date(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).status.date.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t) -> u32 {
+    if provider.is_null() {
+        return 0;
+    }
+    match (*provider).status.state {
+        StatusState::OK => 1,
+        StatusState::PREPARATION => 2,
+        StatusState::BROKEN => 3,
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {
+    ()
+}
+
+// TODO expose general provider overview url?

deltachat-ffi/src/string.rs

@@ -1,421 +0,0 @@
-use std::ffi::{CStr, CString};
-use std::ptr;
-
-/// Duplicates a string
-///
-/// returns an empty string if NULL is given, never returns NULL (exits on errors)
-///
-/// # Examples
-///
-/// ```rust,norun
-/// use crate::string::{dc_strdup, to_string_lossy};
-/// unsafe {
-///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
-///     let str_a_copy = dc_strdup(str_a);
-///     assert_eq!(to_string_lossy(str_a_copy), "foobar");
-///     assert_ne!(str_a, str_a_copy);
-/// }
-/// ```
-unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
-    } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
-    ret
-}
-
-/// Error type for the [OsStrExt] trait
-#[derive(Debug, PartialEq, thiserror::Error)]
-pub(crate) enum CStringError {
-    /// The string contains an interior null byte
-    #[error("String contains an interior null byte")]
-    InteriorNullByte,
-    /// The string is not valid Unicode
-    #[error("String is not valid unicode")]
-    NotUnicode,
-}
-
-/// Extra convenience methods on [std::ffi::OsStr] to work with `*libc::c_char`.
-///
-/// The primary function of this trait is to more easily convert
-/// [OsStr], [OsString] or [Path] into pointers to C strings.  This always
-/// allocates a new string since it is very common for the source
-/// string not to have the required terminal null byte.
-///
-/// It is implemented for `AsRef<std::ffi::OsStr>>` trait, which
-/// allows any type which implements this trait to transparently use
-/// this.  This is how the conversion for [Path] works.
-///
-/// [OsStr]: std::ffi::OsStr
-/// [OsString]: std::ffi::OsString
-/// [Path]: std::path::Path
-///
-/// # Example
-///
-/// ```
-/// use deltachat::dc_tools::{dc_strdup, OsStrExt};
-/// let path = std::path::Path::new("/some/path");
-/// let path_c = path.to_c_string().unwrap();
-/// unsafe {
-///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
-/// }
-/// ```
-pub(crate) trait OsStrExt {
-    /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
-    ///
-    /// This is useful to convert e.g. a [std::path::Path] to
-    /// [*libc::c_char] by using
-    /// [Path::as_os_str()](std::path::Path::as_os_str) and
-    /// [CStr::as_ptr()](std::ffi::CStr::as_ptr).
-    ///
-    /// This returns [CString] and not [&CStr] because not all [OsStr]
-    /// slices end with a null byte, particularly those coming from
-    /// [Path] do not have a null byte and having to handle this as
-    /// the caller would defeat the point of this function.
-    ///
-    /// On Windows this requires that the [OsStr] contains valid
-    /// unicode, which should normally be the case for a [Path].
-    ///
-    /// [CString]: std::ffi::CString
-    /// [CStr]: std::ffi::CStr
-    /// [OsStr]: std::ffi::OsStr
-    /// [Path]: std::path::Path
-    ///
-    /// # Errors
-    ///
-    /// Since a C `*char` is terminated by a NULL byte this conversion
-    /// will fail, when the [OsStr] has an interior null byte.  The
-    /// function will return
-    /// `[Err]([CStringError::InteriorNullByte])`.  When converting
-    /// from a [Path] it should be safe to
-    /// [`.unwrap()`](std::result::Result::unwrap) this anyway since a
-    /// [Path] should not contain interior null bytes.
-    ///
-    /// On windows when the string contains invalid Unicode
-    /// `[Err]([CStringError::NotUnicode])` is returned.
-    fn to_c_string(&self) -> Result<CString, CStringError>;
-}
-
-impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
-    #[cfg(not(target_os = "windows"))]
-    fn to_c_string(&self) -> Result<CString, CStringError> {
-        use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
-        })
-    }
-
-    #[cfg(target_os = "windows")]
-    fn to_c_string(&self) -> Result<CString, CStringError> {
-        os_str_to_c_string_unicode(&self)
-    }
-}
-
-// Implementation for os_str_to_c_string on windows.
-#[allow(dead_code)]
-fn os_str_to_c_string_unicode(
-    os_str: &dyn AsRef<std::ffi::OsStr>,
-) -> Result<CString, CStringError> {
-    match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
-        }),
-        None => Err(CStringError::NotUnicode),
-    }
-}
-
-/// Convenience methods/associated functions for working with [CString]
-trait CStringExt {
-    /// Create a new [CString], best effort
-    ///
-    /// Like the [to_string_lossy] this doesn't give up in the face of
-    /// bad input (embedded null bytes in this case) instead it does
-    /// the best it can by stripping the embedded null bytes.
-    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
-        let mut s = t.into();
-        s.retain(|&c| c != 0);
-        CString::new(s).unwrap_or_default()
-    }
-}
-
-impl CStringExt for CString {}
-
-/// Convenience methods to turn strings into C strings.
-///
-/// To interact with (legacy) C APIs we often need to convert from
-/// Rust strings to raw C strings.  This can be clumsy to do correctly
-/// and the compiler sometimes allows it in an unsafe way.  These
-/// methods make it more succinct and help you get it right.
-pub(crate) trait Strdup {
-    /// Allocate a new raw C `*char` version of this string.
-    ///
-    /// This allocates a new raw C string which must be freed using
-    /// `free`.  It takes care of some common pitfalls with using
-    /// [CString.as_ptr].
-    ///
-    /// [CString.as_ptr]: std::ffi::CString.as_ptr
-    ///
-    /// # Panics
-    ///
-    /// This function will panic when the original string contains an
-    /// interior null byte as this can not be represented in raw C
-    /// strings.
-    unsafe fn strdup(&self) -> *mut libc::c_char;
-}
-
-impl Strdup for str {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
-impl Strdup for std::path::Path {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-/// Convenience methods to turn optional strings into C strings.
-///
-/// This is the same as the [Strdup] trait but a different trait name
-/// to work around the type system not allowing to implement [Strdup]
-/// for `Option<impl Strdup>` When we already have an [Strdup] impl
-/// for `AsRef<&str>`.
-///
-/// When the [Option] is [Option::Some] this behaves just like
-/// [Strdup::strdup], when it is [Option::None] a null pointer is
-/// returned.
-pub(crate) trait OptStrdup {
-    /// Allocate a new raw C `*char` version of this string, or NULL.
-    ///
-    /// See [Strdup::strdup] for details.
-    unsafe fn strdup(&self) -> *mut libc::c_char;
-}
-
-impl<T: AsRef<str>> OptStrdup for Option<T> {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        match self {
-            Some(s) => {
-                let tmp = CString::new_lossy(s.as_ref());
-                dc_strdup(tmp.as_ptr())
-            }
-            None => ptr::null_mut(),
-        }
-    }
-}
-
-pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
-    if s.is_null() {
-        return "".into();
-    }
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_string_lossy().to_string()
-}
-
-pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
-    if s.is_null() {
-        return None;
-    }
-
-    Some(to_string_lossy(s))
-}
-
-/// Convert a C `*char` pointer to a [std::path::Path] slice.
-///
-/// This converts a `*libc::c_char` pointer to a [Path] slice.  This
-/// essentially has to convert the pointer to [std::ffi::OsStr] to do
-/// so and thus is the inverse of [OsStrExt::to_c_string].  Just like
-/// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
-/// requires that the pointer contains valid UTF-8 on Windows.
-///
-/// Because this returns a reference the [Path] slice can not outlive
-/// the original pointer.
-///
-/// [Path]: std::path::Path
-#[cfg(not(target_os = "windows"))]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
-    assert!(!s.is_null(), "cannot be used on null pointers");
-    use std::os::unix::ffi::OsStrExt;
-    unsafe {
-        let c_str = std::ffi::CStr::from_ptr(s).to_bytes();
-        let os_str = std::ffi::OsStr::from_bytes(c_str);
-        std::path::Path::new(os_str)
-    }
-}
-
-// as_path() implementation for windows, documented above.
-#[cfg(target_os = "windows")]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
-    as_path_unicode(s)
-}
-
-// Implementation for as_path() on Windows.
-//
-// Having this as a separate function means it can be tested on unix
-// too.
-#[allow(dead_code)]
-fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
-    assert!(!s.is_null(), "cannot be used on null pointers");
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-    let str = cstr.to_str().unwrap_or_else(|err| panic!("{}", err));
-
-    std::path::Path::new(str)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use libc::{free, strcmp};
-
-    #[test]
-    fn test_os_str_to_c_string_cwd() {
-        let some_dir = std::env::current_dir().unwrap();
-        some_dir.as_os_str().to_c_string().unwrap();
-    }
-
-    #[test]
-    fn test_os_str_to_c_string_unicode() {
-        let some_str = String::from("/some/valid/utf8");
-        let some_dir = std::path::Path::new(&some_str);
-        assert_eq!(
-            some_dir.as_os_str().to_c_string().unwrap(),
-            CString::new("/some/valid/utf8").unwrap()
-        );
-    }
-
-    #[test]
-    fn test_os_str_to_c_string_nul() {
-        let some_str = std::ffi::OsString::from("foo\x00bar");
-        assert_eq!(
-            some_str.to_c_string().err().unwrap(),
-            CStringError::InteriorNullByte
-        )
-    }
-
-    #[test]
-    fn test_path_to_c_string_cwd() {
-        let some_dir = std::env::current_dir().unwrap();
-        some_dir.to_c_string().unwrap();
-    }
-
-    #[test]
-    fn test_path_to_c_string_unicode() {
-        let some_str = String::from("/some/valid/utf8");
-        let some_dir = std::path::Path::new(&some_str);
-        assert_eq!(
-            some_dir.as_os_str().to_c_string().unwrap(),
-            CString::new("/some/valid/utf8").unwrap()
-        );
-    }
-
-    #[test]
-    fn test_os_str_to_c_string_unicode_fn() {
-        let some_str = std::ffi::OsString::from("foo");
-        assert_eq!(
-            os_str_to_c_string_unicode(&some_str).unwrap(),
-            CString::new("foo").unwrap()
-        );
-    }
-
-    #[test]
-    fn test_path_to_c_string_unicode_fn() {
-        let some_str = String::from("/some/path");
-        let some_path = std::path::Path::new(&some_str);
-        assert_eq!(
-            os_str_to_c_string_unicode(&some_path).unwrap(),
-            CString::new("/some/path").unwrap()
-        );
-    }
-
-    #[test]
-    fn test_os_str_to_c_string_unicode_fn_nul() {
-        let some_str = std::ffi::OsString::from("fooz\x00bar");
-        assert_eq!(
-            os_str_to_c_string_unicode(&some_str).err().unwrap(),
-            CStringError::InteriorNullByte
-        );
-    }
-
-    #[test]
-    fn test_as_path() {
-        let some_path = CString::new("/some/path").unwrap();
-        let ptr = some_path.as_ptr();
-        assert_eq!(as_path(ptr), std::ffi::OsString::from("/some/path"))
-    }
-
-    #[test]
-    fn test_as_path_unicode_fn() {
-        let some_path = CString::new("/some/path").unwrap();
-        let ptr = some_path.as_ptr();
-        assert_eq!(as_path_unicode(ptr), std::ffi::OsString::from("/some/path"));
-    }
-
-    #[test]
-    fn test_cstring_new_lossy() {
-        assert!(CString::new("hel\x00lo").is_err());
-        assert!(CString::new(String::from("hel\x00o")).is_err());
-        let r = CString::new("hello").unwrap();
-        assert_eq!(CString::new_lossy("hello"), r);
-        assert_eq!(CString::new_lossy("hel\x00lo"), r);
-        assert_eq!(CString::new_lossy(String::from("hello")), r);
-        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
-    }
-
-    #[test]
-    fn test_strdup_str() {
-        unsafe {
-            let s = "hello".strdup();
-            let cmp = strcmp(s, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(s as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-        }
-    }
-
-    #[test]
-    fn test_strdup_string() {
-        unsafe {
-            let s = String::from("hello").strdup();
-            let cmp = strcmp(s, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(s as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-        }
-    }
-
-    #[test]
-    fn test_strdup_opt_string() {
-        unsafe {
-            let s = Some("hello");
-            let c = s.strdup();
-            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(c as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-
-            let s: Option<&str> = None;
-            let c = s.strdup();
-            assert_eq!(c, ptr::null_mut());
-        }
-    }
-}

deltachat_derive/Cargo.toml

@@ -1,13 +1,12 @@
 [package]
 name = "deltachat_derive"
-version = "2.0.0"
+version = "0.1.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+authors = ["Dmitry Bogatov <KAction@debian.org>"]
 edition = "2018"
-license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "1.0.74"
+syn = "0.14.4"
-quote = "1.0.2"
+quote = "0.6.3"

deltachat_derive/src/lib.rs

@@ -3,6 +3,7 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
+use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to
@@ -35,11 +36,7 @@ pub fn from_sql_derive(input: TokenStream) -> TokenStream {
         impl rusqlite::types::FromSql for #name {
             fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
                 let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
+                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
             }
         }
     };

draft/group-sync.rst

@@ -1,111 +0,0 @@
-
-Problem: missing eventual group consistency
---------------------------------------------
-
-If group members are concurrently adding new members,
-the new members will miss each other's additions, example:
-
-1. Alice and Bob are in a two-member group
-
-2. Then Alice adds Carol, while concurrently Bob adds Doris
-
-Right now, the group has inconsistent memberships:
-
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
-
-
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
-
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
-
-
-
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
-
-For brevity let's abbreviate "group membership modification" as **GMM**.
-
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
-
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
-
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
-
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
-
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
-
-
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
-
-
-Discussions of variants
-++++++++++++++++++++++++
-
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
-  b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
-
-- instead of broadcasting correction messages we could only send it to
-  the sender of the inconsistent member-added/removed message.
-  A receiver of such a correction message would then need to forward
-  the message to the members it thinks also have an inconsistent view.
-  This sounds complicated and error-prone.  Concretely, if Alice
-  receives Bob's "Member-added: Doris" message, then Alice
-  broadcasting the correction message with "Member-added: Carol"
-  would reach all four members, healing group consistency in one step.
-  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
-  Bob would broadcast a correction message to all four members as well.
-  (Imagine a situation where Alice/Bob added Carol/Doris
-  while both being in an offline or bad-connection situation).
-
-

draft/msgwork_new_imap_jobs.rst

@@ -1,66 +0,0 @@
-
-simplify/streamline mark-seen/delete/move/send-mdn job handling
----------------------------------------------------------------
-
-Idea: Introduce a new "msgwork" sql table that looks very
-much like the jobs table but has a primary key "msgid"
-and no job id and no foreign-id anymore. This opens up
-bulk-processing by looking at the whole table and combining
-flag-setting to reduce imap-roundtrips and select-folder calls.
-
-Concretely, these IMAP jobs:
-
-    DeleteMsgOnImap
-    MarkseenMsgOnImap
-    MoveMsg
-
-Would be replaced by a few per-message columns in the new msgwork table:
-
-- needs_mark_seen: (bool) message shall be marked as seen on imap
-- needs_to_move: (bool) message should be moved to mvbox_folder
-- deletion_time: (target_time or 0) message shall be deleted at specified time
-- needs_send_mdn: (bool) MDN shall be sent
-
-The various places that currently add the (replaced) jobs
-would now add/modify the respective message record in the message-work table.
-
-Looking at a single message-work entry conceptually looks like this::
-
-    if msg.server_uid==0:
-        return RetryLater  # nothing can be done without server_uid
-
-    if msg.deletion_time > current_time:
-        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
-        clear(needs_deletion)
-        clear(mark_seen)
-
-    if needs_mark_seen:
-        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
-        clear(needs_mark_seen)
-
-    if needs_send_mdn:
-        schedule_smtp_send_mdn(msg)
-        clear(needs_send_mdn)
-
-    if any_flag_set():
-        retrylater
-    # remove msgwork entry from table
-
-
-Notes/Questions:
-
-- it's unclear how much we need per-message retry-time tracking/backoff
-
-- drafting bulk processing algo is useful before
-  going for the implementation, i.e. including select_folder calls etc.
-
-- maybe it's better to not have bools for the flags but
-
-  0 (no change)
-  1 (set the imap flag)
-  2 (clear the imap flag)
-
-  and design such that we can cover all imap flags.
-
-- It might not be neccessary to keep needs_send_mdn state in this table
-  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/main.rs

@@ -6,116 +6,215 @@
 
 #[macro_use]
 extern crate deltachat;
+#[macro_use]
+extern crate failure;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
 
-use ansi_term::Color;
-use anyhow::{bail, Error};
-use async_std::path::Path;
-use deltachat::chat::ChatId;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::context::*;
+use deltachat::dc_tools::*;
+use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::x::*;
-use log::{error, info, warn};
+use deltachat::Event;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
 
-/// Event Handler
+// Event Handler
-fn receive_event(event: EventType) {
+
-    let yellow = Color::Yellow.normal();
+fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
     match event {
-        EventType::Info(msg) => {
+        Event::GetString { .. } => {}
+        Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            info!("{}", msg);
+            println!("{}", msg);
         }
-        EventType::SmtpConnected(msg) => {
+        Event::SmtpConnected(msg) => {
-            info!("[SMTP_CONNECTED] {}", msg);
+            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
         }
-        EventType::ImapConnected(msg) => {
+        Event::ImapConnected(msg) => {
-            info!("[IMAP_CONNECTED] {}", msg);
+            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
         }
-        EventType::SmtpMessageSent(msg) => {
+        Event::SmtpMessageSent(msg) => {
-            info!("[SMTP_MESSAGE_SENT] {}", msg);
+            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
-            warn!("{}", msg);
+            println!("[Warning] {}", msg);
         }
-        EventType::Error(msg) => {
+        Event::Error(msg) => {
-            error!("{}", msg);
+            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
         }
-        EventType::ErrorSelfNotInGroup(msg) => {
+        Event::ErrorNetwork(msg) => {
-            error!("[SELF_NOT_IN_GROUP] {}", msg);
+            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
         }
-        EventType::MsgsChanged { chat_id, msg_id } => {
+        Event::ErrorSelfNotInGroup(msg) => {
-            info!(
+            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
-                "{}",
+        }
-                yellow.paint(format!(
+        Event::MsgsChanged { chat_id, msg_id } => {
-                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
+            print!(
-                    chat_id, msg_id,
+                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
-                ))
+                chat_id, msg_id,
             );
         }
-        EventType::ContactsChanged(_) => {
+        Event::ContactsChanged(_) => {
-            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
+            print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
         }
-        EventType::LocationChanged(contact) => {
+        Event::LocationChanged(contact) => {
-            info!(
+            print!(
-                "{}",
+                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
-                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+                contact,
             );
         }
-        EventType::ConfigureProgress { progress, comment } => {
+        Event::ConfigureProgress(progress) => {
-            if let Some(comment) = comment {
+            print!(
-                info!(
+                "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
-                    "{}",
+                progress,
-                    yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
-                    ))
-                );
-            } else {
-                info!(
-                    "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-                );
-            }
-        }
-        EventType::ImexProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        EventType::ImexFileWritten(file) => {
+        Event::ImexProgress(progress) => {
-            info!(
+            print!(
-                "{}",
+                "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
-                yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
+                progress,
             );
         }
-        EventType::ChatModified(chat) => {
+        Event::ImexFileWritten(file) => {
-            info!(
+            print!(
-                "{}",
+                "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
-                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
+                file.display()
+            );
+        }
+        Event::ChatModified(chat) => {
+            print!(
+                "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
+                chat
             );
         }
         _ => {
-            info!("Received {:?}", event);
+            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
+
+    0
+}
+
+// Threads for waiting for messages and for jobs
+
+lazy_static! {
+    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
+    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
+}
+
+struct Handle {
+    handle_imap: Option<std::thread::JoinHandle<()>>,
+    handle_mvbox: Option<std::thread::JoinHandle<()>>,
+    handle_sentbox: Option<std::thread::JoinHandle<()>>,
+    handle_smtp: Option<std::thread::JoinHandle<()>>,
+}
+
+macro_rules! while_running {
+    ($code:block) => {
+        if IS_RUNNING.load(Ordering::Relaxed) {
+            $code
+        } else {
+            break;
+        }
+    };
+}
+
+fn start_threads(c: Arc<RwLock<Context>>) {
+    if HANDLE.clone().lock().unwrap().is_some() {
+        return;
+    }
+
+    println!("Starting threads");
+    IS_RUNNING.store(true, Ordering::Relaxed);
+
+    let ctx = c.clone();
+    let handle_imap = std::thread::spawn(move || loop {
+        while_running!({
+            perform_imap_jobs(&ctx.read().unwrap());
+            perform_imap_fetch(&ctx.read().unwrap());
+            while_running!({
+                let context = ctx.read().unwrap();
+                perform_imap_idle(&context);
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_mvbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_mvbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_mvbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_sentbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_sentbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_sentbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c;
+    let handle_smtp = std::thread::spawn(move || loop {
+        while_running!({
+            perform_smtp_jobs(&ctx.read().unwrap());
+            while_running!({
+                perform_smtp_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    *HANDLE.clone().lock().unwrap() = Some(Handle {
+        handle_imap: Some(handle_imap),
+        handle_mvbox: Some(handle_mvbox),
+        handle_sentbox: Some(handle_sentbox),
+        handle_smtp: Some(handle_smtp),
+    });
+}
+
+fn stop_threads(context: &Context) {
+    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
+        println!("Stopping threads");
+        IS_RUNNING.store(false, Ordering::Relaxed);
+
+        interrupt_imap_idle(context);
+        interrupt_mvbox_idle(context);
+        interrupt_sentbox_idle(context);
+        interrupt_smtp_idle(context);
+
+        handle.handle_imap.take().unwrap().join().unwrap();
+        handle.handle_mvbox.take().unwrap().join().unwrap();
+        handle.handle_sentbox.take().unwrap().join().unwrap();
+        handle.handle_smtp.take().unwrap().join().unwrap();
+    }
 }
 
 // === The main loop
@@ -139,7 +238,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 12] = [
+const IMEX_COMMANDS: [&'static str; 12] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -154,24 +253,26 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&'static str; 11] = [
     "info",
+    "open",
+    "close",
     "set",
     "get",
     "oauth2",
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 33] = [
+const CHAT_COMMANDS: [&'static str; 24] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "createchatbymsg",
     "creategroup",
     "createverified",
     "addmember",
@@ -186,58 +287,36 @@ const CHAT_COMMANDS: [&str; 33] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
-    "videochat",
     "draft",
     "listmedia",
     "archive",
     "unarchive",
-    "pin",
-    "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 6] = [
+const MESSAGE_COMMANDS: [&'static str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&'static str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
-const MISC_COMMANDS: [&str; 10] = [
+const MISC_COMMANDS: [&'static str; 9] = [
-    "getqr",
+    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
-    "getbadqr",
-    "checkqr",
-    "event",
-    "fileinfo",
-    "clear",
-    "exit",
-    "quit",
-    "help",
-    "estimatedeletion",
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -258,11 +337,12 @@ impl Hinter for DcHelper {
     }
 }
 
-static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static COLORED_PROMPT: &'static str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &'static str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
-        if default {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -283,83 +363,70 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
-async fn start(args: Vec<String>) -> Result<(), Error> {
+fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        bail!("No db-name specified");
+        return Err(format_err!("No db-name specified"));
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(
-
+        Box::new(receive_event),
-    let events = context.get_event_emitter();
+        "CLI".into(),
-    async_std::task::spawn(async move {
+        Path::new(&args[1]).to_path_buf(),
-        while let Some(event) = events.recv().await {
+    )?;
-            receive_event(event.typ);
-        }
-    });
 
     println!("Delta Chat Core is awaiting your commands.");
 
+    let ctx = Arc::new(RwLock::new(context));
+
     let config = Config::builder()
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let mut selected_chat = ChatId::default();
+    let h = DcHelper {
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
+        completer: FilenameCompleter::new(),
-    let input_loop = async_std::task::spawn_blocking(move || {
+        highlighter: MatchingBracketHighlighter::new(),
-        let h = DcHelper {
+        hinter: HistoryHinter {},
-            completer: FilenameCompleter::new(),
+    };
-            highlighter: MatchingBracketHighlighter::new(),
+    let mut rl = Editor::with_config(config);
-            hinter: HistoryHinter {},
+    rl.set_helper(Some(h));
-        };
+    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-        let mut rl = Editor::with_config(config);
+    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-        rl.set_helper(Some(h));
+    if rl.load_history(".dc-history.txt").is_err() {
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
+        println!("No previous history.");
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+    }
-        if rl.load_history(".dc-history.txt").is_err() {
-            println!("No previous history.");
-        }
 
-        loop {
+    loop {
-            let p = "> ";
+        let p = "> ";
-            let readline = rl.readline(&p);
+        let readline = rl.readline(&p);
-
+        match readline {
-            match readline {
+            Ok(line) => {
-                Ok(line) => {
+                // TODO: ignore "set mail_pw"
-                    // TODO: ignore "set mail_pw"
+                rl.add_history_entry(line.as_str());
-                    rl.add_history_entry(line.as_str());
+                let ctx = ctx.clone();
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                match unsafe { handle_cmd(line.trim(), ctx) } {
-                }
+                    Ok(ExitResult::Continue) => {}
-                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    Ok(ExitResult::Exit) => break,
-                    println!("Exiting...");
+                    Err(err) => println!("Error: {}", err),
-                    drop(reader_s);
-                    break;
-                }
-                Err(err) => {
-                    println!("Error: {}", err);
-                    drop(reader_s);
-                    break;
                 }
             }
-        }
+            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-
+                println!("Exiting...");
-        rl.save_history(".dc-history.txt")?;
+                break;
-        println!("history saved");
+            }
-        Ok::<_, Error>(())
+            Err(err) => {
-    });
+                println!("Error: {}", err);
-
+                break;
-    while let Ok(line) = reader_r.recv().await {
+            }
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
         }
     }
-    context.stop_io().await;
+    rl.save_history(".dc-history.txt")?;
-    input_loop.await?;
+    println!("history saved");
+    {
+        stop_threads(&ctx.read().unwrap());
+    }
 
     Ok(())
 }
@@ -370,29 +437,48 @@ enum ExitResult {
     Exit,
 }
 
-async fn handle_cmd(
+unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
-    line: &str,
-    ctx: Context,
-    selected_chat: &mut ChatId,
-) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
+    let arg1_c = if arg1.is_empty() {
+        std::ptr::null()
+    } else {
+        arg1.strdup()
+    };
 
     match arg0 {
         "connect" => {
-            ctx.start_io().await;
+            start_threads(ctx);
         }
         "disconnect" => {
-            ctx.stop_io().await;
+            stop_threads(&ctx.read().unwrap());
+        }
+        "smtp-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("smtp-jobs are already running in a thread.",);
+            } else {
+                perform_smtp_jobs(&ctx.read().unwrap());
+            }
+        }
+        "imap-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("imap-jobs are already running in a thread.");
+            } else {
+                perform_imap_jobs(&ctx.read().unwrap());
+            }
         }
         "configure" => {
-            ctx.configure().await?;
+            start_threads(ctx.clone());
+            configure(&ctx.read().unwrap());
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
+            if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
-                let oauth2_url =
+                let oauth2_url = dc_get_oauth2_url(
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
+                    &ctx.read().unwrap(),
+                    &addr,
+                    "chat.delta:/com.b44t.messenger",
+                );
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -407,9 +493,10 @@ async fn handle_cmd(
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            ctx.start_io().await;
+            start_threads(ctx.clone());
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) =
-            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
+                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -425,23 +512,25 @@ async fn handle_cmd(
             }
         }
         "joinqr" => {
-            ctx.start_io().await;
+            start_threads(ctx.clone());
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                dc_join_securejoin(&ctx.read().unwrap(), arg1);
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
-        _ => cmdline(ctx.clone(), line, selected_chat).await?,
+        _ => dc_cmdline(&ctx.read().unwrap(), line)?,
     }
 
+    free(arg1_c as *mut _);
+
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+pub fn main() -> Result<(), failure::Error> {
     let _ = pretty_env_logger::try_init();
 
-    let args = std::env::args().collect();
+    let args: Vec<String> = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    main_0(args)?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,100 +1,127 @@
+extern crate deltachat;
+
+use std::sync::{Arc, RwLock};
+use std::{thread, time};
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::message::Message;
+use deltachat::job::{
-use deltachat::EventType;
+    perform_imap_fetch, perform_imap_idle, perform_imap_jobs, perform_smtp_idle, perform_smtp_jobs,
+};
+use deltachat::Event;
+
+fn cb(_ctx: &Context, event: Event) -> usize {
+    print!("[{:?}]", event);
 
-fn cb(event: EventType) {
     match event {
-        EventType::ConfigureProgress { progress, .. } => {
+        Event::ConfigureProgress(progress) => {
-            log::info!("progress: {}", progress);
+            print!("  progress: {}\n", progress);
+            0
         }
-        EventType::Info(msg) => {
+        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            log::info!("{}", msg);
+            print!("  {}\n", msg);
+            0
         }
-        EventType::Warning(msg) => {
+        _ => {
-            log::warn!("{}", msg);
+            print!("\n");
-        }
+            0
-        EventType::Error(msg) => {
-            log::error!("{}", msg);
-        }
-        event => {
-            log::info!("{:?}", event);
         }
     }
 }
 
-/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
+fn main() {
-#[async_std::main]
+    unsafe {
-async fn main() {
+        let dir = tempdir().unwrap();
-    pretty_env_logger::try_init_timed().ok();
+        let dbfile = dir.path().join("db.sqlite");
+        println!("creating database {:?}", dbfile);
+        let ctx =
+            Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+        let running = Arc::new(RwLock::new(true));
+        let info = ctx.get_info();
+        let duration = time::Duration::from_millis(4000);
+        println!("info: {:#?}", info);
 
-    let dir = tempdir().unwrap();
+        let ctx = Arc::new(ctx);
-    let dbfile = dir.path().join("db.sqlite");
+        let ctx1 = ctx.clone();
-    log::info!("creating database {:?}", dbfile);
+        let r1 = running.clone();
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
+        let t1 = thread::spawn(move || {
-        .await
+            while *r1.read().unwrap() {
-        .expect("Failed to create context");
+                perform_imap_jobs(&ctx1);
-    let info = ctx.get_info().await;
+                if *r1.read().unwrap() {
-    log::info!("info: {:#?}", info);
+                    perform_imap_fetch(&ctx1);
 
-    let events = ctx.get_event_emitter();
+                    if *r1.read().unwrap() {
-    let events_spawn = async_std::task::spawn(async move {
+                        perform_imap_idle(&ctx1);
-        while let Some(event) = events.recv().await {
+                    }
-            cb(event.typ);
+                }
+            }
+        });
+
+        let ctx1 = ctx.clone();
+        let r1 = running.clone();
+        let t2 = thread::spawn(move || {
+            while *r1.read().unwrap() {
+                perform_smtp_jobs(&ctx1);
+                if *r1.read().unwrap() {
+                    perform_smtp_idle(&ctx1);
+                }
+            }
+        });
+
+        println!("configuring");
+        let args = std::env::args().collect::<Vec<String>>();
+        assert_eq!(args.len(), 2, "missing password");
+        let pw = args[1].clone();
+        ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+            .unwrap();
+        ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+        configure(&ctx);
+
+        thread::sleep(duration);
+
+        println!("sending a message");
+        let contact_id =
+            Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+        let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+        chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
+
+        println!("fetching chats..");
+        let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
+
+        for i in 0..chats.len() {
+            let summary = chats.get_summary(&ctx, 0, None);
+            let text1 = summary.get_text1();
+            let text2 = summary.get_text2();
+            println!("chat: {} - {:?} - {:?}", i, text1, text2,);
         }
-    });
 
-    log::info!("configuring");
+        thread::sleep(duration);
-    let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 3, "requires email password");
-    let email = args[1].clone();
-    let pw = args[2].clone();
-    ctx.set_config(config::Config::Addr, Some(&email))
-        .await
-        .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw))
-        .await
-        .unwrap();
 
-    ctx.configure().await.unwrap();
+        // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
+        // for i in 0..dc_array_get_cnt(msglist) {
+        //     let msg_id = dc_array_get_id(msglist, i);
+        //     let msg = dc_get_msg(context, msg_id);
+        //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
+        //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
+        //     dc_msg_unref(msg);
+        // }
+        // dc_array_unref(msglist);
 
-    log::info!("------ RUN ------");
+        println!("stopping threads");
-    ctx.start_io().await;
-    log::info!("--- SENDING A MESSAGE ---");
 
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
+        *running.clone().write().unwrap() = false;
-        .await
+        deltachat::job::interrupt_imap_idle(&ctx);
-        .unwrap();
+        deltachat::job::interrupt_smtp_idle(&ctx);
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
 
-    for i in 0..1 {
+        println!("joining");
-        log::info!("sending message {}", i);
+        t1.join().unwrap();
-        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
+        t2.join().unwrap();
-            .await
+
-            .unwrap();
+        println!("closing");
     }
-
-    // wait for the message to be sent out
-    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
-
-    log::info!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
-
-    for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
-            .await
-            .unwrap();
-        log::info!("[{}] msg: {:?}", i, msg);
-    }
-
-    log::info!("stopping");
-    ctx.stop_io().await;
-    log::info!("closing");
-    drop(ctx);
-    events_spawn.await;
 }

proptest-regressions/dc_mimeparser.txt

@@ -1,7 +0,0 @@
-# Seeds for failure cases proptest has generated in the past. It is
-# automatically read and these particular cases re-run before any
-# novel cases are generated.
-#
-# It is recommended to check this file in to source control so that
-# everyone who runs the test benefits from these saved cases.
-cc 03cab93c6d1f3a8245f63cf84dacb307944294fe6333c1e38f078a6600659c7a # shrinks to data = "a\t0aA\ta\t0 \ta\t0 \ta a\t\ta  A\tAA0a0a 0\t a\t aA   \t a\t  A0\t  AAa\taA0\taAAaA\t0\taa0a\ta Aa Aaaa A0A\t  a aA 0\t  A\t0\t0\t\t\t\t\t\tA \t\t a\tA Aa aAA0A0AA0aaA A\t\t  aa0\ta\t  \tAa\taA\t00  AA  A a\tA a aAAa \t 00\t0 \t\t  a A  0\t\t\t aAA Aa \taAAAA0a A0A\t\t1\\E$\t$R\tc\t^=\t\"\tQ<Uk\t\t>A\t\t&\t}v&\tM^\'|\tW5?dn\t\t+\t\tP\te`\t\t>:Brlq--?\t$#Q\tK=zJ\tb\"9*.\"\t`\tF&T*\tBs,\tg\'*\\\t:\t?l$\t\t|A\"HR:Hk\t\\KkV\t\t{=&!^e%:|_*0wV\t[$`\t:\t$%f\t\t[!\"Y. \tP\t\th\'?\'/?%:++NfQo#@\"+?\t(\\??{\t\'\'$Dzj\t0.?{s4?&?Y=/yj]Z=\t4n\t?Ja\"\t{I\t$\t;I}_8V\t&\t?N\'\tI2/\t9.\tFT%9%`\'\tz\to7Y\t|AXP&@G12g\t\'w\t\t%??\t\"h$?F\"\"6%q\\\\{\tT\t\"]$87$}h\'\t<\t$\tc%U:mT2:<v\t#Rl!;U\t\t\"^D\tRZ(BZ{n\t%[$fgH`\t{B}:*\t\t%%*F`%W\t//B}PQ\t\tsu2\tLz<1*!p-X\tnKv&&0\thm4<n\\.\\/.w\'\t<)E1g3#ood\t`?\t\\({N?,8_?h\ty\t0%\t*$A\t\t*w-ViQUj\tTiH\t%\t%&0p\'\'\tA%r**Fo\'Z\\\tNI*ely4=I?}$.o`\t$\ts\'&lt\t\",:~=Nv/0O%=+<LS\t%P\'\t$r%d.\t{G{/L:\t&I&8-`cy*\"{\t/%fP9.P<\t\t\'/`\t\t`\t\t`!t:\t::\t\tW\'X^\t@.uL&a\tN\t\t\t.\t?0*\tvUK>UJ\\\tQbj%w]P=Js\t\"R\t&s^?</=%\t\'VI:\" kT`{b*<\t\tF&?Y\t\t:/\t!C\'e0`\t\t\tx-\t*\\N\\wwu.I4\tb/y\t\"P:P%\"\t\tQ][\\\t^x\t\t):\t\t&s\t$1-\t\t\tXt`0\t;\t/UeCP*\"G\t\t\':\tk@91Hf&#\t(Uzz&`p2;\t{]\t\"I_%=\\%,XD\"\'06QfCd<;$:q^\t8:\"*\"H\t\to\t&xK/\t\ty0/<}<j<|*`~::b\t=S.=}=Ki\t<Y.\'{\tf\t{Ub*H%(_4*?*\tn2\t/\'\t\t\t/,4]\tt\t<y\t\t\tWi\t\tT&\"\t\t\t\t\t=/1Wu?\t\'A\"W-P\t$?e\\\t`\t6`vD\t8`\t\tccD J\tY&jI//?_\t\\j\t_\tsiq$\t?9\tQ\t.^%&..?%Jm??A%+<\tN&*\t.g\tS$W\"\"\tMpq\t\t:&\\\thTz&<iz%/%T&\'F\t\\]&\t\t}\t\t\tXr=dtp`:+<,\t%60Y*<(&K*kJ\todO\t=<V?&\tMU/\"\t= Y!)<\tV\t9\t)\t&v8q{\t\t&pT\t3\ttB,qcq\'i$;gv%j_%M_{[\"&;\t\t\t.B;6Y\\%\t\"\tY/a\t\\`:?\t<\t?]\taNwD;\\\t%l*74%5T?QS :~yR/E*R\t\t=u\t\\\t\t.Q<;\\\t_S/{&F$^\tw_>\'h=|%\t\t:W!\\<U2\'$\tb&`\t=|=L9\t\t\t\\WZ:! }{\t ;\t;\t\t 0.*\t.%\"\'r%{<Mh_t[[}\t-\tJo\"b/AC*-=`=T\tz$2\tC\t\t/k{47\"\t\t,q%\tZ\tT3\t\tf>%\t\'?%@^tx\t7\"1Bk{b{n\t\"Pj3\tHc\t\tt\tY<\t#?\tSh\\yk/N\\\t8 7=R4*9Cw&m\t\\-\'f\t|\'#t(Etu.Hdu(E\t%&v:\'aqW~A5\t\t w.s{J%lX<\"\t\'*%m<&:/B<&\':U}$&`.{)\t\t6S\t:/$*kQ-Z\t^\'t${/tH*\'v\t3\t=\t\tDyp:B\t`I_R&4SO\t\t&-j=*.\t87&\'e*( \t\t\t\'<$\\DJ<$p?{}\'&\tv\t\\Xk<Y:Y`!$K{\tF&\tzd\t\t*i$\tj\'\t<)R*\t%?\t!.\t=\"@#~:=*\t\tXO=_T,1\"\'.%%\"`{\\:\t\"\tfkeOb/\'$I~\ta\t|&\t[\\KK\"1&Z\t<k\t\t)%\'-~\"2n\tj\tW?*<@w{g%d\ta\\\'\'I\t;:ySR%ke:4\tc\t$=\t&9P]x4\tJ=\t6C6%a\t`0\tF\tm-\tTr\t}\t\tQum\t&@\typ|w2&\t\t3`i&t\t\tT5\"\t.&b&e*/==1.\'*\\[U*\tqPL%?$-0/}~|q`\t\t}\t$\tq==o+T$\'!H\t\ti&um\"?\"%%\t/\'p\tg>?{0{J{\t\t/\t\t{zKZ&>=\t[\"1h<H%z/8,/]s\tv{7\t\t:j*H,M//\t\t\td\'.)\t"

proptest-regressions/dc_strencode.txt

@@ -5,4 +5,3 @@
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
 cc 679506fe9ac59df773f8cfa800fdab5f0a32fe49d2ab370394000a1aa5bc2a72 # shrinks to buf = "%0A"
-cc e34960438edb2426904b44fb4215154e7e2880f2fd1c3183b98bfcc76fec4882 # shrinks to input = " 0"

python/CHANGELOG

@@ -1,53 +1,6 @@
-1.51.0
+0.600.1
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
-1.44.0
-------
-
-- fix Chat.get_mute_duration()
-
-1.40.1
----------------
-
-- emit "ac_member_removed" event (with 'actor' being the removed contact) 
-  for when a user leaves a group.
-
-- fix create_contact(addr) when addr is the self-contact. 
-
-
-1.40.0
----------------
-
-- uses latest 1.40+ Delta Chat core
-
-- refactored internals to use plugin-approach 
-
-- introduced PerAccount and Global hooks that plugins can implement 
-
-- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
-
-- introduced two documented examples for an echo and a group-membership
-  tracking plugin. 
-
-
-0.800.0
--------
-
-- use latest core 1.25.0 
-
-- refine tests and some internal changes to core bindings
-
-0.700.0
 ---------
 
-- lots of new Python APIs
-
-- use rust core-beta23 
-
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,36 +3,41 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+which provides imap/smtp/crypto handling as well as chat/group/messages
-a low-level Chat/Contact/Message API to user interfaces and bots.
+handling to Android, Desktop and IO user interfaces.
 
+Installing pre-built packages (linux-only)
+==========================================
 
-Installing pre-built packages (Linux-only)
+If you have a linux system you may install the ``deltachat`` binary "wheel" package
-========================================================
-
-If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-then create a fresh Python virtual environment and activate it in your shell::
+   then create a fresh python environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
         source venv/bin/activate
 
-Afterwards, invoking ``python`` or ``pip install`` only
+   Afterwards, invoking ``python`` or ``pip install`` will only
-modifies files in your ``venv`` directory and leaves
+   modify files in your ``venv`` directory and leave your system installation
-your system installation alone.
+   alone.
+
+2. Install the wheel for linux::
+
+        pip install deltachat
+
+    Verify it worked by typing::
+
+        python -c "import deltachat"
+
+
+Installing a wheel from a PR/branch
+---------------------------------------
 
 For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
+and push them to a python package index. To install the latest github ``master`` branch::
-github ``master`` branch::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
+    pip install -i https://m.devpi.net/dc/master deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
 
 .. note::
 
@@ -41,77 +46,63 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Running tests
+Installing bindings from source
-=============
+===============================
 
-After successful binding installation you can install a few more
+If you can't use "binary" method above then you need to compile
-Python packages before running the tests::
+to core deltachat library::
+
+    git clone https://github.com/deltachat/deltachat-core-rust
+    cd deltachat-core-rust
+    cd python
+
+If you don't have one active, create and activate a python "virtualenv":
+
+    python virtualenv venv  # or python -m venv
+    source venv/bin/activate
+
+Afterwards ``which python`` tells you that it comes out of the "venv"
+directory that contains all python install artifacts. Let's first
+install test tools::
+
+    pip install pytest pytest-timeout pytest-rerunfailures requests
+
+then cargo-build and install the deltachat bindings::
+
+    python install_python_bindings.py
+
+The bindings will be installed in release mode but with debug symbols.
+The release mode is necessary because some tests generate RSA keys
+which is prohibitively slow in debug mode.
+
+After successful binding installation you can finally run the tests::
 
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
     pytest -v tests
 
-This will run all "offline" tests and skip all functional
+.. note::
-end-to-end tests that require accounts on real e-mail servers.
 
-.. _livetests:
+    Some tests are sometimes failing/hanging because of
-
+    https://github.com/deltachat/deltachat-core-rust/issues/331
-running "live" tests with temporary accounts
+    and
----------------------------------------------
+    https://github.com/deltachat/deltachat-core-rust/issues/326
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
-
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
-
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
-
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
-
-Each test run creates new accounts.
 
 
-.. _sourceinstall:
+running "live" tests (experimental)
+-----------------------------------
 
-Installing bindings from source (Updated: July 2020)
+If you want to run "liveconfig" functional tests you can set
-=========================================================
+``DCC_PY_LIVECONFIG`` to:
 
-Install Rust and Cargo first.
+- a particular https-url that you can ask for from the delta
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
+  chat devs.
 
-Bootstrap Rust and Cargo by using rustup::
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pwd=..." a test account login that will
+  be used for the live tests.
 
-   curl https://sh.rustup.rs -sSf | sh
+With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
 
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   python install_python_bindings.py
-
-The core compilation and bindings building might take a while,
-depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
 
 
 Code examples
@@ -124,11 +115,15 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux based wheels
+Building manylinux1 wheels
-====================================
+==========================
 
-Building portable manylinux wheels which come with libdeltachat.so
+.. note::
-can be done with docker-tooling.
+
+   This section may not fully work.
+
+Building portable manylinux1 wheels which come with libdeltachat.so
+and all it's dependencies is easy using the provided docker tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -141,9 +136,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ docker run -e DCC_NEW_TMP_EMAIL \
+    $ bash ci_scripts/ci_run.sh
-       --rm -it -v \$(pwd):/mnt -w /mnt \
+
-       deltachat/coredeps scripts/run_all.sh
+This command runs tests and build-wheel scripts in a docker container.
 
 
 Optionally build your own docker image
@@ -152,9 +147,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/doc/_static/custom.css

@@ -15,7 +15,3 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
-
-div.footer {
-    display: none;
-}

python/doc/_templates/globaltoc.html

@@ -6,10 +6,12 @@
   <li><a href="{{ pathto('install') }}">install</a></li>
   <li><a href="{{ pathto('api') }}">high level API</a></li>
   <li><a href="{{ pathto('lapi') }}">low level API</a></li>
+  <li><a href="{{ pathto('capi') }}">C deltachat.h</a></li>
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
+  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/doc/api.rst

@@ -2,10 +2,14 @@
 high level API reference
 ========================
 
+.. note::
+
+    This API is work in progress and may change in versions prior to 1.0.
+
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
-- :class:`deltachat.contact.Contact`
+- :class:`deltachat.chatting.Contact`
-- :class:`deltachat.chat.Chat`
+- :class:`deltachat.chatting.Chat`
 - :class:`deltachat.message.Message`
 
 Account
@@ -18,13 +22,13 @@ Account
 Contact
 -------
 
-.. autoclass:: deltachat.contact.Contact
+.. autoclass:: deltachat.chatting.Contact
     :members:
 
 Chat
 ----
 
-.. autoclass:: deltachat.chat.Chat
+.. autoclass:: deltachat.chatting.Chat
     :members:
 
 Message

python/doc/capi.rst

@@ -0,0 +1,7 @@
+
+C deltachat interface
+=====================
+
+See :doc:`lapi` for accessing many of the below functions
+through the ``deltachat.capi.lib`` namespace.
+

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+copyright = u'2018, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,60 +1,37 @@
 
+
 examples
 ========
 
+
+Playing around on the commandline
+----------------------------------
+
 Once you have :doc:`installed deltachat bindings <install>`
-you need email/password credentials for an IMAP/SMTP account.
+you can start playing from the python interpreter commandline.
-Delta Chat developers and the CI system use a special URL to create
+For example you can type ``python`` and then::
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
-Receiving a Chat message from the command line
+    # instantiate and configure deltachat account
-----------------------------------------------
+    import deltachat
+    ac = deltachat.Account("/tmp/db")
 
-Here is a simple bot that:
+    # start configuration activity and smtp/imap threads
+    ac.start_threads()
+    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
 
-- receives a message and sends back ("echoes") a message
+    # create a contact and send a message
+    contact = ac.create_contact("someother@email.address")
+    chat = ac.create_chat_by_contact(contact)
+    chat.send_text("hi from the python interpreter command line")
 
-- terminates the bot if the message `/quit` is sent
+Checkout our :doc:`api` for the various high-level things you can do
+to send/receive messages, create contacts and chats.
 
-.. include:: ../examples/echo_and_quit.py
-    :literal:
 
-With this file in your working directory you can run the bot
+Looking at a real example
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    $ cd examples
-    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
-
-While this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Track member additions and removals in a group
-----------------------------------------------
-
-Here is a simple bot that:
-
-- echoes messages sent to it
-
-- tracks if configuration completed
-
-- tracks member additions and removals for all chat groups
-
-.. include:: ../examples/group_tracking.py
-    :literal:
-
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
-
-When this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Writing bots for real
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a little framework for writing deltachat bots in Python.
+contains a real-life example of Python bindings usage.
+
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two layers of bindings for the
+The ``deltachat`` Python package provides two bindings for the core Rust-library
-core Rust-library of the https://delta.chat messaging ecosystem:
+of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core.
+- :doc:`api` is a high level interface to deltachat-core which aims
+  to be memory safe and thoroughly tested through continous tox/pytest runs.
 
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
+- :doc:`capi` is a lowlevel CFFI-binding to the previous
-
+  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
+  replicates exactly the same C-level API).
-  <https://github.com/deltachat/deltachat-core-rust>`_.
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
+   capi
    lapi
-   plugins
 
 ..
     Indices and tables

python/doc/lapi.rst

@@ -2,13 +2,7 @@
 low level API reference
 ===================================
 
-for full doxygen-generated C-docs, defines and functions please checkout
+for full C-docs, defines and function checkout :doc:`capi`
-
-https://c.delta.chat
-
-
-Python low-level capi calls
----------------------------
 
 
 .. automodule:: deltachat.capi.lib

python/doc/plugins.rst

@@ -1,38 +0,0 @@
-
-Implementing Plugin Hooks
-==========================
-
-The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
-for managing global and per-account plugin registration, and performing
-hook calls. There are two kinds of plugins:
-
-- Global plugins that are active for all accounts; they can implement
-  hooks at account-creation and account-shutdown time.
-
-- Account plugins that are only active during the lifetime of a
-  single Account instance.
-
-
-Registering a plugin
---------------------
-
-.. autofunction:: deltachat.register_global_plugin
-    :noindex:
-
-.. automethod:: deltachat.account.Account.add_account_plugin
-    :noindex:
-
-
-Per-Account Hook specifications
--------------------------------
-
-.. autoclass:: deltachat.hookspec.PerAccount
-    :members:
-
-
-Global Hook specifications
---------------------------
-
-.. autoclass:: deltachat.hookspec.Global
-    :members:
-

python/examples/echo_and_quit.py

@@ -1,33 +0,0 @@
-
-# content of echo_and_quit.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class EchoPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.create_chat()
-            addr = message.get_sender_contact().addr
-            if message.is_system_message():
-                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
-            else:
-                text = message.text
-                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_message_delivered(self, message):
-        print("ac_message_delivered", message)
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/group_tracking.py

@@ -1,52 +0,0 @@
-
-# content of group_tracking.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class GroupTrackingPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.create_chat()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
-
-    @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_added(self, chat, contact, actor, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, actor, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/test_examples.py

@@ -1,79 +0,0 @@
-
-import pytest
-import py
-import echo_and_quit
-import group_tracking
-from deltachat.events import FFIEventLogger
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-def test_echo_quit_plugin(acfactory, lp):
-    lp.sec("creating one echo_and_quit bot")
-    botproc = acfactory.run_bot_process(echo_and_quit)
-
-    lp.sec("creating a temp account to contact the bot")
-    ac1 = acfactory.get_one_online_account()
-
-    lp.sec("sending a message to the bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    bot_chat = bot_contact.create_chat()
-    bot_chat.send_text("hello")
-
-    lp.sec("waiting for the reply message from the bot to arrive")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.chat == bot_chat
-    assert "hello" in reply.text
-    lp.sec("send quit sequence")
-    bot_chat.send_text("/quit")
-    botproc.wait()
-
-
-def test_group_tracking_plugin(acfactory, lp):
-    lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
-
-    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
-
-    botproc.fnmatch_lines("""
-        *ac_configure_completed*
-    """)
-    ac1.add_account_plugin(FFIEventLogger(ac1))
-    ac2.add_account_plugin(FFIEventLogger(ac2))
-
-    lp.sec("creating bot test group with bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch = ac1.create_group_chat("bot test group")
-    ch.add_contact(bot_contact)
-    ch.send_text("hello")
-
-    botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
-
-    lp.sec("adding third member {}".format(ac2.get_config("addr")))
-    contact3 = ac1.create_contact(ac2.get_config("addr"))
-    ch.add_contact(contact3)
-
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-
-    lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
-        *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
-
-    lp.sec("contact successfully added, now removing")
-    ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
-        *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -6,27 +6,20 @@
 
 import os
 import subprocess
-import sys
+import os
 
 if __name__ == "__main__":
-    target = os.environ.get("DCC_RS_TARGET")
+    os.environ["DCC_RS_TARGET"] = target = "release"
-    if target is None:
-        os.environ["DCC_RS_TARGET"] = target = "debug"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+    os.environ["RUSTFLAGS"] = "-g"
-
+    subprocess.check_call([
-    if target == 'release':
+        "cargo", "build", "-p", "deltachat_ffi", "--" + target
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
+    ])
-        cmd.append("--release")
-
-    print("running:", " ".join(cmd))
-    subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
-    if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
+    subprocess.check_call([
-        subprocess.check_call([
+        "pip", "install", "-e", "."
-            sys.executable, "-m", "pip", "install", "-e", "."
+    ])
-        ])

python/setup.py

@@ -13,20 +13,14 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
-            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        install_requires=['cffi>=1.0.0', 'six'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
-            ],
-        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,13 +1,6 @@
-import sys
+from deltachat import capi, const
-
+from deltachat.capi import ffi
-from . import capi, const, hookspec # noqa
+from deltachat.account import Account  # noqa
-from .capi import ffi  # noqa
-from .account import Account  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import events
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -17,73 +10,67 @@ except DistributionNotFound:
     __version__ = "0.0.0.dev0-unknown"
 
 
+_DC_CALLBACK_MAP = {}
+
+
+@capi.ffi.def_extern()
+def py_dc_callback(ctx, evt, data1, data2):
+    """The global event handler.
+
+    CFFI only allows us to set one global event handler, so this one
+    looks up the correct event handler for the given context.
+    """
+    try:
+        callback = _DC_CALLBACK_MAP.get(ctx, lambda *a: 0)
+    except AttributeError:
+        # we are in a deep in GC-free/interpreter shutdown land
+        # nothing much better to do here than:
+        return 0
+
+    # the following code relates to the deltachat/_build.py's helper
+    # function which provides us signature info of an event call
+    evt_name = get_dc_event_name(evt)
+    event_sig_types = capi.lib.dc_get_event_signature_types(evt)
+    if data1 and event_sig_types & 1:
+        data1 = ffi.string(ffi.cast('char*', data1)).decode("utf8")
+    if data2 and event_sig_types & 2:
+        data2 = ffi.string(ffi.cast('char*', data2)).decode("utf8")
+        try:
+            if isinstance(data2, bytes):
+                data2 = data2.decode("utf8")
+        except UnicodeDecodeError:
+            # XXX ignoring the decode error is not quite correct but for now
+            # i don't want to hunt down encoding problems in the c lib
+            pass
+    try:
+        ret = callback(ctx, evt_name, data1, data2)
+        if ret is None:
+            ret = 0
+        assert isinstance(ret, int), repr(ret)
+        if event_sig_types & 4:
+            return ffi.cast('uintptr_t', ret)
+        elif event_sig_types & 8:
+            return ffi.cast('int', ret)
+    except:  # noqa
+        raise
+        ret = 0
+    return ret
+
+
+def set_context_callback(dc_context, func):
+    _DC_CALLBACK_MAP[dc_context] = func
+
+
+def clear_context_callback(dc_context):
+    try:
+        _DC_CALLBACK_MAP.pop(dc_context, None)
+    except AttributeError:
+        pass
+
+
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
         for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
-
-
-def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
-    of the :class:`deltachat.hookspec.Global` hooks.
-    """
-    gm = hookspec.Global._get_plugin_manager()
-    gm.register(plugin)
-    gm.check_pending()
-
-
-def unregister_global_plugin(plugin):
-    gm = hookspec.Global._get_plugin_manager()
-    gm.unregister(plugin)
-
-
-register_global_plugin(events)
-
-
-def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
-    import argparse
-    if argv is None:
-        argv = sys.argv
-
-    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
-    parser.add_argument("db", action="store", help="database file")
-    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
-    parser.add_argument("--email", action="store", help="email address")
-    parser.add_argument("--password", action="store", help="password")
-
-    args = parser.parse_args(argv[1:])
-
-    ac = Account(args.db)
-
-    if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
-        ac.add_account_plugin(log)
-
-    for plugin in account_plugins or []:
-        print("adding plugin", plugin)
-        ac.add_account_plugin(plugin)
-
-    if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
-        ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
-
-    # start IO threads and configure if neccessary
-    ac.start_io()
-
-    print("{}: waiting for message".format(ac.get_config("addr")))
-
-    ac.wait_shutdown()

python/src/deltachat/_build.py

@@ -1,63 +1,76 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import os
-import platform
-import re
-import shutil
-import subprocess
 import tempfile
-import textwrap
+import platform
-import types
+import os
-
 import cffi
+import shutil
+from os.path import dirname as dn
+from os.path import abspath
 
 
-def local_build_flags(projdir, target):
+def ffibuilder():
-    """Construct build flags for building against a checkout.
+    projdir = os.environ.get('DCC_RS_DEV')
-
+    if not projdir:
-    :param projdir: The root directory of the deltachat-core-rust project.
+        p = dn(dn(dn(dn(abspath(__file__)))))
-    :param target: The rust build target, `debug` or `release`.
+        projdir = os.environ["DCC_RS_DEV"] = p
-    """
+    target = os.environ.get('DCC_RS_TARGET', 'release')
-    flags = types.SimpleNamespace()
+    if projdir:
-    if platform.system() == 'Darwin':
+        if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
+            libs = ['resolv', 'dl']
-        flags.extra_link_args = [
+            extra_link_args = [
-                '-framework', 'CoreFoundation',
+                    '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
+                    '-framework', 'CoreServices',
-                '-framework', 'Security',
+                    '-framework', 'Security',
-        ]
+            ]
-    elif platform.system() == 'Linux':
+        elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
+            libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+            extra_link_args = []
+        else:
+            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
-        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        libs = ['deltachat']
-    target_dir = os.environ.get("CARGO_TARGET_DIR")
+        objs = []
-    if target_dir is None:
+        incs = []
-        target_dir = os.path.join(projdir, 'target')
+        extra_link_args = []
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    builder = cffi.FFI()
-    assert os.path.exists(flags.objs[0]), flags.objs
+    builder.set_source(
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+        'deltachat.capi',
-    return flags
+        """
-
+            #include <deltachat.h>
-
+            const char * dupstring_helper(const char* string)
-def system_build_flags():
+            {
-    """Construct build flags for building against an installed libdeltachat."""
+                return strdup(string);
-    flags = types.SimpleNamespace()
+            }
-    flags.libs = ['deltachat']
+            int dc_get_event_signature_types(int e)
-    flags.objs = []
+            {
-    flags.incs = []
+                int result = 0;
-    flags.extra_link_args = []
+                if (DC_EVENT_DATA1_IS_STRING(e))
-    return flags
+                    result |= 1;
-
+                if (DC_EVENT_DATA2_IS_STRING(e))
-
+                    result |= 2;
-def extract_functions(flags):
+                if (DC_EVENT_RETURNS_STRING(e))
-    """Extract the function definitions from deltachat.h.
+                    result |= 4;
-
+                if (DC_EVENT_RETURNS_INT(e))
-    This creates a .h file with a single `#include <deltachat.h>` line
+                    result |= 8;
-    in it.  It then runs the C preprocessor to create an output file
+                return result;
-    which contains all function definitions found in `deltachat.h`.
+            }
-    """
+        """,
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
+        extra_link_args=extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern const char * dupstring_helper(const char* string);
+        extern int dc_get_event_signature_types(int);
+    """)
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -69,134 +82,20 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            return dst_fp.read()
+            builder.cdef(dst_fp.read())
     finally:
         shutil.rmtree(tmpdir)
 
-
-def find_header(flags):
-    """Use the compiler to find the deltachat.h header location.
-
-    This uses a small utility in deltachat.h to find the location of
-    the header file location.
-    """
-    distutils.log.set_verbosity(distutils.log.INFO)
-    cc = distutils.ccompiler.new_compiler(force=True)
-    distutils.sysconfig.customize_compiler(cc)
-    tmpdir = tempfile.mkdtemp()
-    try:
-        src_name = os.path.join(tmpdir, "where.c")
-        obj_name = os.path.join(tmpdir, "where.o")
-        dst_name = os.path.join(tmpdir, "where")
-        with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
-                #include <stdio.h>
-                #include <deltachat.h>
-
-                int main(void) {
-                    printf("%s", _dc_header_file_location());
-                    return 0;
-                }
-            """))
-        cwd = os.getcwd()
-        try:
-            os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
-                       macros=[("PY_CFFI_INC", "1")])
-        finally:
-            os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
-        return subprocess.check_output(dst_name)
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-def extract_defines(flags):
-    """Extract the required #DEFINEs from deltachat.h.
-
-    Since #DEFINEs are interpreted by the C preprocessor we can not
-    use the compiler to extract these and need to parse the header
-    file ourselves.
-
-    The defines are returned in a string that can be passed to CFFIs
-    cdef() method.
-    """
-    header = find_header(flags)
-    defines_re = re.compile(r"""
-        \#define\s+   # The start of a define.
-        (             # Begin capturing group which captures the define name.
-            (?:       # A nested group which is not captured, this allows us
-                      #    to build the list of prefixes to extract without
-                      #    creation another capture group.
-                DC_EVENT
-                | DC_QR
-                | DC_MSG
-                | DC_LP
-                | DC_EMPTY
-                | DC_CERTCK
-                | DC_STATE
-                | DC_STR
-                | DC_CONTACT_ID
-                | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
-                | DC_CHAT
-                | DC_PROVIDER
-                | DC_KEY_GEN
-                | DC_IMEX
-                | DC_CONNECTIVITY
-            )         # End of prefix matching
-            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
-        )             # Close the capturing group, this contains
-                      # the entire name e.g. DC_MSG_TEXT.
-        \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
-    defines = []
-    with open(header) as fp:
-        for line in fp:
-            match = defines_re.match(line)
-            if match:
-                defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
-
-
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
-        flags = local_build_flags(projdir, target)
-    else:
-        flags = system_build_flags()
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
-    )
     builder.cdef("""
-        typedef int... time_t;
+        extern "Python" uintptr_t py_dc_callback(
-        void free(void *ptr);
+            dc_context_t* context,
-        extern int dc_event_has_string_data(int);
+            int event,
+            uintptr_t data1,
+            uintptr_t data2);
     """)
-    function_defs = extract_functions(flags)
-    defines = extract_defines(flags)
-    builder.cdef(function_defs)
-    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -1,77 +1,57 @@
 """ Account class implementation. """
 
 from __future__ import print_function
-from contextlib import contextmanager
+import threading
-from email.utils import parseaddr
-from threading import Event
 import os
+import re
+import time
 from array import array
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
+
+import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
-from .chat import Chat
+from .chatting import Contact, Chat, Message
-from .message import Message
-from .contact import Contact
-from .tracker import ImexTracker, ConfigureTracker
-from . import hookspec
-from .events import EventThread
-
-
-class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
-    by the underlying deltachat core library.  All public Account methods are
+    by the underlying deltachat c-library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    MissingCredentials = MissingCredentials
+    def __init__(self, db_path, logid=None, eventlogging=True):
-
-    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        :param eventlogging: if False no eventlogging and no context callback will be configured
         """
-        # initialize per-account plugin system
+        self._dc_context = ffi.gc(
-        self._pm = hookspec.PerAccount._make_plugin_manager()
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        self._logging = logging
+            _destroy_dc_context,
+        )
+        if eventlogging:
+            self._evlogger = EventLogger(self._dc_context, logid)
+            deltachat.set_context_callback(self._dc_context, self._process_event)
+            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
+        else:
+            self._threads = IOThreads(self._dc_context)
 
-        self.add_account_plugin(self)
-
-        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
-
+        if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
-        self._dc_context = ffi.gc(
+            raise ValueError("Could not dc_open: {}".format(db_path))
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
-            lib.dc_context_unref,
-        )
-        if self._dc_context == ffi.NULL:
-            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
-
-        self._shutdown_event = Event()
-        self._event_thread = EventThread(self)
         self._configkeys = self.get_config("sys.config_keys").split()
-        hook = hookspec.Global._get_plugin_manager().hook
+        self._imex_completed = threading.Event()
-        hook.dc_account_init(account=self)
 
-    def disable_logging(self):
+    def __del__(self):
-        """ disable logging. """
+        self.shutdown()
-        self._logging = False
-
-    def enable_logging(self):
-        """ re-enable logging. """
-        self._logging = True
-
-    # def __del__(self):
-    #    self.shutdown()
-
-    def log(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -89,34 +69,6 @@ class Account(object):
             d[key.lower()] = value
         return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
-    def set_stock_translation(self, id, string):
-        """ set stock translation string.
-
-        :param id: id of stock string (const.DC_STR_*)
-        :param value: string to set as new transalation
-        :returns: None
-        """
-        string = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, string)
-        if res == 0:
-            raise ValueError("could not set translation string")
-
     def set_config(self, name, value):
         """ set configuration values.
 
@@ -126,12 +78,9 @@ class Account(object):
         """
         self._check_config_key(name)
         name = name.encode("utf8")
+        value = value.encode("utf8")
         if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
-        if value is not None:
-            value = value.encode("utf8")
-        else:
-            value = ffi.NULL
         lib.dc_set_config(self._dc_context, name, value)
 
     def get_config(self, name):
@@ -148,27 +97,16 @@ class Account(object):
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
+    def configure(self, **kwargs):
-        """See dc_preconfigure_keypair() in deltachat.h.
+        """ set config values and configure this account.
-
-        In other words, you don't need this.
-        """
-        res = lib.dc_preconfigure_keypair(self._dc_context,
-                                          as_dc_charpointer(addr),
-                                          as_dc_charpointer(public),
-                                          as_dc_charpointer(secret))
-        if res == 0:
-            raise Exception("Failed to set key")
-
-    def update_config(self, kwargs):
-        """ update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for key, value in kwargs.items():
+        for name, value in kwargs.items():
-            self.set_config(key, str(value))
+            self.set_config(name, value)
+        lib.dc_configure(self._dc_context)
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -176,32 +114,17 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return lib.dc_is_configured(self._dc_context)
-
-    def set_avatar(self, img_path):
-        """Set self avatar.
-
-        :raises ValueError: if profile image could not be set
-        :returns: None
-        """
-        if img_path is None:
-            self.set_config("selfavatar", None)
-        else:
-            assert os.path.exists(img_path), img_path
-            self.set_config("selfavatar", img_path)
 
     def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir):
+    def get_infostring(self):
-        """ return the latest backup file in a given directory.
+        """ return info of the configured account. """
-        """
+        self.check_is_configured()
-        res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
+        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
-        if res == ffi.NULL:
-            return None
-        return from_dc_charpointer(res)
 
     def get_blobdir(self):
         """ return the directory for files.
@@ -212,53 +135,27 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self):
-        """ return this account's identity as a :class:`deltachat.contact.Contact`.
+        """ return this account's identity as a :class:`deltachat.chatting.Contact`.
 
-        :returns: :class:`deltachat.contact.Contact`
+        :returns: :class:`deltachat.chatting.Contact`
         """
-        return Contact(self, const.DC_CONTACT_ID_SELF)
+        self.check_is_configured()
+        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
+    def create_contact(self, email, name=None):
-        """ create a (new) Contact or return an existing one.
+        """ create a (new) Contact. If there already is a Contact
+        with that e-mail address, it is unblocked and its name is
+        updated.
 
-        Calling this method will always result in the same
+        :param email: email-address (text type)
-        underlying contact id.  If there already is a Contact
+        :param name: display name for this contact (optional)
-        with that e-mail address, it is unblocked and its display
+        :returns: :class:`deltachat.chatting.Contact` instance.
-        `name` is updated if specified.
-
-        :param obj: email-address, Account or Contact instance.
-        :param name: (optional) display name for this contact
-        :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
         name = as_dc_charpointer(name)
-        addr = as_dc_charpointer(addr)
+        email = as_dc_charpointer(email)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name, email)
-        return Contact(self, contact_id)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-
+        return Contact(self._dc_context, contact_id)
-    def get_contact(self, obj):
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name=None):
-        if isinstance(obj, Account):
-            if not obj.is_configured():
-                raise ValueError("can only add addresses from configured accounts")
-            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
-        elif isinstance(obj, Contact):
-            if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
-            addr, displayname = obj.addr, obj.name
-        elif isinstance(obj, str):
-            displayname, addr = parseaddr(obj)
-        else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
-
-        if name is None and displayname:
-            name = displayname
-        return (name, addr)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -267,36 +164,10 @@ class Account(object):
         :returns: True if deletion succeeded (contact was deleted)
         """
         contact_id = contact.id
-        assert contact.account == self
+        assert contact._dc_context == self._dc_context
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
-    def get_blocked_contacts(self):
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -304,7 +175,7 @@ class Account(object):
                       whose name or e-mail matches query.
         :param only_verified: if true only return verified contacts.
         :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.contact.Contact` objects.
+        :returns: list of :class:`deltachat.chatting.Contact` objects.
         """
         flags = 0
         query = as_dc_charpointer(query)
@@ -316,41 +187,57 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
 
-    def get_fresh_messages(self):
+    def create_chat_by_contact(self, contact):
-        """ yield all fresh messages from all chats. """
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
+        :param contact: chat_id (int) or contact object.
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        :returns: a :class:`deltachat.chatting.Chat` object.
-        return self.create_contact(obj).create_chat()
+        """
+        if hasattr(contact, "id"):
+            if contact._dc_context != self._dc_context:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
+        return Chat(self, chat_id)
 
-    def create_group_chat(self, name, contacts=None, verified=False):
+    def create_chat_by_message(self, message):
+        """ create or get an existing chat object for the
+        the specified message.
+
+        :param message: messsage id or message instance.
+        :returns: a :class:`deltachat.chatting.Chat` object.
+        """
+        if hasattr(message, "id"):
+            if self._dc_context != message._dc_context:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
+        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
+        return Chat(self, chat_id)
+
+    def create_group_chat(self, name, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
-        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
-        :returns: a :class:`deltachat.chat.Chat` object.
+        :returns: a :class:`deltachat.chatting.Chat` object.
         """
         bytes_name = name.encode("utf8")
-        chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
+        chat_id = lib.dc_create_group_chat(self._dc_context, verified, bytes_name)
-        chat = Chat(self, chat_id)
+        return Chat(self, chat_id)
-        if contacts is not None:
-            for contact in contacts:
-                chat.add_contact(contact)
-        return chat
 
     def get_chats(self):
         """ return list of chats.
 
-        :returns: a list of :class:`deltachat.chat.Chat` objects.
+        :returns: a list of :class:`deltachat.chatting.Chat` objects.
         """
         dc_chatlist = ffi.gc(
             lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
@@ -364,28 +251,13 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self):
+    def get_deaddrop_chat(self):
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
-        """ return Message instance.
+        """ return Message instance. """
-        :param msg_id: integer id of this message.
-        :returns: :class:`deltachat.message.Message` instance.
-        """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id):
-        """ return Chat instance.
-        :param chat_id: integer id of this chat.
-        :returns: :class:`deltachat.chat.Chat` instance.
-        :raises: ValueError if chat does not exist.
-        """
-        res = lib.dc_get_chat(self._dc_context, chat_id)
-        if res == ffi.NULL:
-            raise ValueError("cannot get chat with id={}".format(chat_id))
-        lib.dc_chat_unref(res)
-        return Chat(self, chat_id)
-
     def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
@@ -402,7 +274,7 @@ class Account(object):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
-        :param chat: :class:`deltachat.chat.Chat` object.
+        :param chat: :class:`deltachat.chatting.Chat` object.
         :returns: None
         """
         msg_ids = [msg.id for msg in messages]
@@ -417,54 +289,31 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
-    def export_self_keys(self, path):
+    def export_to_dir(self, backupdir):
-        """ export public and private keys to the specified directory.
+        """return after all delta chat state is exported to a new file in
-
+        the specified directory.
-        Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        snap_files = os.listdir(backupdir)
+        self._imex_completed.clear()
+        lib.dc_imex(self._dc_context, 11, as_dc_charpointer(backupdir), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        self._imex_completed.wait()
+        for x in os.listdir(backupdir):
+            if x not in snap_files:
+                return os.path.join(backupdir, x)
 
-    def export_all(self, path):
+    def import_from_file(self, path):
-        """return new file containing a backup of all database state
+        """import delta chat state from the specified backup file.
-        (chats, contacts, keys, media, ...). The file is created in the
-        the `path` directory.
-
-        Note that the account has to be stopped; call stop_io() if necessary.
-        """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
-        if len(export_files) != 1:
-            raise RuntimeError("found more than one new file")
-        return export_files[0]
-
-    def _export(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
-            return imex_tracker.wait_finish()
-
-    def import_self_keys(self, path):
-        """ Import private keys found in the `path` directory.
-        The last imported key is made the default keys unless its name
-        contains the string legacy. Public keys are not imported.
-
-        Note that the account does not have to be started.
-        """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
-
-    def import_all(self, path):
-        """import delta chat state from the specified backup `path` (a file).
 
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._imex_completed.clear()
-
+        lib.dc_imex(self._dc_context, 12, as_dc_charpointer(path), ffi.NULL)
-    def _import(self, path, imex_cmd):
+        if not self._threads.is_started():
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
+            lib.dc_perform_imap_jobs(self._dc_context)
-            self.imex(path, imex_cmd)
+        self._imex_completed.wait()
-            imex_tracker.wait_finish()
-
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -472,6 +321,8 @@ class Account(object):
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self._threads.is_started():
+            raise RuntimeError("threads not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -502,7 +353,7 @@ class Account(object):
         """ setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
         is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
@@ -516,7 +367,7 @@ class Account(object):
         """ join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
         is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
@@ -527,158 +378,174 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+    def start_threads(self):
-        """set a new location. It effects all chats where we currently
+        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
-        have enabled location streaming.
-
-        :param latitude: float (use 0.0 if not known)
-        :param longitude: float (use 0.0 if not known)
-        :param accuracy: float (use 0.0 if not known)
-        :raises: ValueError if no chat is currently streaming locations
-        :returns: None
-        """
-        dc_res = lib.dc_set_location(self._dc_context, latitude, longitude, accuracy)
-        if dc_res == 0:
-            raise ValueError("no chat is streaming locations")
-
-    #
-    # meta API for start/stop and event based processing
-    #
-
-    def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
-        the :class:`deltachat.hookspec.PerAccount` hooks.
-        """
-        self._pm.register(plugin, name=name)
-        self._pm.check_pending()
-        return plugin
-
-    def remove_account_plugin(self, plugin, name=None):
-        """ remove an account plugin. """
-        self._pm.unregister(plugin, name=name)
-
-    @contextmanager
-    def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
-        self._pm.register(plugin)
-        yield plugin
-        self._pm.unregister(plugin)
-
-    def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
-        lib.dc_stop_ongoing_process(self._dc_context)
-
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self):
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
-    def start_io(self):
-        """ start this account's IO scheduling (Rust-core async scheduler)
-
-        If this account is not configured an Exception is raised.
-        You need to call account.configure() and account.wait_configure_finish()
-        before.
-
-        You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
-        account is started.
-
-        If you are using this from a test, you may want to call
-        wait_all_initial_fetches() afterwards.
-
-        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
-        :raises ConfigureFailed: if the account could not be configured.
 
+        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
         :returns: None
         """
         if not self.is_configured():
-            raise ValueError("account not configured, cannot start io")
+            self.configure()
-        lib.dc_start_io(self._dc_context)
+        self._threads.start()
 
-    def maybe_network(self):
+    def stop_threads(self, wait=True):
-        """This function should be called when there is a hint
+        """ stop IMAP/SMTP threads. """
-        that the network is available again,
+        lib.dc_stop_ongoing_process(self._dc_context)
-        e.g. as a response to system event reporting network availability.
+        self._threads.stop(wait=wait)
-        The library will try to send pending messages out immediately.
 
-        Moreover, to have a reliable state
+    def shutdown(self, wait=True):
-        when the app comes to foreground with network available,
+        """ stop threads and close and remove underlying dc_context and callbacks. """
-        it may be reasonable to call the function also at that moment.
+        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
+            # print("SHUTDOWN", self)
+            self.stop_threads(wait=False)
+            lib.dc_close(self._dc_context)
+            self.stop_threads(wait=wait)  # to wait for threads
+            deltachat.clear_context_callback(self._dc_context)
+            del self._dc_context
 
-        It is okay to call the function unconditionally when there is
+    def _process_event(self, ctx, evt_name, data1, data2):
-        network available, however, calling the function
+        assert ctx == self._dc_context
-        _without_ having network may interfere with the backoff algorithm
+        if hasattr(self, "_evlogger"):
-        and will led to let the jobs fail faster, with fewer retries
+            self._evlogger(evt_name, data1, data2)
-        and may avoid messages being sent out.
+            method = getattr(self, "on_" + evt_name.lower(), None)
+            if method is not None:
+                method(data1, data2)
+        return 0
 
-        Finally, if the context was created by the dc_accounts_t account manager
+    def on_dc_event_imex_progress(self, data1, data2):
-        (currently not implemented in the Python bindings),
+        if data1 == 1000:
-        use dc_accounts_maybe_network() instead of this function
+            self._imex_completed.set()
-        """
-        lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure=False):
-        """ Start configuration process and return a Configtracker instance
-        on which you can block with wait_finish() to get a True/False success
-        value for the configuration process.
-        """
-        assert self.is_configured() == reconfigure
-        if not self.get_config("addr") or not self.get_config("mail_pw"):
-            raise MissingCredentials("addr or mail_pwd not set in config")
-        configtracker = ConfigureTracker(self)
-        self.add_account_plugin(configtracker)
-        lib.dc_configure(self._dc_context)
-        return configtracker
 
-    def wait_shutdown(self):
+class IOThreads:
-        """ wait until shutdown of this account has completed. """
+    def __init__(self, dc_context, log_event=lambda *args: None):
-        self._shutdown_event.wait()
+        self._dc_context = dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+        self._log_event = log_event
 
-    def stop_io(self):
+    def is_started(self):
-        """ stop core IO scheduler if it is running. """
+        return len(self._name2thread) > 0
-        self.log("stop_ongoing")
-        self.stop_ongoing()
 
-        self.log("dc_stop_io (stop core IO scheduler)")
+    def start(self, imap=True, smtp=True):
-        lib.dc_stop_io(self._dc_context)
+        assert not self.is_started()
+        if imap:
+            self._start_one_thread("imap", self.imap_thread_run)
+        if smtp:
+            self._start_one_thread("smtp", self.smtp_thread_run)
 
-    def shutdown(self):
+    def _start_one_thread(self, name, func):
-        """ shutdown and destroy account (stop callback thread, close and remove
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        underlying dc_context)."""
+        t.setDaemon(1)
-        if self._dc_context is None:
+        t.start()
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                thread.join()
+
+    def imap_thread_run(self):
+        self._log_event("py-bindings-info", 0, "IMAP THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_imap_jobs(self._dc_context)
+            lib.dc_perform_imap_fetch(self._dc_context)
+            lib.dc_perform_imap_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "IMAP THREAD FINISHED")
+
+    def smtp_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_smtp_jobs(self._dc_context)
+            lib.dc_perform_smtp_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
+
+
+class EventLogger:
+    _loglock = threading.RLock()
+
+    def __init__(self, dc_context, logid=None, debug=True):
+        self._dc_context = dc_context
+        self._event_queue = Queue()
+        self._debug = debug
+        if logid is None:
+            logid = str(self._dc_context).strip(">").split()[-1]
+        self.logid = logid
+        self._timeout = None
+        self.init_time = time.time()
+
+    def __call__(self, evt_name, data1, data2):
+        self._log_event(evt_name, data1, data2)
+        self._event_queue.put((evt_name, data1, data2))
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get()
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout or self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev[0] == "DC_EVENT_ERROR":
+            raise ValueError("{}({!r},{!r})".format(*ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev[0]), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True):
+        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get()
+            if rex.match(ev[0]):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev[2]):
+                return ev
+
+    def _log_event(self, evt_name, data1, data2):
+        # don't show events that are anyway empty impls now
+        if evt_name == "DC_EVENT_GET_STRING":
             return
+        if self._debug:
+            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
+            self._log(evpart)
 
-        self.stop_io()
+    def _log(self, msg):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        with self._loglock:
+            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
 
-        self.log("remove dc_context references")
 
-        # if _dc_context is unref'ed the event thread should quickly
+def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
-        # receive the termination signal. However, some python code might
+    # destructor for dc_context
-        # still hold a reference and so we use a secondary signal
+    dc_context_unref(dc_context)
-        # to make sure the even thread terminates if it receives any new
+    try:
-        # event, indepedently from waiting for the core to send NULL to
+        deltachat.clear_context_callback(dc_context)
-        # get_next_event().
+    except (TypeError, AttributeError):
-        self._event_thread.mark_shutdown()
+        # we are deep into Python Interpreter shutdown,
-        self._dc_context = None
+        # so no need to clear the callback context mapping.
-
+        pass
-        self.log("wait for event thread to finish")
-        try:
-            self._event_thread.wait(timeout=2)
-        except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
-
-        if self._event_thread.is_alive():
-            self.log("WARN: event thread did not terminate yet, ignoring.")
-
-        self._shutdown_event.set()
-
-        hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self)
-        self.log("shutdown finished")
 
 
 class ScannedQRCode:

python/src/deltachat/chat.py

@@ -1,534 +0,0 @@
-""" Chat and Location related API. """
-
-import mimetypes
-import calendar
-import json
-from datetime import datetime
-import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
-from .capi import lib, ffi
-from . import const
-from .message import Message
-
-
-class Chat(object):
-    """ Chat object which manages members and through which you can send and retrieve messages.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-
-    def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
-        self.account = account
-        self.id = id
-
-    def __eq__(self, other):
-        return self.id == getattr(other, "id", None) and \
-               self.account._dc_context == other.account._dc_context
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
-
-    @property
-    def _dc_chat(self):
-        return ffi.gc(
-            lib.dc_get_chat(self.account._dc_context, self.id),
-            lib.dc_chat_unref
-        )
-
-    def delete(self):
-        """Delete this chat and all its messages.
-
-        Note:
-
-        - does not delete messages on server
-        - the chat or contact is not blocked, new message will arrive
-        """
-        lib.dc_delete_chat(self.account._dc_context, self.id)
-
-    def block(self):
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self):
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
-    # ------  chat status/metadata API ------------------------------
-
-    def is_group(self):
-        """ return true if this chat is a group chat.
-
-        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
-        """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
-
-    def is_muted(self):
-        """ return true if this chat is muted.
-
-        :returns: True if chat is muted, False otherwise.
-        """
-        return lib.dc_chat_is_muted(self._dc_chat)
-
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
-    def is_promoted(self):
-        """ return True if this chat is promoted, i.e.
-        the member contacts are aware of their membership,
-        have been sent messages.
-
-        :returns: True if chat is promoted, False otherwise.
-        """
-        return not lib.dc_chat_is_unpromoted(self._dc_chat)
-
-    def can_send(self):
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
-
-        :returns: True if the chat is writable, False otherwise
-        """
-        return lib.dc_chat_can_send(self._dc_chat)
-
-    def is_protected(self):
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
-
-    def get_name(self):
-        """ return name of this chat.
-
-        :returns: unicode name
-        """
-        return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
-
-    def set_name(self, name):
-        """ set name of this chat.
-
-        :param name: as a unicode string.
-        :returns: None
-        """
-        name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
-
-    def mute(self, duration=None):
-        """ mutes the chat
-
-        :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
-        :returns: None
-        """
-        if duration is None:
-            mute_duration = -1
-        else:
-            mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
-        if not bool(ret):
-            raise ValueError("Call to dc_set_chat_mute_duration failed")
-
-    def unmute(self):
-        """ unmutes the chat
-
-        :returns: None
-        """
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, 0)
-        if not bool(ret):
-            raise ValueError("Failed to unmute chat")
-
-    def get_mute_duration(self):
-        """ Returns the number of seconds until the mute of this chat is lifted.
-
-        :param duration:
-        :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
-        """
-        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
-
-    def get_ephemeral_timer(self):
-        """ get ephemeral timer.
-
-        :returns: ephemeral timer value in seconds
-        """
-        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
-
-    def set_ephemeral_timer(self, timer):
-        """ set ephemeral timer.
-
-        :param: timer value in seconds
-
-        :returns: None
-        """
-        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
-
-    def get_type(self):
-        """ (deprecated) return type of this chat.
-
-        :returns: one of const.DC_CHAT_TYPE_*
-        """
-        return lib.dc_chat_get_type(self._dc_chat)
-
-    def get_encryption_info(self):
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
-    def get_join_qr(self):
-        """ get/create Join-Group QR Code as ascii-string.
-
-        this string needs to be transferred to another DC account
-        in a second channel (typically used by mobiles with QRcode-show + scan UX)
-        where account.join_with_qrcode(qr) needs to be called.
-        """
-        res = lib.dc_get_securejoin_qr(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
-    # ------  chat messaging API ------------------------------
-
-    def send_msg(self, msg):
-        """send a message by using a ready Message object.
-
-        :param msg: a :class:`deltachat.message.Message` instance
-           previously returned by
-           e.g. :meth:`deltachat.message.Message.new_empty` or
-           :meth:`prepare_file`.
-        :raises ValueError: if message can not be sent.
-
-        :returns: a :class:`deltachat.message.Message` instance as
-           sent out.  This is the same object as was passed in, which
-           has been modified with the new state of the core.
-        """
-        if msg.is_out_preparing():
-            assert msg.id != 0
-            # get a fresh copy of dc_msg, the core needs it
-            msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
-        return msg
-
-    def send_text(self, text):
-        """ send a text message and return the resulting Message instance.
-
-        :param msg: unicode text
-        :raises ValueError: if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.message.Message` instance
-        """
-        msg = as_dc_charpointer(text)
-        msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
-        if msg_id == 0:
-            raise ValueError("message could not be send, does chat exist?")
-        return Message.from_db(self.account, msg_id)
-
-    def send_file(self, path, mime_type="application/octet-stream"):
-        """ send a file and return the resulting Message instance.
-
-        :param path: path to the file.
-        :param mime_type: the mime-type of this file, defaults to application/octet-stream.
-        :raises ValueError: if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.message.Message` instance
-        """
-        msg = Message.new_empty(self.account, view_type="file")
-        msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        return Message.from_db(self.account, sent_id)
-
-    def send_image(self, path):
-        """ send an image message and return the resulting Message instance.
-
-        :param path: path to an image file.
-        :raises ValueError: if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.message.Message` instance
-        """
-        mime_type = mimetypes.guess_type(path)[0]
-        msg = Message.new_empty(self.account, view_type="image")
-        msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        return Message.from_db(self.account, sent_id)
-
-    def prepare_message(self, msg):
-        """ prepare a message for sending.
-
-        :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
-        """
-        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if msg_id == 0:
-            raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
-
-    def prepare_message_file(self, path, mime_type=None, view_type="file"):
-        """ prepare a message for sending and return the resulting Message instance.
-
-        To actually send the message, call :meth:`send_prepared`.
-        The file must be inside the blob directory.
-
-        :param path: path to the file.
-        :param mime_type: the mime-type of this file, defaults to auto-detection.
-        :param view_type: "text", "image", "gif", "audio", "video", "file"
-        :raises ValueError: if message can not be prepared/chat does not exist.
-        :returns: the resulting :class:`Message` instance
-        """
-        msg = Message.new_empty(self.account, view_type)
-        msg.set_file(path, mime_type)
-        return self.prepare_message(msg)
-
-    def send_prepared(self, message):
-        """ send a previously prepared message.
-
-        :param message: a :class:`Message` instance previously returned by
-                        :meth:`prepare_file`.
-        :raises ValueError: if message can not be sent.
-        :returns: a :class:`deltachat.message.Message` instance as sent out.
-        """
-        assert message.id != 0 and message.is_out_preparing()
-        # get a fresh copy of dc_msg, the core needs it
-        msg = Message.from_db(self.account, message.id)
-
-        # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        assert sent_id == msg.id
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
-
-    def set_draft(self, message):
-        """ set message as draft.
-
-        :param message: a :class:`Message` instance
-        :returns: None
-        """
-        if message is None:
-            lib.dc_set_draft(self.account._dc_context, self.id, ffi.NULL)
-        else:
-            lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
-
-    def get_draft(self):
-        """ get draft message for this chat.
-
-        :param message: a :class:`Message` instance
-        :returns: Message object or None (if no draft available)
-        """
-        x = lib.dc_get_draft(self.account._dc_context, self.id)
-        if x == ffi.NULL:
-            return None
-        dc_msg = ffi.gc(x, lib.dc_msg_unref)
-        return Message(self.account, dc_msg)
-
-    def get_messages(self):
-        """ return list of messages in this chat.
-
-        :returns: list of :class:`deltachat.message.Message` objects for this chat.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
-
-    def count_fresh_messages(self):
-        """ return number of fresh messages in this chat.
-
-        :returns: number of fresh messages
-        """
-        return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
-
-    def mark_noticed(self):
-        """ mark all messages in this chat as noticed.
-
-        Noticed messages are no longer fresh.
-        """
-        return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
-
-    def get_summary(self):
-        """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
-        s = from_dc_charpointer(dc_res)
-        return json.loads(s)
-
-    # ------  group management API ------------------------------
-
-    def add_contact(self, obj):
-        """ add a contact to this chat.
-
-        :params obj: Contact, Account or e-mail address.
-        :raises ValueError: if contact could not be added
-        :returns: None
-        """
-        contact = self.account.create_contact(obj)
-        ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
-        if ret != 1:
-            raise ValueError("could not add contact {!r} to chat".format(contact))
-        return contact
-
-    def remove_contact(self, obj):
-        """ remove a contact from this chat.
-
-        :params obj: Contact, Account or e-mail address.
-        :raises ValueError: if contact could not be removed
-        :returns: None
-        """
-        contact = self.account.get_contact(obj)
-        ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
-        if ret != 1:
-            raise ValueError("could not remove contact {!r} from chat".format(contact))
-
-    def get_contacts(self):
-        """ get all contacts for this chat.
-        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-        """
-        from .contact import Contact
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
-        )
-
-    def num_contacts(self):
-        """ return number of contacts in this chat. """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return lib.dc_array_get_cnt(dc_array)
-
-    def set_profile_image(self, img_path):
-        """Set group profile image.
-
-        If the group is already promoted (any message was sent to the group),
-        all group members are informed by a special status message that is sent
-        automatically by this function.
-        :params img_path: path to image object
-        :raises ValueError: if profile image could not be set
-        :returns: None
-        """
-        assert os.path.exists(img_path), img_path
-        p = as_dc_charpointer(img_path)
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
-        if res != 1:
-            raise ValueError("Setting Profile Image {!r} failed".format(p))
-
-    def remove_profile_image(self):
-        """remove group profile image.
-
-        If the group is already promoted (any message was sent to the group),
-        all group members are informed by a special status message that is sent
-        automatically by this function.
-        :raises ValueError: if profile image could not be reset
-        :returns: None
-        """
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, ffi.NULL)
-        if res != 1:
-            raise ValueError("Removing Profile Image failed")
-
-    def get_profile_image(self):
-        """Get group profile image.
-
-        For groups, this is the image set by any group member using
-        set_chat_profile_image(). For normal chats, this is the image
-        set by each remote user on their own using dc_set_config(context,
-        "selfavatar", image).
-        :returns: path to profile image, None if no profile image exists.
-        """
-        dc_res = lib.dc_chat_get_profile_image(self._dc_chat)
-        if dc_res == ffi.NULL:
-            return None
-        return from_dc_charpointer(dc_res)
-
-    def get_color(self):
-        """return the color of the chat.
-        :returns: color as 0x00rrggbb
-        """
-        return lib.dc_chat_get_color(self._dc_chat)
-
-    # ------  location streaming API ------------------------------
-
-    def is_sending_locations(self):
-        """return True if this chat has location-sending enabled currently.
-        :returns: True if location sending is enabled.
-        """
-        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
-
-    def is_archived(self):
-        """return True if this chat is archived.
-        :returns: True if archived.
-        """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
-
-    def enable_sending_locations(self, seconds):
-        """enable sending locations for this chat.
-
-        all subsequent messages will carry a location with them.
-        """
-        lib.dc_send_locations_to_chat(self.account._dc_context, self.id, seconds)
-
-    def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
-        """return list of locations for the given contact in the given timespan.
-
-        :param contact: the contact for which locations shall be returned.
-        :param timespan_from: a datetime object or None (indicating "since beginning")
-        :param timespan_to: a datetime object or None (indicating up till now)
-        :returns: list of :class:`deltachat.chat.Location` objects.
-        """
-        if timestamp_from is None:
-            time_from = 0
-        else:
-            time_from = calendar.timegm(timestamp_from.utctimetuple())
-        if timestamp_to is None:
-            time_to = 0
-        else:
-            time_to = calendar.timegm(timestamp_to.utctimetuple())
-
-        if contact is None:
-            contact_id = 0
-        else:
-            contact_id = contact.id
-
-        dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
-        return [
-            Location(
-                latitude=lib.dc_array_get_latitude(dc_array, i),
-                longitude=lib.dc_array_get_longitude(dc_array, i),
-                accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i)
-                ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
-            for i in range(lib.dc_array_get_cnt(dc_array))
-        ]
-
-
-class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
-        assert isinstance(timestamp, datetime)
-        self.latitude = latitude
-        self.longitude = longitude
-        self.accuracy = accuracy
-        self.timestamp = timestamp
-        self.marker = marker
-
-    def __eq__(self, other):
-        return self.__dict__ == other.__dict__

python/src/deltachat/chatting.py

@@ -0,0 +1,360 @@
+""" chatting related objects: Contact, Chat, Message. """
+
+import mimetypes
+import os
+from . import props
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
+from .capi import lib, ffi
+from . import const
+from .message import Message
+
+
+class Contact(object):
+    """ Delta-Chat Contact.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self._dc_context == other._dc_context and self.id == other.id
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+
+    @property
+    def _dc_contact(self):
+        return ffi.gc(
+            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_contact_unref
+        )
+
+    @props.with_doc
+    def addr(self):
+        """ normalized e-mail address for this account. """
+        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
+
+    @props.with_doc
+    def display_name(self):
+        """ display name for this contact. """
+        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
+
+    def is_blocked(self):
+        """ Return True if the contact is blocked. """
+        return lib.dc_contact_is_blocked(self._dc_contact)
+
+    def is_verified(self):
+        """ Return True if the contact is verified. """
+        return lib.dc_contact_is_verified(self._dc_contact)
+
+
+class Chat(object):
+    """ Chat object which manages members and through which you can send and retrieve messages.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+
+    def __init__(self, account, id):
+        self.account = account
+        self._dc_context = account._dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self.id == getattr(other, "id", None) and \
+               self._dc_context == getattr(other, "_dc_context", None)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
+
+    @property
+    def _dc_chat(self):
+        return ffi.gc(
+            lib.dc_get_chat(self._dc_context, self.id),
+            lib.dc_chat_unref
+        )
+
+    def delete(self):
+        """Delete this chat and all its messages.
+
+        Note:
+
+        - does not delete messages on server
+        - the chat or contact is not blocked, new message will arrive
+        """
+        lib.dc_delete_chat(self._dc_context, self.id)
+
+    # ------  chat status/metadata API ------------------------------
+
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
+    def is_promoted(self):
+        """ return True if this chat is promoted, i.e.
+        the member contacts are aware of their membership,
+        have been sent messages.
+
+        :returns: True if chat is promoted, False otherwise.
+        """
+        return not lib.dc_chat_is_unpromoted(self._dc_chat)
+
+    def get_name(self):
+        """ return name of this chat.
+
+        :returns: unicode name
+        """
+        return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
+
+    def set_name(self, name):
+        """ set name of this chat.
+
+        :param: name as a unicode string.
+        :returns: None
+        """
+        name = as_dc_charpointer(name)
+        return lib.dc_set_chat_name(self._dc_context, self.id, name)
+
+    def get_type(self):
+        """ return type of this chat.
+
+        :returns: one of const.DC_CHAT_TYPE_*
+        """
+        return lib.dc_chat_get_type(self._dc_chat)
+
+    def get_join_qr(self):
+        """ get/create Join-Group QR Code as ascii-string.
+
+        this string needs to be transferred to another DC account
+        in a second channel (typically used by mobiles with QRcode-show + scan UX)
+        where account.join_with_qrcode(qr) needs to be called.
+        """
+        res = lib.dc_get_securejoin_qr(self._dc_context, self.id)
+        return from_dc_charpointer(res)
+
+    # ------  chat messaging API ------------------------------
+
+    def send_text(self, text):
+        """ send a text message and return the resulting Message instance.
+
+        :param msg: unicode text
+        :raises ValueError: if message can not be send/chat does not exist.
+        :returns: the resulting :class:`deltachat.message.Message` instance
+        """
+        msg = as_dc_charpointer(text)
+        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
+        if msg_id == 0:
+            raise ValueError("message could not be send, does chat exist?")
+        return Message.from_db(self.account, msg_id)
+
+    def send_file(self, path, mime_type="application/octet-stream"):
+        """ send a file and return the resulting Message instance.
+
+        :param path: path to the file.
+        :param mime_type: the mime-type of this file, defaults to application/octet-stream.
+        :raises ValueError: if message can not be send/chat does not exist.
+        :returns: the resulting :class:`deltachat.message.Message` instance
+        """
+        msg = self.prepare_message_file(path=path, mime_type=mime_type)
+        self.send_prepared(msg)
+        return msg
+
+    def send_image(self, path):
+        """ send an image message and return the resulting Message instance.
+
+        :param path: path to an image file.
+        :raises ValueError: if message can not be send/chat does not exist.
+        :returns: the resulting :class:`deltachat.message.Message` instance
+        """
+        mime_type = mimetypes.guess_type(path)[0]
+        msg = self.prepare_message_file(path=path, mime_type=mime_type, view_type="image")
+        self.send_prepared(msg)
+        return msg
+
+    def prepare_message(self, msg):
+        """ create a new prepared message.
+
+        :param msg: the message to be prepared.
+        :returns: :class:`deltachat.message.Message` instance.
+        """
+        msg_id = lib.dc_prepare_msg(self._dc_context, self.id, msg._dc_msg)
+        if msg_id == 0:
+            raise ValueError("message could not be prepared")
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
+
+    def prepare_message_file(self, path, mime_type=None, view_type="file"):
+        """ prepare a message for sending and return the resulting Message instance.
+
+        To actually send the message, call :meth:`send_prepared`.
+        The file must be inside the blob directory.
+
+        :param path: path to the file.
+        :param mime_type: the mime-type of this file, defaults to auto-detection.
+        :param view_type: "text", "image", "gif", "audio", "video", "file"
+        :raises ValueError: if message can not be prepared/chat does not exist.
+        :returns: the resulting :class:`Message` instance
+        """
+        msg = Message.new_empty(self.account, view_type)
+        msg.set_file(path, mime_type)
+        return self.prepare_message(msg)
+
+    def send_prepared(self, message):
+        """ send a previously prepared message.
+
+        :param message: a :class:`Message` instance previously returned by
+                        :meth:`prepare_file`.
+        :raises ValueError: if message can not be sent.
+        :returns: a :class:`deltachat.message.Message` instance as sent out.
+        """
+        assert message.id != 0 and message.is_out_preparing()
+        # get a fresh copy of dc_msg, the core needs it
+        msg = Message.from_db(self.account, message.id)
+
+        # pass 0 as chat-id because core-docs say it's ok when out-preparing
+        sent_id = lib.dc_send_msg(self._dc_context, 0, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        assert sent_id == msg.id
+        # modify message in place to avoid bad state for the caller
+        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
+
+    def set_draft(self, message):
+        """ set message as draft.
+
+        :param message: a :class:`Message` instance
+        :returns: None
+        """
+        if message is None:
+            lib.dc_set_draft(self._dc_context, self.id, ffi.NULL)
+        else:
+            lib.dc_set_draft(self._dc_context, self.id, message._dc_msg)
+
+    def get_draft(self):
+        """ get draft message for this chat.
+
+        :param message: a :class:`Message` instance
+        :returns: Message object or None (if no draft available)
+        """
+        x = lib.dc_get_draft(self._dc_context, self.id)
+        if x == ffi.NULL:
+            return None
+        dc_msg = ffi.gc(x, lib.dc_msg_unref)
+        return Message(self.account, dc_msg)
+
+    def get_messages(self):
+        """ return list of messages in this chat.
+
+        :returns: list of :class:`deltachat.message.Message` objects for this chat.
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
+            lib.dc_array_unref
+        )
+        return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
+
+    def count_fresh_messages(self):
+        """ return number of fresh messages in this chat.
+
+        :returns: number of fresh messages
+        """
+        return lib.dc_get_fresh_msg_cnt(self._dc_context, self.id)
+
+    def mark_noticed(self):
+        """ mark all messages in this chat as noticed.
+
+        Noticed messages are no longer fresh.
+        """
+        return lib.dc_marknoticed_chat(self._dc_context, self.id)
+
+    # ------  group management API ------------------------------
+
+    def add_contact(self, contact):
+        """ add a contact to this chat.
+
+        :params: contact object.
+        :raises ValueError: if contact could not be added
+        :returns: None
+        """
+        ret = lib.dc_add_contact_to_chat(self._dc_context, self.id, contact.id)
+        if ret != 1:
+            raise ValueError("could not add contact {!r} to chat".format(contact))
+
+    def remove_contact(self, contact):
+        """ remove a contact from this chat.
+
+        :params: contact object.
+        :raises ValueError: if contact could not be removed
+        :returns: None
+        """
+        ret = lib.dc_remove_contact_from_chat(self._dc_context, self.id, contact.id)
+        if ret != 1:
+            raise ValueError("could not remove contact {!r} from chat".format(contact))
+
+    def get_contacts(self):
+        """ get all contacts for this chat.
+        :params: contact object.
+        :returns: list of :class:`deltachat.chatting.Contact` objects for this chat
+
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return list(iter_array(
+            dc_array, lambda id: Contact(self._dc_context, id))
+        )
+
+    def set_profile_image(self, img_path):
+        """Set group profile image.
+
+        If the group is already promoted (any message was sent to the group),
+        all group members are informed by a special status message that is sent
+        automatically by this function.
+        :params img_path: path to image object
+        :raises ValueError: if profile image could not be set
+        :returns: None
+        """
+        assert os.path.exists(img_path), img_path
+        p = as_dc_charpointer(img_path)
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, p)
+        if res != 1:
+            raise ValueError("Setting Profile Image {!r} failed".format(p))
+
+    def remove_profile_image(self):
+        """remove group profile image.
+
+        If the group is already promoted (any message was sent to the group),
+        all group members are informed by a special status message that is sent
+        automatically by this function.
+        :raises ValueError: if profile image could not be reset
+        :returns: None
+        """
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, ffi.NULL)
+        if res != 1:
+            raise ValueError("Removing Profile Image failed")
+
+    def get_profile_image(self):
+        """Get group profile image.
+
+        For groups, this is the image set by any group member using
+        set_chat_profile_image(). For normal chats, this is the image
+        set by each remote user on their own using dc_set_config(context,
+        "selfavatar", image).
+        :returns: path to profile image, None if no profile image exists.
+        """
+        dc_res = lib.dc_chat_get_profile_image(self._dc_chat)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)

python/src/deltachat/const.py

@@ -1,7 +1,124 @@
-from .capi import lib
+import sys
+import re
+import os
+from os.path import dirname, abspath
+from os.path import join as joinpath
+
+# the following const are generated from deltachat.h
+# this works well when you in a git-checkout
+# run "python deltachat/const.py" to regenerate events
+# begin const generated
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_GCL_ARCHIVED_ONLY = 0x01
+DC_GCL_NO_SPECIALS = 0x02
+DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_VERIFIED_ONLY = 0x01
+DC_GCL_ADD_SELF = 0x02
+DC_QR_ASK_VERIFYCONTACT = 200
+DC_QR_ASK_VERIFYGROUP = 202
+DC_QR_FPR_OK = 210
+DC_QR_FPR_MISMATCH = 220
+DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ADDR = 320
+DC_QR_TEXT = 330
+DC_QR_URL = 332
+DC_QR_ERROR = 400
+DC_CHAT_ID_DEADDROP = 1
+DC_CHAT_ID_TRASH = 3
+DC_CHAT_ID_MSGS_IN_CREATION = 4
+DC_CHAT_ID_STARRED = 5
+DC_CHAT_ID_ARCHIVED_LINK = 6
+DC_CHAT_ID_ALLDONE_HINT = 7
+DC_CHAT_ID_LAST_SPECIAL = 9
+DC_CHAT_TYPE_UNDEFINED = 0
+DC_CHAT_TYPE_SINGLE = 100
+DC_CHAT_TYPE_GROUP = 120
+DC_CHAT_TYPE_VERIFIED_GROUP = 130
+DC_MSG_ID_MARKER1 = 1
+DC_MSG_ID_DAYMARKER = 9
+DC_MSG_ID_LAST_SPECIAL = 9
+DC_STATE_UNDEFINED = 0
+DC_STATE_IN_FRESH = 10
+DC_STATE_IN_NOTICED = 13
+DC_STATE_IN_SEEN = 16
+DC_STATE_OUT_PREPARING = 18
+DC_STATE_OUT_DRAFT = 19
+DC_STATE_OUT_PENDING = 20
+DC_STATE_OUT_FAILED = 24
+DC_STATE_OUT_DELIVERED = 26
+DC_STATE_OUT_MDN_RCVD = 28
+DC_CONTACT_ID_SELF = 1
+DC_CONTACT_ID_DEVICE = 2
+DC_CONTACT_ID_LAST_SPECIAL = 9
+DC_MSG_TEXT = 10
+DC_MSG_IMAGE = 20
+DC_MSG_GIF = 21
+DC_MSG_AUDIO = 40
+DC_MSG_VOICE = 41
+DC_MSG_VIDEO = 50
+DC_MSG_FILE = 60
+DC_EVENT_INFO = 100
+DC_EVENT_SMTP_CONNECTED = 101
+DC_EVENT_IMAP_CONNECTED = 102
+DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_WARNING = 300
+DC_EVENT_ERROR = 400
+DC_EVENT_ERROR_NETWORK = 401
+DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
+DC_EVENT_MSGS_CHANGED = 2000
+DC_EVENT_INCOMING_MSG = 2005
+DC_EVENT_MSG_DELIVERED = 2010
+DC_EVENT_MSG_FAILED = 2012
+DC_EVENT_MSG_READ = 2015
+DC_EVENT_CHAT_MODIFIED = 2020
+DC_EVENT_CONTACTS_CHANGED = 2030
+DC_EVENT_LOCATION_CHANGED = 2035
+DC_EVENT_CONFIGURE_PROGRESS = 2041
+DC_EVENT_IMEX_PROGRESS = 2051
+DC_EVENT_IMEX_FILE_WRITTEN = 2052
+DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
+DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_GET_STRING = 2091
+DC_EVENT_FILE_COPIED = 2055
+DC_EVENT_IS_OFFLINE = 2081
+# end const generated
 
 
-for name in dir(lib):
+def read_event_defines(f):
-    if name.startswith("DC_"):
+    rex = re.compile(r'#define\s+((?:DC_EVENT_|DC_QR|DC_MSG|DC_STATE_|'
-        globals()[name] = getattr(lib, name)
+                     r'DC_CONTACT_ID_|DC_GCL|DC_CHAT|DC_PROVIDER)\S+)\s+([x\d]+).*')
-del name
+    for line in f:
+        m = rex.match(line)
+        if m:
+            yield m.groups()
+
+
+if __name__ == "__main__":
+    here = abspath(__file__).rstrip("oc")
+    here_dir = dirname(here)
+    if len(sys.argv) >= 2:
+        deltah = sys.argv[1]
+    else:
+        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
+    assert os.path.exists(deltah)
+
+    lines = []
+    skip_to_end = False
+    for orig_line in open(here):
+        if skip_to_end:
+            if not orig_line.startswith("# end const"):
+                continue
+            skip_to_end = False
+        lines.append(orig_line)
+        if orig_line.startswith("# begin const"):
+            with open(deltah) as f:
+                for name, item in read_event_defines(f):
+                    lines.append("{} = {}\n".format(name, item))
+            skip_to_end = True
+
+    tmpname = here + ".tmp"
+    with open(tmpname, "w") as f:
+        f.write("".join(lines))
+    os.rename(tmpname, here)

python/src/deltachat/contact.py

@@ -1,100 +0,0 @@
-""" Contact object. """
-
-from . import props
-from .cutil import from_dc_charpointer
-from .capi import lib, ffi
-from .chat import Chat
-from . import const
-
-
-class Contact(object):
-    """ Delta-Chat Contact.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-    def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
-        self.account = account
-        self.id = id
-
-    def __eq__(self, other):
-        return self.account._dc_context == other.account._dc_context and self.id == other.id
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
-
-    @property
-    def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self.account._dc_context, self.id),
-            lib.dc_contact_unref
-        )
-
-    @props.with_doc
-    def addr(self):
-        """ normalized e-mail address for this account. """
-        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
-
-    @props.with_doc
-    def name(self):
-        """ display name for this contact. """
-        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
-
-    # deprecated alias
-    display_name = name
-
-    def is_blocked(self):
-        """ Return True if the contact is blocked. """
-        return lib.dc_contact_is_blocked(self._dc_contact)
-
-    def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, block)
-
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
-    def is_verified(self):
-        """ Return True if the contact is verified. """
-        return lib.dc_contact_is_verified(self._dc_contact)
-
-    def get_profile_image(self):
-        """Get contact profile image.
-
-        :returns: path to profile image, None if no profile image exists.
-        """
-        dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        if dc_res == ffi.NULL:
-            return None
-        return from_dc_charpointer(dc_res)
-
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
-
-    def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        dc_context = self.account._dc_context
-        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
-        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
-        return Chat(self.account, chat_id)
-
-    # deprecated name
-    get_chat = create_chat

python/src/deltachat/cutil.py

@@ -17,8 +17,7 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    if obj != ffi.NULL:
+    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/src/deltachat/direct_imap.py

@@ -1,266 +0,0 @@
-"""
-Internal Python-level IMAP handling used by the testplugin
-and for cleaning up inbox/mvbox for each test function run.
-"""
-
-import io
-import email
-import ssl
-import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
-import imaplib
-import deltachat
-from deltachat import const
-
-
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
-ALL = "1:*"
-
-
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.delete_folder(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
-
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
-class DirectImap:
-    def __init__(self, account):
-        self.account = account
-        self.logid = account.get_config("displayname") or id(account)
-        self._idling = False
-        self.connect()
-
-    def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
-
-        user = self.account.get_config("addr")
-        pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
-        self.conn.login(user, pw)
-
-        self.select_folder("INBOX")
-
-    def shutdown(self):
-        try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
-            pass
-        try:
-            self.conn.logout()
-        except (OSError, IMAPClientError):
-            print("Could not logout direct_imap conn")
-
-    def create_folder(self, foldername):
-        try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
-    def select_folder(self, foldername):
-        assert not self._idling
-        return self.conn.select_folder(foldername)
-
-    def select_config_folder(self, config_name):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
-        if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
-        foldername = self.account.get_config(config_name)
-        if foldername:
-            return self.select_folder(foldername)
-
-    def list_folders(self):
-        """ return list of all existing folder names"""
-        assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
-
-    def delete(self, range, expunge=True):
-        """ delete a range of messages (imap-syntax).
-        If expunge is true, perform the expunge-operation
-        to make sure the messages are really gone and not
-        just flagged as deleted.
-        """
-        self.conn.set_flags(range, [DELETED])
-        if expunge:
-            self.conn.expunge()
-
-    def get_all_messages(self):
-        assert not self._idling
-
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
-        assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
-
-    def mark_all_read(self):
-        messages = self.get_unread_messages()
-        if messages:
-            res = self.conn.set_flags(messages, [SEEN])
-            print("marked seen:", messages, res)
-
-    def get_unread_cnt(self):
-        return len(self.get_unread_messages())
-
-    def dump_imap_structures(self, dir, logfile):
-        assert not self._idling
-        stream = io.StringIO()
-
-        def log(*args, **kwargs):
-            kwargs["file"] = stream
-            print(*args, **kwargs)
-
-        empty_folders = []
-        for imapfolder in self.list_folders():
-            self.select_folder(imapfolder)
-            messages = list(self.get_all_messages())
-            if not messages:
-                empty_folders.append(imapfolder)
-                continue
-
-            log("---------", imapfolder, len(messages), "messages ---------")
-            # get message content without auto-marking it as seen
-            # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
-                    continue
-
-                flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
-                path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
-
-        if empty_folders:
-            log("--------- EMPTY FOLDERS:", empty_folders)
-
-        print(stream.getvalue(), file=logfile)
-
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False):
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
-        if terminate:
-            self.idle_done()
-        self.account.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
-        """
-        while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle_done()
-            self._idling = False
-            return res
-
-    def append(self, folder, msg):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
-        """
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
-
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]

python/src/deltachat/events.py

@@ -1,279 +0,0 @@
-import threading
-import time
-import re
-import os
-from queue import Queue, Empty
-
-import deltachat
-from .hookspec import account_hookimpl
-from contextlib import contextmanager
-from .capi import ffi, lib
-from .message import map_system_message
-from .cutil import from_dc_charpointer
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account):
-        self.account = account
-        self.logid = self.account.get_config("displayname")
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self.account.log(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-
-        if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
-            if message.startswith("DC_EVENT_WARNING"):
-                s = WARN + s + ENDC
-            if message.startswith("DC_EVENT_ERROR"):
-                s = ERROR + s + ENDC
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError("unexpected event: {}".format(ev))
-        return ev
-
-    def iter_events(self, timeout=None, check_error=True):
-        while 1:
-            yield self.get(timeout=timeout, check_error=check_error)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        for ev in self.iter_events(timeout=timeout, check_error=check_error):
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_contains(self, regex):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.search(ev.data2):
-                return ev
-
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def wait_securejoin_inviter_progress(self, target):
-        while 1:
-            event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-            if event.data2 >= target:
-                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
-                break
-
-    def wait_all_initial_fetches(self):
-        """Has to be called after start_io() to wait for fetch_existing_msgs to run
-        so that new messages are not mistaken for old ones:
-        - ac1 and ac2 are created
-        - ac1 sends a message to ac2
-        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("Done fetching existing messages")
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)
-
-    def wait_msg_delivered(self, msg):
-        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == msg.chat.id
-        assert ev.data2 == msg.id
-        assert msg.is_out_delivered()
-
-
-class EventThread(threading.Thread):
-    """ Event Thread for an account.
-
-    With each Account init this callback thread is started.
-    """
-    def __init__(self, account):
-        self.account = account
-        super(EventThread, self).__init__(name="events")
-        self.setDaemon(True)
-        self._marked_for_shutdown = False
-        self.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.log(message + " START")
-        yield
-        self.account.log(message + " FINISHED")
-
-    def mark_shutdown(self):
-        self._marked_for_shutdown = True
-
-    def wait(self, timeout=None):
-        if self == threading.current_thread():
-            # we are in the callback thread and thus cannot
-            # wait for the thread-loop to finish.
-            return
-        self.join(timeout=timeout)
-
-    def run(self):
-        """ get and run events until shutdown. """
-        with self.log_execution("EVENT THREAD"):
-            self._inner_run()
-
-    def _inner_run(self):
-        event_emitter = ffi.gc(
-            lib.dc_get_event_emitter(self.account._dc_context),
-            lib.dc_event_emitter_unref,
-        )
-        while not self._marked_for_shutdown:
-            event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL:
-                break
-            if self._marked_for_shutdown:
-                break
-            evt = lib.dc_event_get_id(event)
-            data1 = lib.dc_event_get_data1_int(event)
-            # the following code relates to the deltachat/_build.py's helper
-            # function which provides us signature info of an event call
-            evt_name = deltachat.get_dc_event_name(evt)
-            if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
-            else:
-                data2 = lib.dc_event_get_data2_int(event)
-
-            lib.dc_event_unref(event)
-            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            try:
-                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-                for name, kwargs in self._map_ffi_event(ffi_event):
-                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
-                    hook = getattr(self.account._pm.hook, name)
-                    hook(**kwargs)
-            except Exception:
-                if self.account._dc_context is not None:
-                    raise
-
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        account = self.account
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = account.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = account.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)

python/src/deltachat/hookspec.py

@@ -1,115 +0,0 @@
-""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
-
-import pluggy
-
-
-account_spec_name = "deltachat-account"
-account_hookspec = pluggy.HookspecMarker(account_spec_name)
-account_hookimpl = pluggy.HookimplMarker(account_spec_name)
-
-global_spec_name = "deltachat-global"
-global_hookspec = pluggy.HookspecMarker(global_spec_name)
-global_hookimpl = pluggy.HookimplMarker(global_spec_name)
-
-
-class PerAccount:
-    """ per-Account-instance hook specifications.
-
-    All hooks are executed in a dedicated Event thread.
-    Hooks are generally not allowed to block/last long as this
-    blocks overall event processing on the python side.
-    """
-    @classmethod
-    def _make_plugin_manager(cls):
-        pm = pluggy.PluginManager(account_spec_name)
-        pm.add_hookspecs(cls)
-        return pm
-
-    @account_hookspec
-    def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
-
-        ffi_event has "name", "data1", "data2" values as specified
-        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-        """
-
-    @account_hookspec
-    def ac_log_line(self, message):
-        """ log a message related to the account. """
-
-    @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
-
-    @account_hookspec
-    def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
-
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
-
-    @account_hookspec
-    def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
-
-        :param message: Message that was just delivered.
-        """
-
-    @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
-
-        :param chat: Chat which was modified.
-        """
-
-    @account_hookspec
-    def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
-
-        :param chat: Chat where contact was added.
-        :param contact: Contact that was added.
-        :param actor: Who added the contact (None if it was our self-addr)
-        :param message: The original system message that reports the addition.
-        """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
-
-        :param chat: Chat where contact was removed.
-        :param contact: Contact that was removed.
-        :param actor: Who removed the contact (None if it was our self-addr)
-        :param message: The original system message that reports the removal.
-        """
-
-
-class Global:
-    """ global hook specifications using a per-process singleton
-    plugin manager instance.
-
-    """
-    _plugin_manager = None
-
-    @classmethod
-    def _get_plugin_manager(cls):
-        if cls._plugin_manager is None:
-            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
-            pm.add_hookspecs(cls)
-        return cls._plugin_manager
-
-    @global_hookspec
-    def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
-
-    @global_hookspec
-    def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
-
-        This hook can be used to perform extra work before
-        ac_configure_completed is called.
-        """
-
-    @global_hookspec
-    def dc_account_after_shutdown(self, account):
-        """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -1,7 +1,7 @@
-""" The Message object. """
+""" chatting related objects: Contact, Chat, Message. """
 
 import os
-import re
+import shutil
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -13,26 +13,23 @@ class Message(object):
     """ Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
-    :class:`deltachat.chat.Chat`.
+    :class:`deltachat.chatting.Chat`.
     """
     def __init__(self, account, dc_msg):
         self.account = account
-        assert isinstance(self.account._dc_context, ffi.CData)
+        self._dc_context = account._dc_context
+        assert isinstance(self._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
+        self.id = lib.dc_msg_get_id(dc_msg)
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
-        typ = "outgoing" if self.is_outgoing() else "incoming"
-        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ, self.is_system_message(), repr(self.text[:10]),
-            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
 
     @classmethod
     def from_db(cls, account, id):
@@ -46,34 +43,14 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
+        :param: view_type is "text", "audio", "video", "file"
-           "text", "audio", "video", "file", "sticker"
         """
-        if isinstance(view_type, int):
+        view_type_code = get_viewtype_code_from_name(view_type)
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
         ))
 
-    def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
-
-        If the message is a contact request
-        the sender will become an accepted contact.
-
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        self.chat.accept()
-        return self.chat
-
-    @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
-
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -81,25 +58,10 @@ class Message(object):
 
     def set_text(self, text):
         """set text of this message. """
+        assert self.id > 0, "message not prepared"
+        assert self.is_out_preparing()
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self):
-        """html text of this messages (might be empty if not an html message). """
-        return from_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -110,6 +72,19 @@ class Message(object):
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError("path does not exist: {!r}".format(path))
+        blobdir = self.account.get_blobdir()
+        if not path.startswith(blobdir):
+            for i in range(50):
+                ext = "" if i == 0 else "-" + str(i)
+                dest = os.path.join(blobdir, os.path.basename(path) + ext)
+                if os.path.exists(dest):
+                    continue
+                shutil.copyfile(path, dest)
+                break
+            else:
+                raise ValueError("could not create blobdir-path for {}".format(path))
+            path = dest
+        assert path.startswith(blobdir), path
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
@@ -122,41 +97,21 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return bool(lib.dc_msg_is_info(self._dc_msg))
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self):
-        """ return the first characters of a setup code in a setup message. """
-        return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
-
-    def is_encrypted(self):
-        """ return True if this message was encrypted. """
-        return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
-
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
-    def is_forwarded(self):
-        """ return True if this message was forwarded. """
-        return bool(lib.dc_msg_is_forwarded(self._dc_msg))
-
     def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
         """
-        return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
+        return from_dc_charpointer(lib.dc_get_msg_info(self._dc_context, self.id))
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
         res = lib.dc_continue_key_transfer(
-                self.account._dc_context,
+                self._dc_context,
                 self.id,
                 as_dc_charpointer(setup_code)
         )
@@ -182,47 +137,6 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
-    @props.with_doc
-    def ephemeral_timer(self):
-        """Ephemeral timer in seconds
-
-        :returns: timer in seconds or None if there is no timer
-        """
-        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
-        if timer:
-            return timer
-
-    @props.with_doc
-    def ephemeral_timestamp(self):
-        """UTC time when the message will be deleted.
-
-        :returns: naive datetime.datetime() object or None if the timer is not started.
-        """
-        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
-        if ts:
-            return datetime.utcfromtimestamp(ts)
-
-    @property
-    def quoted_text(self):
-        """Text inside the quote
-
-        :returns: Quoted text"""
-        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
-
-    @property
-    def quote(self):
-        """Quote getter
-
-        :returns: Quoted message, if found in the database"""
-        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
-        if msg:
-            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
-
-    @quote.setter
-    def quote(self, quoted_message):
-        """Quote setter"""
-        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
-
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -232,57 +146,31 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
-        mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
+        mime_headers = lib.dc_get_mime_headers(self._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
-                return email.message_from_bytes(s)
+                s = s.decode("ascii")
             return email.message_from_string(s)
 
-    @property
-    def error(self):
-        """Error message"""
-        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
 
-        :returns: :class:`deltachat.chat.Chat` object
+        :returns: :class:`deltachat.chatting.Chat` object
         """
-        from .chat import Chat
+        from .chatting import Chat
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self):
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
-    def get_sender_chat(self):
-        """return the 1:1 chat with the sender of this message.
-
-        :returns: :class:`deltachat.chat.Chat` instance
-        """
-        return self.get_sender_contact().get_chat()
-
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
-        :returns: :class:`deltachat.chat.Contact` instance
+        :returns: :class:`deltachat.chatting.Contact` instance
         """
-        from .contact import Contact
+        from .chatting import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self.account, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     #
     # Message State query methods
@@ -290,11 +178,11 @@ class Message(object):
     @property
     def _msgstate(self):
         if self.id == 0:
-            dc_msg = self._dc_msg
+            dc_msg = self.message._dc_msg
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
+                lib.dc_get_msg(self._dc_context, self.id),
                 lib.dc_msg_unref
             )
         return lib.dc_msg_get_state(dc_msg)
@@ -323,13 +211,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -380,10 +261,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -396,76 +273,22 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
-    def mark_seen(self):
-        """ mark this message as seen. """
-        self.account.mark_seen_messages([self.id])
-
 
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
+    const.DC_MSG_TEXT: 'text',
-    'image': const.DC_MSG_IMAGE,
+    const.DC_MSG_IMAGE: 'image',
-    'gif': const.DC_MSG_GIF,
+    const.DC_MSG_GIF: 'gif',
-    'audio': const.DC_MSG_AUDIO,
+    const.DC_MSG_AUDIO: 'audio',
-    'video': const.DC_MSG_VIDEO,
+    const.DC_MSG_VIDEO: 'video',
-    'file': const.DC_MSG_FILE,
+    const.DC_MSG_FILE: 'file'
-    'sticker': const.DC_MSG_STICKER,
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
+    for code, value in _view_type_mapping.items():
-    if code is not None:
+        if value == view_type_name:
-        return code
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if not res:
-            return
-        action, affected, actor = res
-        affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
-        return "ac_member_" + res[0], d
-
-
-def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
-    if m:
-        text = m.group(1)
-    text = text.rstrip(".")
-    return text.strip()
-
-
-def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
-
-    returns a (action, affected, actor) triple """
-
-    # Member Me (x@y) removed by a@b.
-    # Member x@y added by a@b
-    # Member With space (tmp1@x.org) removed by tmp2@x.org.
-    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-    # Group left by some one (tmp1@x.org).
-    # Group left by tmp1@x.org.
-    text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
-    if m:
-        affected, action, actor = m.groups()
-        return action, extract_addr(affected), extract_addr(actor)
-    if text.startswith("group left by "):
-        addr = extract_addr(text[13:])
-        if addr:
-            return "removed", addr, addr

python/src/deltachat/provider.py

@@ -11,18 +11,27 @@ class ProviderNotFoundError(Exception):
 class Provider(object):
     """Provider information.
 
-    :param domain: The email to get the provider info for.
+    :param domain: The domain to get the provider info for, this is
+    normally the part following the `@` of the domain.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, domain):
         provider = ffi.gc(
-            lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
+            lib.dc_provider_new_from_domain(as_dc_charpointer(domain)),
             lib.dc_provider_unref,
         )
         if provider == ffi.NULL:
             raise ProviderNotFoundError("Provider not found")
         self._provider = provider
 
+    @classmethod
+    def from_email(cls, email):
+        """Create provider info from an email address.
+
+        :param email: Email address to get provider info for.
+        """
+        return cls(email.split('@')[-1])
+
     @property
     def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
@@ -30,10 +39,21 @@ class Provider(object):
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self):
+    def name(self):
-        """Should be shown to the user on login."""
+        """The name of the provider."""
+        return from_dc_charpointer(lib.dc_provider_get_name(self._provider))
+
+    @property
+    def markdown(self):
+        """Content of the information page, formatted as markdown."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hints(self._provider))
+            lib.dc_provider_get_markdown(self._provider))
+
+    @property
+    def status_date(self):
+        """The date the provider info was last updated, as a string."""
+        return from_dc_charpointer(
+            lib.dc_provider_get_status_date(self._provider))
 
     @property
     def status(self):

python/src/deltachat/testplugin.py

@@ -1,534 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-import io
-import subprocess
-import queue
-import threading
-import fnmatch
-import time
-import weakref
-import tempfile
-
-import pytest
-import requests
-
-from . import Account, const
-from .capi import lib
-from .events import FFIEventLogger, FFIEventTracker
-from _pytest._code import Source
-from deltachat import direct_imap
-
-import deltachat
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-    parser.addoption(
-        "--strict-tls", action="store_true",
-        help="Never accept invalid TLS certificates for test accounts",
-    )
-
-
-def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
-
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            if item.get_closest_marker("ignored"):
-                if not item.config.getvalue("ignored"):
-                    pytest.skip("use --ignored to run this test")
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    try:
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-         info['journal_mode'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def data(request):
-    class Data:
-        def __init__(self):
-            # trying to find test data heuristically
-            # because we are run from a dev-setup with pytest direct,
-            # through tox, and then maybe also from deltachat-binding
-            # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
-
-        def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
-            for path in self.paths:
-                fn = os.path.join(path, *bn.split("/"))
-                if os.path.exists(fn):
-                    return fn
-            print("WARNING: path does not exist: {!r}".format(fn))
-
-        def read_path(self, bn, mode="r"):
-            fn = self.get_path(bn)
-            if fn is not None:
-                with open(fn, mode) as f:
-                    return f.read()
-
-    return Data()
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self._accounts = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
-
-        def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
-            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
-            ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
-            if not quiet:
-                logger = FFIEventLogger(ac)
-                logger.init_time = self.init_time
-                ac.add_account_plugin(logger)
-            self._accounts.append(ac)
-            return ac
-
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
-                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
-                if fname_pub and fname_sec:
-                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                    return True
-                else:
-                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True, quiet=False):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            if pytestconfig.getoption("--strict-tls"):
-                # Enable strict certificate checks for online accounts
-                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
-                                           pre_generated_key=True, quiet=False, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key, quiet=quiet)
-            configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
-            configdict["mvbox_move"] = str(int(move))
-            configdict["sentbox_watch"] = str(int(sentbox))
-            ac.update_config(configdict)
-            ac._configtracker = ac.configure()
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io([ac1])
-            return ac1
-
-        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
-            ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
-            return ac1, ac2
-
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
-                        for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
-            for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
-            return accounts
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
-
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac.update_config(dict(
-                addr=account.get_config("addr"),
-                mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
-                mvbox_move=account.get_config("mvbox_move"),
-                sentbox_watch=account.get_config("sentbox_watch"),
-            ))
-            if hasattr(account, "direct_imap"):
-                # Attach the existing direct_imap. If we did not do this, a new one would be created and
-                # delete existing messages (see dc_account_extra_configure(configure))
-                ac.direct_imap = account.direct_imap
-            ac._configtracker = ac.configure()
-            return ac
-
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
-            started_accounts = []
-            for acc in accounts:
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
-            for acc in started_accounts:
-                acc._evtracker.wait_all_initial_fetches()
-
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
-
-        def run_bot_process(self, module, ffi=True):
-            fn = module.__file__
-
-            bot_ac, bot_cfg = self.get_online_config()
-
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
-            args = [
-                sys.executable,
-                "-u",
-                fn,
-                "--email", bot_cfg["addr"],
-                "--password", bot_cfg["mail_pw"],
-                bot_ac.db_path,
-            ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
-            print("$", " ".join(args))
-            popen = subprocess.Popen(
-                args=args,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
-                close_fds=True,            # close all FDs other than 0/1/2
-                universal_newlines=True    # give back text
-            )
-            bot = BotProcess(popen, bot_cfg)
-            self._finalizers.append(bot.kill)
-            return bot
-
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1, ac2):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
-
-
-class BotProcess:
-    def __init__(self, popen, bot_cfg):
-        self.popen = popen
-        self.addr = bot_cfg["addr"]
-
-        # we read stdout as quickly as we can in a thread and make
-        # the (unicode) lines available for readers through a queue.
-        self.stdout_queue = queue.Queue()
-        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
-        t.start()
-
-    def _run_stdout_thread(self):
-        try:
-            while 1:
-                line = self.popen.stdout.readline()
-                if not line:
-                    break
-                line = line.strip()
-                self.stdout_queue.put(line)
-                print("bot-stdout: ", line)
-        finally:
-            self.stdout_queue.put(None)
-
-    def kill(self):
-        self.popen.kill()
-
-    def wait(self, timeout=30):
-        self.popen.wait(timeout=timeout)
-
-    def fnmatch_lines(self, pattern_lines):
-        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
-        for next_pattern in patterns:
-            print("+++FNMATCH:", next_pattern)
-            ignored = []
-            while 1:
-                line = self.stdout_queue.get(timeout=15)
-                if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
-                    raise IOError("BOT stdout-thread terminated")
-                if fnmatch.fnmatch(line, next_pattern):
-                    print("+++MATCHED:", line)
-                    break
-                else:
-                    print("+++IGN:", line)
-                    ignored.append(line)
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-
-        def indent(self, msg):
-            print("  " + msg)
-
-    return Printer()
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    # execute all other hooks to obtain the report object
-    outcome = yield
-    rep = outcome.get_result()
-
-    # set a report attribute for each phase of a call, which can
-    # be "setup", "call", "teardown"
-
-    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -1,100 +0,0 @@
-
-from queue import Queue
-from threading import Event
-
-from .hookspec import account_hookimpl, Global
-
-
-class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
-
-
-class ImexTracker:
-    def __init__(self):
-        self._imex_events = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(ffi_event.data1)
-        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data2)
-
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
-
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ImexFailed("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
-
-
-class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
-
-
-class ConfigureTracker:
-    ConfigureFailed = ConfigureFailed
-
-    def __init__(self, account):
-        self.account = account
-        self._configure_events = Queue()
-        self._smtp_finished = Event()
-        self._imap_finished = Event()
-        self._ffi_events = []
-        self._progress = Queue()
-        self._gm = Global._get_plugin_manager()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._ffi_events.append(ffi_event)
-        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
-            self._smtp_finished.set()
-        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
-            self._imap_finished.set()
-        elif ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS":
-            self._progress.put(ffi_event.data1)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        if success:
-            self._gm.hook.dc_account_extra_configure(account=self.account)
-        self._configure_events.put(success)
-        self.account.remove_account_plugin(self)
-
-    def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
-        self._smtp_finished.wait()
-
-    def wait_imap_connected(self):
-        """ wait until smtp is configured. """
-        self._imap_finished.wait()
-
-    def wait_progress(self, data1=None):
-        while 1:
-            evdata = self._progress.get()
-            if data1 is None or evdata == data1:
-                break
-
-    def wait_finish(self):
-        """ wait until configure is completed.
-
-        Raise Exception if Configure failed
-        """
-        if not self._configure_events.get():
-            content = "\n".join(map(str, self._ffi_events))
-            raise ConfigureFailed(content)

python/tests/auditwheels.py

@@ -1,17 +1,13 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(
+            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])
-                ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])

python/tests/conftest.py

@@ -0,0 +1,246 @@
+from __future__ import print_function
+import os
+import pytest
+import requests
+import time
+from deltachat import Account
+from deltachat.capi import lib
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+
+
+def pytest_configure(config):
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_PY_LIVECONFIG')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t, eventlogging=False)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "#" in cfg:
+            url, token = cfg.split("#", 1)
+            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+@pytest.fixture(scope="session")
+def data():
+    class Data:
+        def __init__(self):
+            self.path = os.path.join(os.path.dirname(__file__), "data")
+
+        def get_path(self, bn):
+            fn = os.path.join(self.path, bn)
+            assert os.path.exists(fn)
+            return fn
+    return Data()
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url, create_token):
+        self.configlist = []
+        for i in range(2):
+            res = requests.post(url, json={"token_create_user": int(create_token)})
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            url, create_token = liveconfig_opt.split("#", 1)
+            return SessionLiveConfigFromURL(url, create_token)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid):
+            ac = Account(path, logid=logid)
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(2)
+            return ac
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_configuring_account(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            ac.configure(**configdict)
+            ac.start_threads()
+            return ac
+
+        def get_two_online_accounts(self):
+            ac1 = self.get_online_configuring_account()
+            ac2 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            wait_successful_IMAP_SMTP_connection(ac2)
+            wait_configuration_progress(ac2, 1000)
+            return ac1, ac2
+
+        def clone_online_account(self, account):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
+            ac.start_threads()
+            return ac
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()
+
+
+def wait_configuration_progress(account, target):
+    while 1:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if data1 >= target:
+            print("** CONFIG PROGRESS {}".format(target), account)
+            break
+
+
+def wait_securejoin_inviter_progress(account, target):
+    while 1:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if data2 >= target:
+            print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
+            break
+
+
+def wait_successful_IMAP_SMTP_connection(account):
+    imap_ok = smtp_ok = False
+    while not imap_ok or not smtp_ok:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
+        if evt_name == "DC_EVENT_IMAP_CONNECTED":
+            imap_ok = True
+            print("** IMAP OK", account)
+        if evt_name == "DC_EVENT_SMTP_CONNECTED":
+            smtp_ok = True
+            print("** SMTP OK", account)
+    print("** IMAP and SMTP logins successful", account)
+
+
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev[1] == chat_id
+    if msg_id is not None:
+        assert ev[2] == msg_id
+    return ev[2]

python/tests/data/key

@@ -1 +0,0 @@
-../../../test-data/key

python/tests/stress_test_db.py

@@ -1,135 +0,0 @@
-
-import time
-import threading
-import pytest
-import os
-from queue import Queue, Empty
-
-import deltachat
-
-
-def test_db_busy_error(acfactory, tmpdir):
-    starttime = time.time()
-    log_lock = threading.RLock()
-
-    def log(string):
-        with log_lock:
-            print("%3.2f %s" % (time.time() - starttime, string))
-
-    # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3, quiet=True)
-    log("created %s accounts" % len(accounts))
-
-    # put a bigfile into each account
-    for acc in accounts:
-        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
-        with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
-    log("created %s bigfiles" % len(accounts))
-
-    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
-    chat = accounts[0].create_group_chat("stress-group")
-    for addr in contact_addrs[1:]:
-        chat.add_contact(chat.account.create_contact(addr))
-
-    # setup auto-responder bots which report back failures/actions
-    report_queue = Queue()
-
-    def report_func(replier, report_type, *report_args):
-        report_queue.put((replier, report_type, report_args))
-
-    # each replier receives all events and sends report events to receive_queue
-    repliers = []
-    for acc in accounts:
-        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
-        acc.add_account_plugin(replier)
-        repliers.append(replier)
-
-    # kick off message sending
-    # after which repliers will reply to each other
-    chat.send_text("hello")
-
-    alive_count = len(accounts)
-    while alive_count > 0:
-        try:
-            replier, report_type, report_args = report_queue.get(timeout=10)
-        except Empty:
-            log("timeout waiting for next event")
-            pytest.fail("timeout exceeded")
-        if report_type == ReportType.exit:
-            replier.log("EXIT")
-        elif report_type == ReportType.ffi_error:
-            replier.log("ERROR: {}".format(report_args[0]))
-        elif report_type == ReportType.message_echo:
-            continue
-        else:
-            raise ValueError("{} unknown report type {}, args={}".format(
-                addr, report_type, report_args
-            ))
-        alive_count -= 1
-        replier.log("shutting down")
-        replier.account.shutdown()
-        replier.log("shut down complete, remaining={}".format(alive_count))
-
-
-class ReportType:
-    exit = "exit"
-    ffi_error = "ffi-error"
-    message_echo = "message-echo"
-
-
-class AutoReplier:
-    def __init__(self, account, log, num_send, num_bigfiles, report_func):
-        self.account = account
-        self._log = log
-        self.report_func = report_func
-        self.num_send = num_send
-        self.num_bigfiles = num_bigfiles
-        self.current_sent = 0
-        self.addr = self.account.get_self_contact().addr
-
-        self._thread = threading.Thread(
-            name="Stats{}".format(self.account),
-            target=self.thread_stats
-        )
-        self._thread.setDaemon(True)
-        self._thread.start()
-
-    def log(self, message):
-        self._log("{} {}".format(self.addr, message))
-
-    def thread_stats(self):
-        # XXX later use, for now we just quit
-        return
-        while 1:
-            time.sleep(1.0)
-            break
-
-    @deltachat.account_hookimpl
-    def ac_incoming_message(self, message):
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-        message.create_chat()
-        message.mark_seen()
-        self.log("incoming message: {}".format(message))
-
-        self.current_sent += 1
-        # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
-            message.chat.send_text("send big file as reply to: {}".format(message.text))
-            msg = message.chat.send_file(self.account.bigfile)
-        else:
-            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
-            assert msg.text
-        self.log("message-sent: {}".format(msg))
-        self.report_func(self, ReportType.message_echo)
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-
-    @deltachat.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self.log(ffi_event)
-        if ffi_event.name == "DC_EVENT_ERROR":
-            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -1,115 +1,69 @@
 from __future__ import print_function
-
-import os.path
-import shutil
-
-import pytest
 from filecmp import cmp
-
 from deltachat import const
-
+from conftest import wait_configuration_progress, wait_msgs_changed
-
-def wait_msg_delivered(account, msg_list):
-    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
-    msg_list = list(msg_list)
-    while msg_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        msg_list.remove((ev.data1, ev.data2))
-
-
-def wait_msgs_changed(account, msgs_list):
-    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
-    account.log("waiting for msgs_list={}".format(msgs_list))
-    msgs_list = list(msgs_list)
-    while msgs_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        for i, (data1, data2) in enumerate(msgs_list):
-            if ev.data1 == data1:
-                if data2 is None or ev.data2 == data2:
-                    del msgs_list[i]
-                    break
-        else:
-            account.log("waiting mismatch data1={} data2={}".format(data1, data2))
-    return ev.data1, ev.data2
 
 
 class TestOnlineInCreation:
-    def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating in-creation file outside of blobdir")
-        assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt').ensure(file=1)
-        with pytest.raises(Exception):
-            chat.prepare_message_file(src.strpath)
-
-    def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating file outside of blobdir")
-        assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt')
-        src.write("hello there\n")
-        chat.send_file(src.strpath)
-
-        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
-        assert os.path.exists(blob_src), "file.txt not copied to blobdir"
-
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
 
-        chat = ac1.create_chat(ac2)
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
+        chat = ac1.create_chat_by_contact(c2)
+        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
-        orig = data.get_path("d.png")
+        path = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), 'd.png')
-        with open(path, "x") as fp:
-            fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(ac2)
+        chat2.add_contact(c2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
+        wait_msgs_changed(ac1, 0, 0)  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them
         # is the one caused by forwarding
-        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        forwarded_id = wait_msgs_changed(ac1, chat2.id)
+        if forwarded_id == 0:
+            forwarded_id = wait_msgs_changed(ac1, chat2.id)
+            assert forwarded_id
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 
         lp.sec("finish creating the file and send it")
         assert prepared_original.is_out_preparing()
-        shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
-        lp.sec("check that both forwarded and original message are proper.")
+        lp.sec("expect the forwarded message to be sent now too")
-        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
+        wait_msgs_changed(ac1, chat2.id, forwarded_id)
-
         fwd_msg = ac1.get_message_by_id(forwarded_id)
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
-        lp.sec("wait for both messages to be delivered to SMTP")
+        lp.sec("wait for the messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-            (chat2.id, forwarded_id),
+        assert ev[1] == chat.id
-            (chat.id, prepared_original.id)
+        assert ev[2] == prepared_original.id
-        ])
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat2.id
+        assert ev[2] == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev1[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
+        received_original = ac2.get_message_by_id(ev1[2])
-        assert cmp(received_original.filename, orig, shallow=False)
+        assert cmp(received_original.filename, path, False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
+        assert ev2[1] != ev1[1]
-        received_copy = ac2.get_message_by_id(ev2.data2)
+        received_copy = ac2.get_message_by_id(ev2[2])
-        assert cmp(received_copy.filename, orig, shallow=False)
+        assert cmp(received_copy.filename, path, False)

python/tests/test_lowlevel.py

@@ -1,52 +1,62 @@
 from __future__ import print_function
-
+from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
-from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
-# from deltachat.account import EventLogger
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_context_unref(ctx)
+    capi.lib.dc_close(ctx)
 
 
-def test_dc_close_events(tmpdir, acfactory):
+def test_callback_None2int():
-    ac1 = acfactory.get_unconfigured_account()
+    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    set_context_callback(ctx, lambda *args: None)
+    capi.lib.dc_close(ctx)
+    clear_context_callback(ctx)
 
-    # register after_shutdown function
-    shutdowns = Queue()
 
-    class ShutdownPlugin:
+def test_dc_close_events(tmpdir):
-        @global_hookimpl
+    ctx = ffi.gc(
-        def dc_account_after_shutdown(self, account):
+        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
-            assert account._dc_context is None
+        lib.dc_context_unref,
-            shutdowns.put(account)
+    )
-    register_global_plugin(ShutdownPlugin())
+    evlog = EventLogger(ctx)
-    assert hasattr(ac1, "_dc_context")
+    evlog.set_timeout(5)
-    ac1.shutdown()
+    set_context_callback(
-    shutdowns.get(timeout=2)
+        ctx,
+        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
+    )
+    p = tmpdir.join("hello.db")
+    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+    capi.lib.dc_close(ctx)
+
+    def find(info_string):
+        while 1:
+            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+            data2 = ev[2]
+            if info_string in data2:
+                return
+            else:
+                print("skipping event", *ev)
+
+    find("disconnecting INBOX-watch")
+    find("disconnecting sentbox-thread")
+    find("disconnecting mvbox-thread")
+    find("disconnecting SMTP")
+    find("Database closed")
 
 
 def test_wrong_db(tmpdir):
+    dc_context = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
     p = tmpdir.join("hello.db")
     # write an invalid database file
     p.write("x123" * 10)
-
+    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
-    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
-
-
-def test_empty_blobdir(tmpdir):
-    db_fname = tmpdir.join("hello.db")
-    # Apparently some client code expects this to be the same as passing NULL.
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
-        lib.dc_context_unref,
-    )
-    assert ctx != ffi.NULL
 
 
 def test_event_defines():
@@ -55,50 +65,36 @@ def test_event_defines():
 
 
 def test_sig():
-    sig = capi.lib.dc_event_has_string_data
+    sig = capi.lib.dc_get_event_signature_types
-    assert not sig(const.DC_EVENT_MSGS_CHANGED)
+    assert sig(const.DC_EVENT_INFO) == 2
-    assert sig(const.DC_EVENT_INFO)
+    assert sig(const.DC_EVENT_WARNING) == 2
-    assert sig(const.DC_EVENT_WARNING)
+    assert sig(const.DC_EVENT_ERROR) == 2
-    assert sig(const.DC_EVENT_ERROR)
+    assert sig(const.DC_EVENT_SMTP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_SMTP_CONNECTED)
+    assert sig(const.DC_EVENT_IMAP_CONNECTED) == 2
-    assert sig(const.DC_EVENT_IMAP_CONNECTED)
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT) == 2
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
-    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
 
 
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
-
+    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = ac1.create_chat_by_contact(contact1)
-    chat = contact1.create_chat()
     chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
-def test_get_special_message_id_returns_empty_message(acfactory):
+def test_provider_info():
-    ac1 = acfactory.get_configured_offline_account()
+    provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
-    for i in range(1, 10):
+    assert cutil.from_dc_charpointer(
-        msg = ac1.get_message_by_id(i)
+        lib.dc_provider_get_overview_page(provider)
-        assert msg.id == 0
+    ) == "https://providers.delta.chat/example.com"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_name(provider)) == "Example"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_markdown(provider)) == "\n..."
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_status_date(provider)) == "2018-09"
+    assert lib.dc_provider_get_status(provider) == const.DC_PROVIDER_STATUS_PREPARATION
 
 
 def test_provider_info_none():
-    ctx = ffi.gc(
+    assert lib.dc_provider_new_from_email(cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
-        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
-
-
-def test_get_info_open(tmpdir):
-    db_fname = tmpdir.join("test.db")
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
-        lib.dc_context_unref,
-    )
-    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' in info

python/tests/test_provider_info.py

@@ -0,0 +1,27 @@
+import pytest
+
+from deltachat import const
+from deltachat import provider
+
+
+def test_provider_info_from_email():
+    example = provider.Provider.from_email("email@example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_from_domain():
+    example = provider.Provider("example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_none():
+    with pytest.raises(provider.ProviderNotFoundError):
+        provider.Provider.from_email("email@unexistent.no")

python/tox.ini

@@ -1,20 +1,19 @@
 [tox]
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py35
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -v -rsXx {posargs:tests}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
-    DCC_NEW_TMP_EMAIL 
+    DCC_PY_LIVECONFIG
-    CARGO_TARGET_DIR
-    RUSTC_WRAPPER
 deps = 
     pytest
     pytest-rerunfailures
@@ -29,9 +28,10 @@ deps = auditwheel
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
+
 [testenv:lint]
 skipsdist = True
-skip_install = True
+usedevelop = True
 deps =
     flake8
     # pygments required by rst-lint
@@ -39,39 +39,27 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/ examples/
+    flake8 tests/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
-changedir=doc
+basepython = python3.5
 deps =
-    # Pin dependencies to the versions which actually work with Python 3.5.
+    sphinx==2.2.0
-    sphinx==3.4.3
+    breathe
-    breathe==4.28.0
+
+changedir = doc
 commands =
-    sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
+    sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html
-
-
-[testenv:lintdoc]
-skipsdist = True
-usedevelop = True
-deps =
-    {[testenv:lint]deps}
-    {[testenv:doc]deps}
-commands =
-    {[testenv:lint]commands}
-    {[testenv:doc]commands}
-
 
 
 [pytest]
-addopts = -v -ra --strict-markers
+addopts = -v -rs --reruns 3 --reruns-delay 2
+python_files = tests/test_*.py 
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 90
+timeout = 60 
 timeout_method = thread
-markers = 
-    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   scripts/run-integration-tests.sh -e py35
+#   ./run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   scripts/run-integration-tests.sh -e py35 -- -x
+#   ./run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,6 +23,9 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
+if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then
+    export DCC_PY_LIVECONFIG=liveconfig
+fi
 tox "$@"
 ret=$?
 popd

rust-toolchain

@@ -1 +1 @@
-1.54.0
+nightly-2019-08-13

scripts/README.md

@@ -1,51 +0,0 @@
-# Continuous Integration Scripts for Delta Chat
-
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
-
-## Description of scripts 
-
-- `../.github/workflows` contains jobs run by GitHub Actions.
-
-- `remote_tests_python.sh` rsyncs to a build machine and runs
-  `run-python-test.sh` remotely on the build machine. 
-
-- `remote_tests_rust.sh` rsyncs to the build machine and runs
-  `run-rust-test.sh` remotely on the build machine. 
-
-- `doxygen/Dockerfile` specifies an image that contains
-  the doxygen tool which is used by `run-doxygen.sh` 
-  to generate C-docs which are then uploaded 
-  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-  (and the master branch is linked to https://c.delta.chat proper). 
-
-
-## Triggering runs on the build machine locally (fast!)
-
-There is experimental support for triggering a remote Python or Rust test run 
-from your local checkout/branch. You will need to be authorized to login to 
-the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
-
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
-
-This will **rsync** your current checkout to the remote build machine 
-(no need to commit before) and then run either rust or python tests. 
-
-# Outdated files (for later re-use)
-
-`coredeps/Dockerfile` specifies an image that contains all 
-of Delta Chat's core dependencies. It used to run 
-python tests and build wheels (binary packages for Python)
-
-You can build the docker images yourself locally
-to avoid the relatively large download:: 
- 
-    cd scripts  # where all CI things are 
-    docker build -t deltachat/coredeps docker-coredeps
-    docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

scripts/cleanup_devpi_indices.py

@@ -1,72 +0,0 @@
-"""
-Remove old "dc" indices except for master which always stays.
-
-"""
-from requests import Session
-import datetime
-import sys
-import subprocess
-
-MAXDAYS=7
-
-session = Session()
-session.headers["Accept"] = "application/json"
-
-
-def get_indexes(baseurl, username):
-    response = session.get(baseurl + username)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["indexes"]
-
-
-def get_projectnames(baseurl, username, indexname):
-    response = session.get(baseurl + username + "/" + indexname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["projects"]
-
-
-def get_release_dates(baseurl, username, indexname, projectname):
-    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    dates = set()
-    for value in result.values():
-        if "+links" not in value:
-            continue
-        for link in value["+links"]:
-            for log in link["log"]:
-                dates.add(tuple(log["when"]))
-    return dates
-
-
-def run():
-    baseurl = "https://m.devpi.net/"
-    username = "dc"
-    for indexname in get_indexes(baseurl, username):
-        projectnames = get_projectnames(baseurl, username, indexname)
-        if indexname == "master" or not indexname:
-            continue
-        clear_index = not projectnames
-        for projectname in projectnames:
-            dates = get_release_dates(baseurl, username, indexname, projectname)
-            if not dates:
-                print(
-                    "%s has no releases" % (baseurl + username + "/" + indexname),
-                    file=sys.stderr)
-                date = datetime.datetime.now()
-            else:
-                date = datetime.datetime(*max(dates))
-            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
-                assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
-
-
-
-if __name__ == '__main__':
-    run()

scripts/concourse/README.md

@@ -1,21 +0,0 @@
-# Concourse CI pipeline
-
-`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
-that builds C documentation, Python documentation, Python wheels for `x86_64`
-and `aarch64` and Python source packages, and uploads them.
-
-To setup the pipeline run
-```
-fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
-```
-where `secret.yml` contains the following secrets:
-```
-c.delta.chat:
-  private_key: |
-    -----BEGIN RSA PRIVATE KEY-----
-    ...
-    -----END RSA PRIVATE KEY-----
-devpi:
-  login: dc
-  password: ...
-```