Rename ci_scripts/ into scripts/

scripts/README.md

@@ -0,0 +1,50 @@
+# Continuous Integration Scripts for Delta Chat
+
+Continuous Integration, run through [GitHub
+Actions](https://docs.github.com/actions),
+[CircleCI](https://app.circleci.com/) and an own build machine.
+
+## Description of scripts 
+
+- `../.github/workflows` contains jobs run by GitHub Actions.
+
+- `../.circleci/config.yml` describing the build jobs that are run
+  by CircleCI.
+
+- `remote_tests_python.sh` rsyncs to a build machine and runs
+  `run-python-test.sh` remotely on the build machine. 
+
+- `remote_tests_rust.sh` rsyncs to the build machine and runs
+  `run-rust-test.sh` remotely on the build machine. 
+
+- `doxygen/Dockerfile` specifies an image that contains
+  the doxygen tool which is used by `run-doxygen.sh` 
+  to generate C-docs which are then uploaded 
+  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+  (and the master branch is linked to https://c.delta.chat proper). 
+
+
+## Triggering runs on the build machine locally (fast!)
+
+There is experimental support for triggering a remote Python or Rust test run 
+from your local checkout/branch. You will need to be authorized to login to 
+the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
+
+    scripts/manual_remote_tests.sh rust
+    scripts/manual_remote_tests.sh python
+
+This will **rsync** your current checkout to the remote build machine 
+(no need to commit before) and then run either rust or python tests. 
+
+# Outdated files (for later re-use)
+
+`coredeps/Dockerfile` specifies an image that contains all 
+of Delta Chat's core dependencies. It used to run 
+python tests and build wheels (binary packages for Python)
+
+You can build the docker images yourself locally
+to avoid the relatively large download:: 
+ 
+    cd scripts  # where all CI things are 
+    docker build -t deltachat/coredeps docker-coredeps
+    docker build -t deltachat/doxygen docker-doxygen 

scripts/ci_upload.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
+
+
+# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
+export BRANCH=${CIRCLE_BRANCH:master}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
+
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
+
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

scripts/cleanup_devpi_indices.py

@@ -0,0 +1,72 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+    response = session.get(baseurl + username)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+    response = session.get(baseurl + username + "/" + indexname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    dates = set()
+    for value in result.values():
+        if "+links" not in value:
+            continue
+        for link in value["+links"]:
+            for log in link["log"]:
+                dates.add(tuple(log["when"]))
+    return dates
+
+
+def run():
+    baseurl = "https://m.devpi.net/"
+    username = "dc"
+    for indexname in get_indexes(baseurl, username):
+        projectnames = get_projectnames(baseurl, username, indexname)
+        if indexname == "master" or not indexname:
+            continue
+        clear_index = not projectnames
+        for projectname in projectnames:
+            dates = get_release_dates(baseurl, username, indexname, projectname)
+            if not dates:
+                print(
+                    "%s has no releases" % (baseurl + username + "/" + indexname),
+                    file=sys.stderr)
+                date = datetime.datetime.now()
+            else:
+                date = datetime.datetime(*max(dates))
+            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+                assert username and indexname
+                clear_index = True
+                break
+        if clear_index:
+            url = baseurl + username + "/" + indexname
+            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+    run()

scripts/coverage.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+
+if ! which grcov 2>/dev/null 1>&2; then
+    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
+    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
+    exit 1
+fi
+
+# Allow `-Z` flags without using nightly Rust.
+export RUSTC_BOOTSTRAP=1
+
+# We are using `-Zprofile` instead of source-based coverage [1]
+# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
+#
+# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
+# [2] https://github.com/mozilla/grcov/issues/595
+
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTDOCFLAGS="-Cpanic=abort"
+cargo clean
+cargo build
+cargo test
+
+grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/docker-coredeps/Dockerfile

@@ -0,0 +1,21 @@
+FROM quay.io/pypa/manylinux2010_x86_64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+# Install a recent Perl, needed to install the openssl crate 
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_openssl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
+
+./Configure -de
+make
+make install

scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share

scripts/docker-doxygen/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:stable
+
+# this is tagged as deltachat/doxygen 
+
+RUN apt-get update && apt-get install -y doxygen 

scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash scripts/$CIRCLE_JOB.sh 

scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

scripts/old/run-python.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

scripts/remote_python_packaging.sh

@@ -0,0 +1,49 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    shopt -s huponexit
+    cd $BUILDDIR
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

scripts/remote_tests_python.sh

@@ -0,0 +1,46 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    export RUSTC_WRAPPER=\`which sccache\`
+    cd $BUILDDIR
+    export TARGET=release
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+
+    #we rely on tox/virtualenv being available in the host
+    #rm -rf virtualenv venv
+    #virtualenv -q -p python3.7 venv 
+    #source venv/bin/activate
+    #pip install -q tox virtualenv
+
+    set -x
+    which python
+    source \$HOME/venv/bin/activate
+    which python
+
+    bash scripts/run-python-test.sh 
+_HERE

scripts/remote_tests_rust.sh

@@ -0,0 +1,30 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
+    cd $BUILDDIR
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash scripts/run-rust-test.sh
+_HERE
+

scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

scripts/run-python-test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Run functional tests for Delta Chat core using the python bindings 
+# and tox/pytest. 
+
+set -e -x
+shopt -s huponexit
+
+# for core-building and python install step
+export DCC_RS_TARGET=debug
+export DCC_RS_DEV=`pwd`
+
+cd python
+
+python install_python_bindings.py onlybuild
+
+# remove and inhibit writing PYC files 
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+export PYTHONDONTWRITEBYTECODE=1
+
+# run python tests (tox invokes pytest to run tests in python/tests)
+#TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
+tox -e lint
+tox -e doc,py37

scripts/run-rust-test.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -ex
+shopt -s huponexit
+
+#export RUST_TEST_THREADS=1
+export RUST_BACKTRACE=1
+export RUSTFLAGS='--deny warnings'
+export OPT="--target=$TARGET"
+export OPT_RELEASE="--release ${OPT}"
+export OPT_FFI_RELEASE="--manifest-path=deltachat-ffi/Cargo.toml --release"
+
+# Select cargo command: use cross by default
+export CARGO_CMD=cross
+
+# On Appveyor (windows) and Travis (x86_64-unknown-linux-gnu and apple) native targets we use cargo (no need to cross-compile):
+if [[ $TARGET = *"windows"* ]] || [[ $TARGET == "x86_64-unknown-linux-gnu" ]] || [[ $TARGET = *"apple"* ]]; then
+    export CARGO_CMD=cargo
+fi
+
+# Install cross if necessary:
+if [[ $CARGO_CMD == "cross" ]]; then
+    cargo install --git https://github.com/dignifiedquire/cross --rev fix-tty --force
+fi
+
+# Make sure TARGET is installed when using cargo:
+if [[ $CARGO_CMD == "cargo" ]]; then
+    rustup target add $TARGET || true
+fi
+
+# If the build should not run tests, just check that the code builds:
+if [[ $NORUN == "1" ]]; then
+    export CARGO_SUBCMD="build"
+else
+    export CARGO_SUBCMD="test --all"
+    export OPT="${OPT} "
+    export OPT_RELEASE="${OPT_RELEASE} "
+    export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
+fi
+
+# Run all the test configurations 
+# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
+unset RUSTC_WRAPPER
+$CARGO_CMD $CARGO_SUBCMD $OPT
+$CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
+$CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED

scripts/run_all.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Build the Delta Chat Core Rust library, Python wheels and docs 
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# compile core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+export DCC_RS_TARGET=release
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
+popd
+
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
+
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances 
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already. 
+unset DCC_NEW_TMP_EMAIL
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)