various cleanups, better parallelism and build-dir structure

.circleci/config.yml

@@ -155,13 +155,19 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests:
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
     machine: true
     steps:
       - checkout
       #- attach_workspace:
       #    at: workspace
-      - run: ci_scripts/remote_tests.sh
+      - run: ci_scripts/remote_tests_python.sh
     # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
@@ -192,7 +198,10 @@ workflows:
       - cargo_fetch
       # - build_doxygen
 
-      - remote_tests
+      - remote_tests_rust
+
+      - remote_tests_python
+
       # - upload_docs_wheels:
       #     requires:
       #       - build_test_docs_wheel

ci_scripts/remote_tests.sh

@@ -1,42 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-GITURL=https://github.com/deltachat/deltachat-core-rust
-
-ssh -oStrictHostKeyChecking=no  ci@b1.delta.chat <<_HERE
-    set -xe
-    mkdir -p $BRANCH 
-    cd $BRANCH/
-    echo "--------------------------------------------------"
-    echo "                Checkout"
-    echo "--------------------------------------------------"
-
-    if [ -d "deltachat-core-rust" ] ; then
-        cd deltachat-core-rust 
-        git fetch origin
-        git clean -q -fd
-        git checkout $BRANCH
-        git reset --hard origin/$BRANCH
-    else
-        git clone $GITURL 
-        cd deltachat-core-rust
-        git checkout $BRANCH
-    fi
-    export TARGET=x86_64-unknown-linux-gnu
-
-    echo "--------------------------------------------------"
-    echo "                running rust tests"
-    echo "--------------------------------------------------"
-    bash ci_scripts/run-rust-test.sh
-
-    echo "--------------------------------------------------"
-    echo "             running python tests"
-    echo "--------------------------------------------------"
-    virtualenv -p python3.7 venv 
-    source venv/bin/activate
-    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export CARGO_TARGET_DIR=\`pwd\`/target-py
-   
-    pip install -q tox virtualenv
-    bash ci_scripts/run-python-test.sh 
-_HERE

ci_scripts/remote_tests_python.sh

@@ -0,0 +1,41 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export GITURL=${CIRCLE_REPOSITORY_URL:?git url of core-rust repo}
+
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-tree -r $BRANCH -r --name-only >.rsynclist 
+rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+# we seem to need .git for setuptools_scm versioning 
+rsync -az .git "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+    rm -rf virtualenv venv
+    virtualenv -q -p python3.7 venv 
+    source venv/bin/activate
+    set -x
+
+    pip install -q tox virtualenv
+    bash ci_scripts/run-python-test.sh 
+_HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,33 @@
+#!/bin/bash 
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+
+export GITURL=${CIRCLE_REPOSITORY_URL:?git url of core-rust repo}
+
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-tree -r $BRANCH -r --name-only >.rsynclist 
+rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-python-test.sh

@@ -1,32 +1,32 @@
 #!/bin/bash
 #
-# Run Python functional test for Delta Chat core. 
-#
+# Run functional tests for Delta Chat core using the python bindings 
+# and tox/pytest. 
 
-set -e -x
+set -e +x
 
-# build the core library 
-cargo build --release -p deltachat_ffi --target=$TARGET
+# make sure we have proper settings to run Online tests 
+X=${DCC_PY_LIVECONFIG:?need env var to run Online tests}
+set -x
 
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
+# for core-building and python install step
+export DCC_RS_TARGET=release 
+export DCC_RS_DEV=`pwd`
 
 cd python
 
+python install_python_bindings.py 
+
 # remove and inhibit writing PYC files 
 rm -rf tests/__pycache__
 rm -rf src/deltachat/__pycache__
 export PYTHONDONTWRITEBYTECODE=1
 
-# make sure we have proper settings to run Online tests 
-X=${DCC_PY_LIVECONFIG:?need env var to run Online tests}
-
 # run python tests (tox invokes pytest to run tests in python/tests)
 # we split out qr-tests run to minimize likelyness of flaky tests
 # (some qr tests are pretty heavy in terms of send/received
 # messages and async-imap's likely has concurrency problems, 
 # eg https://github.com/async-email/async-imap/issues/4 )
-tox -e lint,py37 -- --reruns 3 -k "not qr" tests/
-tox -e py37 -- --reruns 5 -k "qr" tests/
+tox -e lint,py37 
 unset DCC_PY_LIVECONFIG
 

ci_scripts/run-rust-test.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-export RUST_TEST_THREADS=1
+#export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1
 export RUSTFLAGS='--deny warnings'
 export OPT="--target=$TARGET"
@@ -38,7 +38,9 @@ else
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 
-# Run all the test configurations:
+# Run all the test configurations 
+# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
+unset RUSTC_WRAPPER
 $CARGO_CMD $CARGO_SUBCMD $OPT
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED

python/install_python_bindings.py

@@ -9,11 +9,16 @@ import subprocess
 import sys
 
 if __name__ == "__main__":
-    os.environ["DCC_RS_TARGET"] = target = "release"
+    target = os.environ.get("DCC_RS_TARGET")
+    if target is None:
+        os.environ["DCC_RS_TARGET"] = target = "release"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
+    # build the core library  in release + debug mode because
+    # as of Nov 2019 rPGP generates RSA keys which take
+    # prohibitively long for non-release installs
     os.environ["RUSTFLAGS"] = "-g"
     subprocess.check_call([
         "cargo", "build", "-p", "deltachat_ffi", "--" + target

python/src/deltachat/_build.py

@@ -29,7 +29,10 @@ def ffibuilder():
             extra_link_args = []
         else:
             raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
         assert os.path.exists(objs[0]), objs
         incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:

python/tox.ini

@@ -1,19 +1,22 @@
 [tox]
 # make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py35
+    py37
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 -v -rsXx {posargs:tests}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pytest -n6 -v -rsXx -k "not qr" {posargs:tests} 
+    pytest -n6 -v -rsXx -k "qr" {posargs:tests} 
+    # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
+    CARGO_TARGET_DIR
+    RUSTC_WRAPPER
 deps = 
     pytest
     pytest-rerunfailures