improve python caching (#468)

(@dignifiedquire and @hpk42) 
- introduce rust-caching to python test runs 
- skip release and ffi runs, they are check using python bindings
- shuffle files such that ci_scripts/ contains all the ci scripts
- partly parallelize python tox runs

ci_scripts/ci_run.sh

@@ -1,22 +0,0 @@
-# perform CI jobs on PRs and after merges to master.
-# triggered from .circleci/config.yml
-
-set -e -x
-
-export BRANCH=${CIRCLE_BRANCH:-test7}
-    
-# run doxygen on c-source (needed by later doc-generation steps).
-# XXX modifies the host filesystem docs/xml and docs/html directories
-# XXX which you can then only remove with sudo as they belong to root
-
-# XXX we don't do doxygen doc generation with Rust anymore, needs to be
-# substituted with rust-docs 
-#if [ -n "$DOCS" ] ; then 
-#    docker run --rm -it -v $PWD:/mnt -w /mnt/docs deltachat/doxygen doxygen
-#fi
-
-# run everything else inside docker (TESTS, DOCS, WHEELS) 
-docker run -e DCC_PY_LIVECONFIG -e BRANCH -e TESTS -e DOCS \
-           --rm -it -v $(pwd):/mnt -w /mnt \
-           deltachat/coredeps ci_scripts/run_all.sh
-

ci_scripts/run-python.sh

@@ -36,20 +36,21 @@ if [ -n "$TESTS" ]; then
     rm -rf src/deltachat/__pycache__
     export PYTHONDONTWRITEBYTECODE=1
 
-    # run tox
-    # XXX we don't run liveconfig tests because they hang sometimes
-    # see https://github.com/deltachat/deltachat-core-rust/issues/331
-    # unset DCC_PY_LIVECONFIG
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
 
-    tox --workdir "$TOXWORKDIR" -e lint,py35,py36,py37,auditwheels -- -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py35,py36,py37 -- -k "qr"
+    tox --workdir "$TOXWORKDIR" -e py37 
+    unset DCC_PY_LIVECONFIG
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
     popd
 fi
 
 
-if [ -n "$DOCS" ]; then
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-fi
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/run-rust-test.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export RUST_TEST_THREADS=1
+export RUST_BACKTRACE=1
+export RUSTFLAGS='--deny warnings'
+export OPT="--target=$TARGET"
+export OPT_RELEASE="--release ${OPT}"
+export OPT_FFI_RELEASE="--manifest-path=deltachat-ffi/Cargo.toml --release"
+
+# Select cargo command: use cross by default
+export CARGO_CMD=cross
+
+# On Appveyor (windows) and Travis (x86_64-unknown-linux-gnu and apple) native targets we use cargo (no need to cross-compile):
+if [[ $TARGET = *"windows"* ]] || [[ $TARGET == "x86_64-unknown-linux-gnu" ]] || [[ $TARGET = *"apple"* ]]; then
+    export CARGO_CMD=cargo
+fi
+
+# Install cross if necessary:
+if [[ $CARGO_CMD == "cross" ]]; then
+    cargo install --git https://github.com/dignifiedquire/cross --rev fix-tty --force
+fi
+
+# Make sure TARGET is installed when using cargo:
+if [[ $CARGO_CMD == "cargo" ]]; then
+    rustup target add $TARGET || true
+fi
+
+# If the build should not run tests, just check that the code builds:
+if [[ $NORUN == "1" ]]; then
+    export CARGO_SUBCMD="build"
+else
+    export CARGO_SUBCMD="test"
+    export OPT="${OPT} "
+    export OPT_RELEASE="${OPT_RELEASE} "
+    export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
+fi
+
+# Run all the test configurations:
+$CARGO_CMD $CARGO_SUBCMD $OPT
+$CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED