reconfigure running of rust and python tests

.circleci/config.yml

@@ -155,6 +155,15 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests:
+    machine: true
+    steps:
+      - checkout
+      #- attach_workspace:
+      #    at: workspace
+      - run: ci_scripts/remote_tests.sh
+    # workspace/py-docs workspace/wheelhouse workspace/c-docs
+
   upload_docs_wheels:
     machine: true
     steps:
@@ -181,15 +190,13 @@ workflows:
   test:
     jobs:
       - cargo_fetch
-      - build_doxygen
+      # - build_doxygen
 
-      - build_test_docs_wheel:
+      - remote_tests
-          requires:
+      # - upload_docs_wheels:
-            - cargo_fetch
+      #     requires:
-      - upload_docs_wheels:
+      #       - build_test_docs_wheel
-          requires:
+      #       - build_doxygen
-            - build_test_docs_wheel
-            - build_doxygen
       - rustfmt:
           requires:
             - cargo_fetch
@@ -198,9 +205,9 @@ workflows:
            - cargo_fetch
 
       # Linux Desktop 64bit
-      - test_x86_64-unknown-linux-gnu:
+      # - test_x86_64-unknown-linux-gnu:
-          requires:
+      #     requires:
-            - cargo_fetch
+      #       - cargo_fetch
 
       # Linux Desktop 32bit
       # - test_i686-unknown-linux-gnu:

ci_scripts/remote_tests.sh

@@ -0,0 +1,42 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+GITURL=https://github.com/deltachat/deltachat-core-rust
+
+ssh -oStrictHostKeyChecking=no  ci@b1.delta.chat <<_HERE
+    set -xe
+    mkdir -p $BRANCH 
+    cd $BRANCH/
+    echo "--------------------------------------------------"
+    echo "                Checkout"
+    echo "--------------------------------------------------"
+
+    if [ -d "deltachat-core-rust" ] ; then
+        cd deltachat-core-rust 
+        git fetch origin
+        git clean -q -fd
+        git checkout $BRANCH
+        git reset --hard origin/$BRANCH
+    else
+        git clone $GITURL 
+        cd deltachat-core-rust
+        git checkout $BRANCH
+    fi
+    export TARGET=x86_64-unknown-linux-gnu
+
+    echo "--------------------------------------------------"
+    echo "                running rust tests"
+    echo "--------------------------------------------------"
+    bash ci_scripts/run-rust-test.sh
+
+    echo "--------------------------------------------------"
+    echo "             running python tests"
+    echo "--------------------------------------------------"
+    virtualenv -p python3.7 venv 
+    source venv/bin/activate
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export CARGO_TARGET_DIR=\`pwd\`/target-py
+   
+    pip install -q tox virtualenv
+    bash ci_scripts/run-python-test.sh 
+_HERE

ci_scripts/run-python-test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Run Python functional test for Delta Chat core. 
+#
+
+set -e -x
+
+# build the core library 
+cargo build --release -p deltachat_ffi --target=$TARGET
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+cd python
+
+# remove and inhibit writing PYC files 
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+export PYTHONDONTWRITEBYTECODE=1
+
+# run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+# allows running of "liveconfig" tests but for speed reasons
+# we run them only for the highest python version we support
+
+# we split out qr-tests run to minimize likelyness of flaky tests
+# (some qr tests are pretty heavy in terms of send/received
+# messages and async-imap's likely has concurrency problems, 
+# eg https://github.com/async-email/async-imap/issues/4 )
+tox -e lint,py37 -- --reruns 3 -k "not qr"
+tox -e py37 -- --reruns 5 -k "qr"
+unset DCC_PY_LIVECONFIG
+tox  -p4 -e lint,py35,py36,doc
+

ci_scripts/run-rust-test.sh

@@ -10,7 +10,8 @@ export OPT_RELEASE="--release ${OPT}"
 export OPT_FFI_RELEASE="--manifest-path=deltachat-ffi/Cargo.toml --release"
 
 # Select cargo command: use cross by default
-export CARGO_CMD=cross
+# export CARGO_CMD=cross
+export CARGO_CMD=cargo
 
 # On Appveyor (windows) and Travis (x86_64-unknown-linux-gnu and apple) native targets we use cargo (no need to cross-compile):
 if [[ $TARGET = *"windows"* ]] || [[ $TARGET == "x86_64-unknown-linux-gnu" ]] || [[ $TARGET = *"apple"* ]]; then
@@ -18,9 +19,9 @@ if [[ $TARGET = *"windows"* ]] || [[ $TARGET == "x86_64-unknown-linux-gnu" ]] ||
 fi
 
 # Install cross if necessary:
-if [[ $CARGO_CMD == "cross" ]]; then
+#if [[ $CARGO_CMD == "cross" ]]; then
-    cargo install --git https://github.com/dignifiedquire/cross --rev fix-tty --force
+#    cargo install --git https://github.com/dignifiedquire/cross --rev fix-tty --force
-fi
+##fi
 
 # Make sure TARGET is installed when using cargo:
 if [[ $CARGO_CMD == "cargo" ]]; then
@@ -32,8 +33,8 @@ if [[ $NORUN == "1" ]]; then
     export CARGO_SUBCMD="build"
 else
     export CARGO_SUBCMD="test --all"
-    export OPT="-j1 ${OPT} "
+    export OPT="${OPT} "
-    export OPT_RELEASE="-j2 ${OPT_RELEASE} "
+    export OPT_RELEASE="${OPT_RELEASE} "
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -v -rsXx {posargs:tests}
+    pytest -n6 -v -rsXx {posargs:tests}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

src/imap.rs

@@ -780,10 +780,8 @@ impl Imap {
             let timeout = Duration::from_secs(23 * 60);
             if let Some(session) = session {
                 match session.idle() {
-
                     // BEWARE: If you change the Secure branch you
                     // typically also need to change the Insecure branch.
-
                     IdleHandle::Secure(mut handle) => {
                         if let Err(err) = handle.init().await {
                             warn!(context, "Failed to establish IDLE connection: {:?}", err);
@@ -810,9 +808,12 @@ impl Imap {
                             Err(err) => {
                                 // if we cannot terminate IDLE it probably
                                 // means that we waited long (with idle_wait)
-                                // but the network went away/changed 
+                                // but the network went away/changed
                                 self.trigger_reconnect();
-                                warn!(context, "Failed to terminate IMAP IDLE connection: {:?}", err);
+                                warn!(
+                                    context,
+                                    "Failed to terminate IMAP IDLE connection: {:?}", err
+                                );
                             }
                         }
                     }
@@ -843,7 +844,7 @@ impl Imap {
                             Err(err) => {
                                 // if we cannot terminate IDLE it probably
                                 // means that we waited long (with idle_wait)
-                                // but the network went away/changed 
+                                // but the network went away/changed
                                 self.trigger_reconnect();
                                 warn!(context, "Failed to close IMAP IDLE connection: {:?}", err);
                             }