diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e9983d4b..2947dbbe1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,13 +155,19 @@ jobs: - py-docs - wheelhouse - remote_tests: + remote_tests_rust: + machine: true + steps: + - checkout + - run: ci_scripts/remote_tests_rust.sh + + remote_tests_python: machine: true steps: - checkout #- attach_workspace: # at: workspace - - run: ci_scripts/remote_tests.sh + - run: ci_scripts/remote_tests_python.sh # workspace/py-docs workspace/wheelhouse workspace/c-docs upload_docs_wheels: @@ -192,7 +198,10 @@ workflows: - cargo_fetch # - build_doxygen - - remote_tests + - remote_tests_rust + + - remote_tests_python + # - upload_docs_wheels: # requires: # - build_test_docs_wheel diff --git a/ci_scripts/remote_tests.sh b/ci_scripts/remote_tests.sh deleted file mode 100755 index 726ea6963..000000000 --- a/ci_scripts/remote_tests.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -export BRANCH=${CIRCLE_BRANCH:?branch to build} -GITURL=https://github.com/deltachat/deltachat-core-rust - -ssh -oStrictHostKeyChecking=no ci@b1.delta.chat <<_HERE - set -xe - mkdir -p $BRANCH - cd $BRANCH/ - echo "--------------------------------------------------" - echo " Checkout" - echo "--------------------------------------------------" - - if [ -d "deltachat-core-rust" ] ; then - cd deltachat-core-rust - git fetch origin - git clean -q -fd - git checkout $BRANCH - git reset --hard origin/$BRANCH - else - git clone $GITURL - cd deltachat-core-rust - git checkout $BRANCH - fi - export TARGET=x86_64-unknown-linux-gnu - - echo "--------------------------------------------------" - echo " running rust tests" - echo "--------------------------------------------------" - bash ci_scripts/run-rust-test.sh - - echo "--------------------------------------------------" - echo " running python tests" - echo "--------------------------------------------------" - virtualenv -p python3.7 venv - source venv/bin/activate - export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG - export CARGO_TARGET_DIR=\`pwd\`/target-py - - pip install -q tox virtualenv - bash ci_scripts/run-python-test.sh -_HERE diff --git a/ci_scripts/remote_tests_python.sh b/ci_scripts/remote_tests_python.sh new file mode 100755 index 000000000..2dd1bf255 --- /dev/null +++ b/ci_scripts/remote_tests_python.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +export BRANCH=${CIRCLE_BRANCH:?branch to build} +export GITURL=${CIRCLE_REPOSITORY_URL:?git url of core-rust repo} + +export SSHTARGET=ci@b1.delta.chat + +# we construct the BUILDDIR such that we can easily share the +# CARGO_TARGET_DIR between runs ("..") +export BUILDDIR=ci_builds/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number} + +set -e + +echo "--- Copying files to $SSHTARGET:$BUILDDIR" + +ssh -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" +git ls-tree -r $BRANCH -r --name-only >.rsynclist +rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" +# we seem to need .git for setuptools_scm versioning +rsync -az .git "$SSHTARGET:$BUILDDIR" + +echo "--- Running $CIRCLE_JOB remotely" + +ssh $SSHTARGET <<_HERE + set +x -e + cd $BUILDDIR + # let's share the target dir with our last run on this branch/job-type + # cargo will make sure to block/unblock us properly + export CARGO_TARGET_DIR=\`pwd\`/../target + export TARGET=x86_64-unknown-linux-gnu + export RUSTC_WRAPPER=sccache + export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG + + rm -rf virtualenv venv + virtualenv -q -p python3.7 venv + source venv/bin/activate + set -x + + pip install -q tox virtualenv + bash ci_scripts/run-python-test.sh +_HERE diff --git a/ci_scripts/remote_tests_rust.sh b/ci_scripts/remote_tests_rust.sh new file mode 100755 index 000000000..b311440eb --- /dev/null +++ b/ci_scripts/remote_tests_rust.sh @@ -0,0 +1,33 @@ +#!/bin/bash +export BRANCH=${CIRCLE_BRANCH:?branch to build} + +export GITURL=${CIRCLE_REPOSITORY_URL:?git url of core-rust repo} + +export SSHTARGET=ci@b1.delta.chat + +# we construct the BUILDDIR such that we can easily share the +# CARGO_TARGET_DIR between runs ("..") +export BUILDDIR=ci_builds/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number} + +set -e + +echo "--- Copying files to $SSHTARGET:$BUILDDIR" + +ssh -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" +git ls-tree -r $BRANCH -r --name-only >.rsynclist +rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" + +echo "--- Running $CIRCLE_JOB remotely" + +ssh $SSHTARGET <<_HERE + set +x -e + cd $BUILDDIR + # let's share the target dir with our last run on this branch/job-type + # cargo will make sure to block/unblock us properly + export CARGO_TARGET_DIR=\`pwd\`/../target + export TARGET=x86_64-unknown-linux-gnu + export RUSTC_WRAPPER=sccache + + bash ci_scripts/run-rust-test.sh +_HERE + diff --git a/ci_scripts/run-python-test.sh b/ci_scripts/run-python-test.sh index 7080c10b5..7ca15b00a 100755 --- a/ci_scripts/run-python-test.sh +++ b/ci_scripts/run-python-test.sh @@ -1,32 +1,32 @@ #!/bin/bash # -# Run Python functional test for Delta Chat core. -# +# Run functional tests for Delta Chat core using the python bindings +# and tox/pytest. -set -e -x +set -e +x -# build the core library -cargo build --release -p deltachat_ffi --target=$TARGET +# make sure we have proper settings to run Online tests +X=${DCC_PY_LIVECONFIG:?need env var to run Online tests} +set -x -# Statically link against libdeltachat.a. -export DCC_RS_DEV=$(pwd) +# for core-building and python install step +export DCC_RS_TARGET=release +export DCC_RS_DEV=`pwd` cd python +python install_python_bindings.py + # remove and inhibit writing PYC files rm -rf tests/__pycache__ rm -rf src/deltachat/__pycache__ export PYTHONDONTWRITEBYTECODE=1 -# make sure we have proper settings to run Online tests -X=${DCC_PY_LIVECONFIG:?need env var to run Online tests} - # run python tests (tox invokes pytest to run tests in python/tests) # we split out qr-tests run to minimize likelyness of flaky tests # (some qr tests are pretty heavy in terms of send/received # messages and async-imap's likely has concurrency problems, # eg https://github.com/async-email/async-imap/issues/4 ) -tox -e lint,py37 -- --reruns 3 -k "not qr" tests/ -tox -e py37 -- --reruns 5 -k "qr" tests/ +tox -e lint,py37 unset DCC_PY_LIVECONFIG diff --git a/ci_scripts/run-rust-test.sh b/ci_scripts/run-rust-test.sh index a99d5ff7d..e0d600c5e 100755 --- a/ci_scripts/run-rust-test.sh +++ b/ci_scripts/run-rust-test.sh @@ -2,7 +2,7 @@ set -ex -export RUST_TEST_THREADS=1 +#export RUST_TEST_THREADS=1 export RUST_BACKTRACE=1 export RUSTFLAGS='--deny warnings' export OPT="--target=$TARGET" @@ -38,7 +38,9 @@ else export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored" fi -# Run all the test configurations: +# Run all the test configurations +# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test +unset RUSTC_WRAPPER $CARGO_CMD $CARGO_SUBCMD $OPT $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED diff --git a/python/install_python_bindings.py b/python/install_python_bindings.py index 1bada38ad..7eb7f4bf7 100755 --- a/python/install_python_bindings.py +++ b/python/install_python_bindings.py @@ -9,11 +9,16 @@ import subprocess import sys if __name__ == "__main__": - os.environ["DCC_RS_TARGET"] = target = "release" + target = os.environ.get("DCC_RS_TARGET") + if target is None: + os.environ["DCC_RS_TARGET"] = target = "release" if "DCC_RS_DEV" not in os.environ: dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.environ["DCC_RS_DEV"] = dn + # build the core library in release + debug mode because + # as of Nov 2019 rPGP generates RSA keys which take + # prohibitively long for non-release installs os.environ["RUSTFLAGS"] = "-g" subprocess.check_call([ "cargo", "build", "-p", "deltachat_ffi", "--" + target diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 69820b1db..7e0039fa4 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -29,7 +29,10 @@ def ffibuilder(): extra_link_args = [] else: raise NotImplementedError("Compilation not supported yet on Windows, can you help?") - objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')] + target_dir = os.environ.get("CARGO_TARGET_DIR") + if target_dir is None: + target_dir = os.path.join(projdir, 'target') + objs = [os.path.join(target_dir, target, 'libdeltachat.a')] assert os.path.exists(objs[0]), objs incs = [os.path.join(projdir, 'deltachat-ffi')] else: diff --git a/python/tox.ini b/python/tox.ini index c5adc4670..3c7c93764 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,19 +1,22 @@ [tox] # make sure to update environment list in travis.yml and appveyor.yml envlist = - py35 + py37 lint auditwheels [testenv] commands = - pytest -n6 -v -rsXx {posargs:tests} - python tests/package_wheels.py {toxworkdir}/wheelhouse + pytest -n6 -v -rsXx -k "not qr" {posargs:tests} + pytest -n6 -v -rsXx -k "qr" {posargs:tests} + # python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS DCC_RS_DEV DCC_RS_TARGET DCC_PY_LIVECONFIG + CARGO_TARGET_DIR + RUSTC_WRAPPER deps = pytest pytest-rerunfailures