From 254b0619214439df72194afe2120e7897f56f37a Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 22 Nov 2019 00:48:56 +0100 Subject: [PATCH] update docs and add a simple manual script to run python/rust tests --- ci_scripts/README.md | 68 +++++++++++------------- ci_scripts/ci_upload.sh | 16 +++--- ci_scripts/manual_remote_tests.sh | 9 ++++ ci_scripts/{ => old}/gh-actions-rust.yml | 0 ci_scripts/{ => old}/run-python.sh | 0 ci_scripts/remote_tests_python.sh | 12 +++-- ci_scripts/remote_tests_rust.sh | 4 +- ci_scripts/run-python-test.sh | 12 ++--- python/install_python_bindings.py | 7 +-- python/tox.ini | 20 +++++-- 10 files changed, 81 insertions(+), 67 deletions(-) create mode 100755 ci_scripts/manual_remote_tests.sh rename ci_scripts/{ => old}/gh-actions-rust.yml (100%) rename ci_scripts/{ => old}/run-python.sh (100%) diff --git a/ci_scripts/README.md b/ci_scripts/README.md index 6a2397542..9742474d1 100644 --- a/ci_scripts/README.md +++ b/ci_scripts/README.md @@ -1,52 +1,46 @@ - # Continuous Integration Scripts for Delta Chat -Continuous Integration is run through CircleCI -but is largely independent of it. +Continuous Integration, run through CircleCI and an own build machine. + +## Description of scripts + +- `../.circleci/config.yml` describing the build jobs that are run + by Circle-CI + +- `remote_tests_python.sh` rsyncs to a build machine and runs + `run-python-test.sh` remotely on the build machine. + +- `remote_tests_rust.sh` rsyncs to the build machine and runs + `run-rust-test.sh` remotely on the build machine. + +- `doxygen/Dockerfile` specifies an image that contains + the doxygen tool which is used by `run-doxygen.sh` + to generate C-docs which are then uploaded + via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/` + (and the master branch is linked to https://c.delta.chat proper). -## Generating docker containers for performing build step work +## Triggering runs on the build machine locally (fast!) -All tests, docs and wheel building is run in docker containers: +There is experimental support for triggering a remote Python or Rust test run +from your local checkout/branch. You will need to be authorized to login to +the build machine (ask your friendly sysadmin on #deltachat freenode) to type:: -- **coredeps/Dockerfile** specifies an image that contains all - of Delta Chat's core dependencies as linkable libraries. - It also serves to run python tests and build wheels - (binary packages for Python). + ci_scripts/manual_remote_tests.sh rust + ci_scripts/manual_remote_tests.sh python -- **doxygen/Dockerfile** specifies an image that contains - the doxygen tool which is used to generate C-docs. +This will **rsync** your current checkout to the remote build machine +(no need to commit before) and then run either rust or python tests. -To run tests locally you can pull existing images from "docker.io", -the hub for sharing Docker images:: +# Outdated files (for later re-use) - docker pull deltachat/coredeps - docker pull deltachat/doxygen +`coredeps/Dockerfile` specifies an image that contains all +of Delta Chat's core dependencies. It used to run +python tests and build wheels (binary packages for Python) -or you can build the docker images yourself locally +You can build the docker images yourself locally to avoid the relatively large download:: cd ci_scripts # where all CI things are docker build -t deltachat/coredeps docker-coredeps docker build -t deltachat/doxygen docker-doxygen - -## ci_run.sh (main entrypoint called by circle-ci) - -Once you have the docker images available -you can run python testing, documentation generation -and building binary wheels:: - - sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh - -## ci_upload.sh (uploading artifacts on success) - -- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/` - -- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/` - -- python wheels to `https://m.devpi.net/dc/` - so that you install fully self-contained wheels like this: - `pip install -U -i https://m.devpi.net/dc/ deltachat` - - - diff --git a/ci_scripts/ci_upload.sh b/ci_scripts/ci_upload.sh index 75f0275c4..50abe69b6 100755 --- a/ci_scripts/ci_upload.sh +++ b/ci_scripts/ci_upload.sh @@ -14,12 +14,12 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found} export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes} -# python docs to py.delta.chat -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH} -rsync -avz \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - "$PYDOCDIR/html/" \ - delta@py.delta.chat:build/${BRANCH} +# DISABLED: python docs to py.delta.chat +#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH} +#rsync -avz \ +# -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ +# "$PYDOCDIR/html/" \ +# delta@py.delta.chat:build/${BRANCH} # C docs to c.delta.chat ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH} @@ -28,6 +28,10 @@ rsync -avz \ "$DOXYDOCDIR/html/" \ delta@c.delta.chat:build-c/${BRANCH} +exit 0 + +# OUTDATED -- for re-use from python release-scripts + echo ----------------------- echo upload wheels echo ----------------------- diff --git a/ci_scripts/manual_remote_tests.sh b/ci_scripts/manual_remote_tests.sh new file mode 100755 index 000000000..295ca2958 --- /dev/null +++ b/ci_scripts/manual_remote_tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -xe +export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} +export CIRCLE_BUILD_NUM=$USER +export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2` +export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`) + +time bash ci_scripts/$CIRCLE_JOB.sh diff --git a/ci_scripts/gh-actions-rust.yml b/ci_scripts/old/gh-actions-rust.yml similarity index 100% rename from ci_scripts/gh-actions-rust.yml rename to ci_scripts/old/gh-actions-rust.yml diff --git a/ci_scripts/run-python.sh b/ci_scripts/old/run-python.sh similarity index 100% rename from ci_scripts/run-python.sh rename to ci_scripts/old/run-python.sh diff --git a/ci_scripts/remote_tests_python.sh b/ci_scripts/remote_tests_python.sh index b8e2a29b8..283313776 100755 --- a/ci_scripts/remote_tests_python.sh +++ b/ci_scripts/remote_tests_python.sh @@ -8,15 +8,17 @@ export SSHTARGET=ci@b1.delta.chat # CARGO_TARGET_DIR between runs ("..") export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number} -set -e - echo "--- Copying files to $SSHTARGET:$BUILDDIR" +set -xe + ssh -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" -git ls-tree -r $BRANCH -r --name-only >.rsynclist -rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" +git ls-files >.rsynclist # we seem to need .git for setuptools_scm versioning -rsync -az .git "$SSHTARGET:$BUILDDIR" +find .git >>.rsynclist +rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" + +set +x echo "--- Running $CIRCLE_JOB remotely" diff --git a/ci_scripts/remote_tests_rust.sh b/ci_scripts/remote_tests_rust.sh index edbe7977b..d9d9c7080 100755 --- a/ci_scripts/remote_tests_rust.sh +++ b/ci_scripts/remote_tests_rust.sh @@ -13,8 +13,8 @@ set -e echo "--- Copying files to $SSHTARGET:$BUILDDIR" ssh -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" -git ls-tree -r $BRANCH -r --name-only >.rsynclist -rsync --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" +git ls-files >.rsynclist +rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" echo "--- Running $CIRCLE_JOB remotely" diff --git a/ci_scripts/run-python-test.sh b/ci_scripts/run-python-test.sh index cb2659cfa..f6aaf251e 100755 --- a/ci_scripts/run-python-test.sh +++ b/ci_scripts/run-python-test.sh @@ -3,11 +3,7 @@ # Run functional tests for Delta Chat core using the python bindings # and tox/pytest. -set -e +x - -# make sure we have proper settings to run Online tests -X=${DCC_PY_LIVECONFIG:?need env var to run Online tests} -set -x +set -e -x # for core-building and python install step export DCC_RS_TARGET=release @@ -15,7 +11,7 @@ export DCC_RS_DEV=`pwd` cd python -python install_python_bindings.py +python install_python_bindings.py onlybuild # remove and inhibit writing PYC files rm -rf tests/__pycache__ @@ -23,6 +19,4 @@ rm -rf src/deltachat/__pycache__ export PYTHONDONTWRITEBYTECODE=1 # run python tests (tox invokes pytest to run tests in python/tests) -tox -e lint,py37 -unset DCC_PY_LIVECONFIG - +tox -e lintdoc,py37 diff --git a/python/install_python_bindings.py b/python/install_python_bindings.py index 7eb7f4bf7..ee891193f 100755 --- a/python/install_python_bindings.py +++ b/python/install_python_bindings.py @@ -25,6 +25,7 @@ if __name__ == "__main__": ]) subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True) - subprocess.check_call([ - sys.executable, "-m", "pip", "install", "-e", "." - ]) + if len(sys.argv) > 1 and sys.argv[1] != "onlybuild": + subprocess.check_call([ + sys.executable, "-m", "pip", "install", "-e", "." + ]) diff --git a/python/tox.ini b/python/tox.ini index 8351467eb..32efe8df0 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -34,7 +34,6 @@ deps = auditwheel commands = python tests/auditwheels.py {toxworkdir}/wheelhouse - [testenv:lint] skipsdist = True usedevelop = True @@ -49,14 +48,25 @@ commands = rst-lint --encoding 'utf-8' README.rst [testenv:doc] -basepython = python3.5 +skipsdist = True +usedevelop = True deps = sphinx==2.2.0 breathe - -changedir = doc commands = - sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html + sphinx-build -w toxdoc-warnings.log -b html doc doc/_build/html + + +[testenv:lintdoc] +skipsdist = True +usedevelop = True +deps = + {[testenv:lint]deps} + {[testenv:doc]deps} +commands = + {[testenv:lint]commands} + {[testenv:doc]commands} + [pytest]