diff --git a/.circleci/config.yml b/.circleci/config.yml index bd44ab540..9e33b9987 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,32 +120,19 @@ jobs: paths: - c-docs - # build_test_docs_wheel: - # docker: - # - image: deltachat/coredeps - # environment: - # TESTS: 1 - # DOCS: 1 - # working_directory: /mnt/crate - # steps: - # - *restore-workspace - # - *restore-cache - # - run: - # name: build docs, run tests and build wheels - # command: ci_scripts/run-python.sh - # - run: - # name: copying docs and wheels to workspace - # command: | - # mkdir -p workspace/python - # # cp -av docs workspace/c-docs - # cp -av python/.docker-tox/wheelhouse workspace/ - # cp -av python/doc/_build/ workspace/py-docs - # - persist_to_workspace: - # root: workspace - # paths: - # # - c-docs - # - py-docs - # - wheelhouse + remote_python_packaging: + machine: true + steps: + - checkout + # the following commands on success produces + # workspace/{wheelhouse,py-docs} as artefact directories + - run: bash ci_scripts/remote_python_packaging.sh + - persist_to_workspace: + root: workspace + paths: + # - c-docs + - py-docs + - wheelhouse remote_tests_rust: machine: true @@ -157,10 +144,7 @@ jobs: machine: true steps: - checkout - #- attach_workspace: - # at: workspace - run: ci_scripts/remote_tests_python.sh - # workspace/py-docs workspace/wheelhouse workspace/c-docs upload_docs_wheels: machine: true @@ -168,6 +152,7 @@ jobs: - checkout - attach_workspace: at: workspace + - run: pyenv versions - run: pyenv global 3.5.2 - run: ls -laR workspace - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs @@ -193,10 +178,15 @@ workflows: - remote_tests_python - # - upload_docs_wheels: - # requires: - # - build_test_docs_wheel - # - build_doxygen + - remote_python_packaging: + requires: + - remote_tests_python + - remote_tests_rust + + - upload_docs_wheels: + requires: + - remote_python_packaging + - build_doxygen # - rustfmt: # requires: # - cargo_fetch diff --git a/ci_scripts/ci_upload.sh b/ci_scripts/ci_upload.sh index 947073bbc..c609bfc0b 100755 --- a/ci_scripts/ci_upload.sh +++ b/ci_scripts/ci_upload.sh @@ -28,10 +28,6 @@ rsync -avz \ "$DOXYDOCDIR/html/" \ delta@c.delta.chat:build-c/${BRANCH} -exit 0 - -# OUTDATED -- for re-use from python release-scripts - echo ----------------------- echo upload wheels echo ----------------------- @@ -39,6 +35,7 @@ echo ----------------------- # Bundle external shared libraries into the wheels pushd $WHEELHOUSEDIR +pip3 install -U pip pip3 install devpi-client devpi use https://m.devpi.net devpi login dc --password $DEVPI_LOGIN @@ -50,6 +47,9 @@ devpi use dc/$N_BRANCH || { devpi use dc/$N_BRANCH } devpi index $N_BRANCH bases=/root/pypi -devpi upload deltachat*.whl +devpi upload deltachat* popd + +# remove devpi non-master dc indices if thy are too old +python ci_scripts/cleanup_devpi_indices.py diff --git a/ci_scripts/cleanup_devpi_indices.py b/ci_scripts/cleanup_devpi_indices.py new file mode 100644 index 000000000..628c47d84 --- /dev/null +++ b/ci_scripts/cleanup_devpi_indices.py @@ -0,0 +1,69 @@ +""" +Remove old "dc" indices except for master which always stays. + +""" +from requests import Session +import datetime +import sys +import subprocess + +MAXDAYS=7 + +session = Session() +session.headers["Accept"] = "application/json" + + +def get_indexes(baseurl, username): + response = session.get(baseurl + username) + assert response.status_code == 200 + result = response.json()["result"] + return result["indexes"] + + +def get_projectnames(baseurl, username, indexname): + response = session.get(baseurl + username + "/" + indexname) + assert response.status_code == 200 + result = response.json()["result"] + return result["projects"] + + +def get_release_dates(baseurl, username, indexname, projectname): + response = session.get(baseurl + username + "/" + indexname + "/" + projectname) + assert response.status_code == 200 + result = response.json()["result"] + dates = set() + for value in result.values(): + if "+links" not in value: + continue + for link in value["+links"]: + for log in link["log"]: + dates.add(tuple(log["when"])) + return dates + + +def run(): + baseurl = "https://m.devpi.net/" + username = "dc" + for indexname in get_indexes(baseurl, username): + projectnames = get_projectnames(baseurl, username, indexname) + if indexname == "master" or not indexname: + continue + assert projectnames == ["deltachat"] + for projectname in projectnames: + dates = get_release_dates(baseurl, username, indexname, projectname) + if not dates: + print( + "%s has no releases" % (baseurl + username + "/" + indexname), + file=sys.stderr) + date = datetime.datetime.now() + else: + date = datetime.datetime(*max(dates)) + if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS): + assert username and indexname + url = baseurl + username + "/" + indexname + subprocess.check_call(["devpi", "index", "-y", "--delete", url]) + + + +if __name__ == '__main__': + run() diff --git a/ci_scripts/docker-coredeps/Dockerfile b/ci_scripts/docker-coredeps/Dockerfile index 9e163636a..50f53d1d5 100644 --- a/ci_scripts/docker-coredeps/Dockerfile +++ b/ci_scripts/docker-coredeps/Dockerfile @@ -5,14 +5,11 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \ echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig -# Install a recent Perl, needed to install OpenSSL +# Install a recent Perl, needed to install the openssl crate ADD deps/build_perl.sh /builder/build_perl.sh +RUN rm /usr/bin/perl RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1 -# Install OpenSSL -ADD deps/build_openssl.sh /builder/build_openssl.sh -RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1 - ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # Install python tools (auditwheels,tox, ...) diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh index 95b08ddd5..3e0a232c4 100755 --- a/ci_scripts/docker-coredeps/deps/build_rust.sh +++ b/ci_scripts/docker-coredeps/deps/build_rust.sh @@ -3,6 +3,9 @@ set -e -x # Install Rust -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y export PATH=/root/.cargo/bin:$PATH rustc --version + +# remove some 300-400 MB that we don't need for automated builds +rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/ diff --git a/ci_scripts/docker-coredeps/deps/run_all.sh b/ci_scripts/docker-coredeps/deps/run_all.sh deleted file mode 100755 index 93fca67b5..000000000 --- a/ci_scripts/docker-coredeps/deps/run_all.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Build the Delta Chat C/Rust library -# -set -e -x - -# perform clean build of core and install -export TOXWORKDIR=.docker-tox - -# build core library - -cargo build --release -p deltachat_ffi - -# configure access to a base python and -# to several python interpreters needed by tox below -export PATH=$PATH:/opt/python/cp35-cp35m/bin -export PYTHONDONTWRITEBYTECODE=1 -pushd /bin -ln -s /opt/python/cp27-cp27m/bin/python2.7 -ln -s /opt/python/cp36-cp36m/bin/python3.6 -ln -s /opt/python/cp37-cp37m/bin/python3.7 -popd - -# -# run python tests -# - -if [ -n "$TESTS" ]; then - - echo ---------------- - echo run python tests - echo ---------------- - - pushd python - # first run all tests ... - rm -rf tests/__pycache__ - rm -rf src/deltachat/__pycache__ - export PYTHONDONTWRITEBYTECODE=1 - tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37 - popd -fi - - -if [ -n "$DOCS" ]; then - echo ----------------------- - echo generating python docs - echo ----------------------- - (cd python && tox --workdir "$TOXWORKDIR" -e doc) -fi diff --git a/ci_scripts/remote_python_packaging.sh b/ci_scripts/remote_python_packaging.sh new file mode 100755 index 000000000..414632ca5 --- /dev/null +++ b/ci_scripts/remote_python_packaging.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +export BRANCH=${CIRCLE_BRANCH:?branch to build} +export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name} +export SSHTARGET=${SSHTARGET-ci@b1.delta.chat} + +# we construct the BUILDDIR such that we can easily share the +# CARGO_TARGET_DIR between runs ("..") +export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number} + +echo "--- Copying files to $SSHTARGET:$BUILDDIR" + +set -xe + +ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" +git ls-files >.rsynclist +# we seem to need .git for setuptools_scm versioning +find .git >>.rsynclist +rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" + +set +x + +# we have to create a remote file for the remote-docker run +# so we can do a simple ssh command with a TTY +# so that when our job dies, all container-runs are aborted. +# sidenote: the circle-ci machinery will kill ongoing jobs +# if there are new commits and we want to ensure that +# everything is terminated/cleaned up and we have no orphaned +# useless still-running docker-containers consuming resources. + +ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE + set +x -e + cd $BUILDDIR + export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG + + set -x + + # run everything else inside docker + docker run -e DCC_PY_LIVECONFIG \ + --rm -it -v \$(pwd):/mnt -w /mnt \ + deltachat/coredeps ci_scripts/run_all.sh + +_HERE + +echo "--- Running $CIRCLE_JOB remotely" + +ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run" +mkdir -p workspace +rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse" workspace/ +rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/ +rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs diff --git a/ci_scripts/run_all.sh b/ci_scripts/run_all.sh new file mode 100755 index 000000000..0d32d4645 --- /dev/null +++ b/ci_scripts/run_all.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Build the Delta Chat Core Rust library, Python wheels and docs + +set -e -x + +# Perform clean build of core and install. +export TOXWORKDIR=.docker-tox + +# compile core lib + +export PATH=/root/.cargo/bin:$PATH +cargo build --release -p deltachat_ffi +# cargo test --all --all-features + +# Statically link against libdeltachat.a. +export DCC_RS_DEV=$(pwd) + +# Configure access to a base python and to several python interpreters +# needed by tox below. +export PATH=$PATH:/opt/python/cp35-cp35m/bin +export PYTHONDONTWRITEBYTECODE=1 +pushd /bin +ln -s /opt/python/cp27-cp27m/bin/python2.7 +ln -s /opt/python/cp36-cp36m/bin/python3.6 +ln -s /opt/python/cp37-cp37m/bin/python3.7 +ln -s /opt/python/cp38-cp38/bin/python3.8 +popd + +pushd python +# prepare a clean tox run +rm -rf tests/__pycache__ +rm -rf src/deltachat/__pycache__ +mkdir -p $TOXWORKDIR + +# disable live-account testing to speed up test runs and wheel building +# XXX we may switch on some live-tests on for better ensurances +# Note that the independent remote_tests_python step does all kinds of +# live-testing already. +unset DCC_PY_LIVECONFIG +tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels +popd + + +echo ----------------------- +echo generating python docs +echo ----------------------- +(cd python && tox --workdir "$TOXWORKDIR" -e doc) diff --git a/python/README.rst b/python/README.rst index c58457775..21c4d1ea6 100644 --- a/python/README.rst +++ b/python/README.rst @@ -75,42 +75,31 @@ With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real e-mail accounts and run through all functional "liveconfig" tests. -============================================================================================================================ -(21-Dec-2019) THE BELOW WHEELS ARE CURRENTLY NOT WORKING/BROKEN, COMPILE FROM SOURCE USING ABOVE INSTRUCTIONS INSTEAD -============================================================================================================================ - -Installing pre-built packages (linux-only) (OUTDATED) +Installing pre-built packages (Linux-only) ======================================================== -If you have a linux system you may install the ``deltachat`` binary "wheel" package +If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages without any "build-from-source" steps. -1. `Install virtualenv `_, +First of all we suggest to `Install virtualenv `_, then create a fresh python environment and activate it in your shell:: virtualenv venv # or: python -m venv source venv/bin/activate - Afterwards, invoking ``python`` or ``pip install`` will only - modify files in your ``venv`` directory and leave your system installation - alone. - -2. Install the wheel for linux:: - - pip install deltachat - - Verify it worked by typing:: - - python -c "import deltachat" - - -Installing a wheel from a PR/branch (OUTDATED) -------------------------------------------------- +Afterwards, invoking ``python`` or ``pip install`` will only +modify files in your ``venv`` directory and leave your system installation +alone. For Linux, we automatically build wheels for all github PR branches -and push them to a python package index. To install the latest github ``master`` branch:: +and push them to a python package index. To install the latest +github ``master`` branch:: - pip install -i https://m.devpi.net/dc/master deltachat + pip install --pre -i https://m.devpi.net/dc/master deltachat + +To verify it worked:: + + python -c "import deltachat" .. note:: @@ -119,10 +108,6 @@ and push them to a python package index. To install the latest github ``master`` `in contact with us `_. - - - - Code examples ============= @@ -133,15 +118,11 @@ You may look at `examples `_. .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust -Building manylinux1 wheels -========================== - -.. note:: - - This section may not fully work. +Building manylinux1 based wheels +================================ Building portable manylinux1 wheels which come with libdeltachat.so -and all it's dependencies is easy using the provided docker tooling. +can be done with docker-tooling. using docker pull / premade images ------------------------------------ @@ -154,9 +135,9 @@ organization:: This docker image can be used to run tests and build Python wheels for all interpreters:: - $ bash ci_scripts/ci_run.sh - -This command runs tests and build-wheel scripts in a docker container. + $ docker run -e DCC_PY_LIVECONFIG \ + --rm -it -v \$(pwd):/mnt -w /mnt \ + deltachat/coredeps ci_scripts/run_all.sh Optionally build your own docker image diff --git a/python/tox.ini b/python/tox.ini index fb0b65eb0..5d3ad4091 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -8,7 +8,7 @@ envlist = [testenv] commands = pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests} - # python tests/package_wheels.py {toxworkdir}/wheelhouse + python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS DCC_RS_DEV