diff --git a/.circleci/config.yml b/.circleci/config.yml
index bd44ab540..9e33b9987 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -120,32 +120,19 @@ jobs:
paths:
- c-docs
- # build_test_docs_wheel:
- # docker:
- # - image: deltachat/coredeps
- # environment:
- # TESTS: 1
- # DOCS: 1
- # working_directory: /mnt/crate
- # steps:
- # - *restore-workspace
- # - *restore-cache
- # - run:
- # name: build docs, run tests and build wheels
- # command: ci_scripts/run-python.sh
- # - run:
- # name: copying docs and wheels to workspace
- # command: |
- # mkdir -p workspace/python
- # # cp -av docs workspace/c-docs
- # cp -av python/.docker-tox/wheelhouse workspace/
- # cp -av python/doc/_build/ workspace/py-docs
- # - persist_to_workspace:
- # root: workspace
- # paths:
- # # - c-docs
- # - py-docs
- # - wheelhouse
+ remote_python_packaging:
+ machine: true
+ steps:
+ - checkout
+ # the following commands on success produces
+ # workspace/{wheelhouse,py-docs} as artefact directories
+ - run: bash ci_scripts/remote_python_packaging.sh
+ - persist_to_workspace:
+ root: workspace
+ paths:
+ # - c-docs
+ - py-docs
+ - wheelhouse
remote_tests_rust:
machine: true
@@ -157,10 +144,7 @@ jobs:
machine: true
steps:
- checkout
- #- attach_workspace:
- # at: workspace
- run: ci_scripts/remote_tests_python.sh
- # workspace/py-docs workspace/wheelhouse workspace/c-docs
upload_docs_wheels:
machine: true
@@ -168,6 +152,7 @@ jobs:
- checkout
- attach_workspace:
at: workspace
+ - run: pyenv versions
- run: pyenv global 3.5.2
- run: ls -laR workspace
- run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
@@ -193,10 +178,15 @@ workflows:
- remote_tests_python
- # - upload_docs_wheels:
- # requires:
- # - build_test_docs_wheel
- # - build_doxygen
+ - remote_python_packaging:
+ requires:
+ - remote_tests_python
+ - remote_tests_rust
+
+ - upload_docs_wheels:
+ requires:
+ - remote_python_packaging
+ - build_doxygen
# - rustfmt:
# requires:
# - cargo_fetch
diff --git a/ci_scripts/ci_upload.sh b/ci_scripts/ci_upload.sh
index 947073bbc..c609bfc0b 100755
--- a/ci_scripts/ci_upload.sh
+++ b/ci_scripts/ci_upload.sh
@@ -28,10 +28,6 @@ rsync -avz \
"$DOXYDOCDIR/html/" \
delta@c.delta.chat:build-c/${BRANCH}
-exit 0
-
-# OUTDATED -- for re-use from python release-scripts
-
echo -----------------------
echo upload wheels
echo -----------------------
@@ -39,6 +35,7 @@ echo -----------------------
# Bundle external shared libraries into the wheels
pushd $WHEELHOUSEDIR
+pip3 install -U pip
pip3 install devpi-client
devpi use https://m.devpi.net
devpi login dc --password $DEVPI_LOGIN
@@ -50,6 +47,9 @@ devpi use dc/$N_BRANCH || {
devpi use dc/$N_BRANCH
}
devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*.whl
+devpi upload deltachat*
popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py
diff --git a/ci_scripts/cleanup_devpi_indices.py b/ci_scripts/cleanup_devpi_indices.py
new file mode 100644
index 000000000..628c47d84
--- /dev/null
+++ b/ci_scripts/cleanup_devpi_indices.py
@@ -0,0 +1,69 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+ response = session.get(baseurl + username)
+ assert response.status_code == 200
+ result = response.json()["result"]
+ return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+ response = session.get(baseurl + username + "/" + indexname)
+ assert response.status_code == 200
+ result = response.json()["result"]
+ return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+ response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+ assert response.status_code == 200
+ result = response.json()["result"]
+ dates = set()
+ for value in result.values():
+ if "+links" not in value:
+ continue
+ for link in value["+links"]:
+ for log in link["log"]:
+ dates.add(tuple(log["when"]))
+ return dates
+
+
+def run():
+ baseurl = "https://m.devpi.net/"
+ username = "dc"
+ for indexname in get_indexes(baseurl, username):
+ projectnames = get_projectnames(baseurl, username, indexname)
+ if indexname == "master" or not indexname:
+ continue
+ assert projectnames == ["deltachat"]
+ for projectname in projectnames:
+ dates = get_release_dates(baseurl, username, indexname, projectname)
+ if not dates:
+ print(
+ "%s has no releases" % (baseurl + username + "/" + indexname),
+ file=sys.stderr)
+ date = datetime.datetime.now()
+ else:
+ date = datetime.datetime(*max(dates))
+ if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+ assert username and indexname
+ url = baseurl + username + "/" + indexname
+ subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+ run()
diff --git a/ci_scripts/docker-coredeps/Dockerfile b/ci_scripts/docker-coredeps/Dockerfile
index 9e163636a..50f53d1d5 100644
--- a/ci_scripts/docker-coredeps/Dockerfile
+++ b/ci_scripts/docker-coredeps/Dockerfile
@@ -5,14 +5,11 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-# Install a recent Perl, needed to install OpenSSL
+# Install a recent Perl, needed to install the openssl crate
ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-# Install OpenSSL
-ADD deps/build_openssl.sh /builder/build_openssl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
-
ENV PIP_DISABLE_PIP_VERSION_CHECK 1
# Install python tools (auditwheels,tox, ...)
diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh
index 95b08ddd5..3e0a232c4 100755
--- a/ci_scripts/docker-coredeps/deps/build_rust.sh
+++ b/ci_scripts/docker-coredeps/deps/build_rust.sh
@@ -3,6 +3,9 @@
set -e -x
# Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y
export PATH=/root/.cargo/bin:$PATH
rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/
diff --git a/ci_scripts/docker-coredeps/deps/run_all.sh b/ci_scripts/docker-coredeps/deps/run_all.sh
deleted file mode 100755
index 93fca67b5..000000000
--- a/ci_scripts/docker-coredeps/deps/run_all.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library
-#
-set -e -x
-
-# perform clean build of core and install
-export TOXWORKDIR=.docker-tox
-
-# build core library
-
-cargo build --release -p deltachat_ffi
-
-# configure access to a base python and
-# to several python interpreters needed by tox below
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-#
-# run python tests
-#
-
-if [ -n "$TESTS" ]; then
-
- echo ----------------
- echo run python tests
- echo ----------------
-
- pushd python
- # first run all tests ...
- rm -rf tests/__pycache__
- rm -rf src/deltachat/__pycache__
- export PYTHONDONTWRITEBYTECODE=1
- tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
- popd
-fi
-
-
-if [ -n "$DOCS" ]; then
- echo -----------------------
- echo generating python docs
- echo -----------------------
- (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-fi
diff --git a/ci_scripts/remote_python_packaging.sh b/ci_scripts/remote_python_packaging.sh
new file mode 100755
index 000000000..414632ca5
--- /dev/null
+++ b/ci_scripts/remote_python_packaging.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist
+# we seem to need .git for setuptools_scm versioning
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources.
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+ set +x -e
+ cd $BUILDDIR
+ export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+ set -x
+
+ # run everything else inside docker
+ docker run -e DCC_PY_LIVECONFIG \
+ --rm -it -v \$(pwd):/mnt -w /mnt \
+ deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse" workspace/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs
diff --git a/ci_scripts/run_all.sh b/ci_scripts/run_all.sh
new file mode 100755
index 000000000..0d32d4645
--- /dev/null
+++ b/ci_scripts/run_all.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Build the Delta Chat Core Rust library, Python wheels and docs
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# compile core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
+popd
+
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
+
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already.
+unset DCC_PY_LIVECONFIG
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)
diff --git a/python/README.rst b/python/README.rst
index c58457775..21c4d1ea6 100644
--- a/python/README.rst
+++ b/python/README.rst
@@ -75,42 +75,31 @@ With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
e-mail accounts and run through all functional "liveconfig" tests.
-============================================================================================================================
-(21-Dec-2019) THE BELOW WHEELS ARE CURRENTLY NOT WORKING/BROKEN, COMPILE FROM SOURCE USING ABOVE INSTRUCTIONS INSTEAD
-============================================================================================================================
-
-Installing pre-built packages (linux-only) (OUTDATED)
+Installing pre-built packages (Linux-only)
========================================================
-If you have a linux system you may install the ``deltachat`` binary "wheel" package
+If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
without any "build-from-source" steps.
-1. `Install virtualenv `_,
+First of all we suggest to `Install virtualenv `_,
then create a fresh python environment and activate it in your shell::
virtualenv venv # or: python -m venv
source venv/bin/activate
- Afterwards, invoking ``python`` or ``pip install`` will only
- modify files in your ``venv`` directory and leave your system installation
- alone.
-
-2. Install the wheel for linux::
-
- pip install deltachat
-
- Verify it worked by typing::
-
- python -c "import deltachat"
-
-
-Installing a wheel from a PR/branch (OUTDATED)
--------------------------------------------------
+Afterwards, invoking ``python`` or ``pip install`` will only
+modify files in your ``venv`` directory and leave your system installation
+alone.
For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest github ``master`` branch::
+and push them to a python package index. To install the latest
+github ``master`` branch::
- pip install -i https://m.devpi.net/dc/master deltachat
+ pip install --pre -i https://m.devpi.net/dc/master deltachat
+
+To verify it worked::
+
+ python -c "import deltachat"
.. note::
@@ -119,10 +108,6 @@ and push them to a python package index. To install the latest github ``master``
`in contact with us `_.
-
-
-
-
Code examples
=============
@@ -133,15 +118,11 @@ You may look at `examples `_.
.. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
-Building manylinux1 wheels
-==========================
-
-.. note::
-
- This section may not fully work.
+Building manylinux1 based wheels
+================================
Building portable manylinux1 wheels which come with libdeltachat.so
-and all it's dependencies is easy using the provided docker tooling.
+can be done with docker-tooling.
using docker pull / premade images
------------------------------------
@@ -154,9 +135,9 @@ organization::
This docker image can be used to run tests and build Python wheels for all interpreters::
- $ bash ci_scripts/ci_run.sh
-
-This command runs tests and build-wheel scripts in a docker container.
+ $ docker run -e DCC_PY_LIVECONFIG \
+ --rm -it -v \$(pwd):/mnt -w /mnt \
+ deltachat/coredeps ci_scripts/run_all.sh
Optionally build your own docker image
diff --git a/python/tox.ini b/python/tox.ini
index fb0b65eb0..5d3ad4091 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -8,7 +8,7 @@ envlist =
[testenv]
commands =
pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests}
- # python tests/package_wheels.py {toxworkdir}/wheelhouse
+ python tests/package_wheels.py {toxworkdir}/wheelhouse
passenv =
TRAVIS
DCC_RS_DEV