bringing back wheel building for linux, and fixing c.delta.chat

ci_scripts/ci_upload.sh

@@ -28,10 +28,6 @@ rsync -avz \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
-exit 0
-
-# OUTDATED -- for re-use from python release-scripts 
-
 echo -----------------------
 echo upload wheels 
 echo -----------------------
@@ -39,6 +35,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -50,6 +47,9 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*.whl
+devpi upload deltachat*
 
 popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -0,0 +1,69 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+    response = session.get(baseurl + username)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+    response = session.get(baseurl + username + "/" + indexname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    dates = set()
+    for value in result.values():
+        if "+links" not in value:
+            continue
+        for link in value["+links"]:
+            for log in link["log"]:
+                dates.add(tuple(log["when"]))
+    return dates
+
+
+def run():
+    baseurl = "https://m.devpi.net/"
+    username = "dc"
+    for indexname in get_indexes(baseurl, username):
+        projectnames = get_projectnames(baseurl, username, indexname)
+        if indexname == "master" or not indexname:
+            continue
+        assert projectnames == ["deltachat"]
+        for projectname in projectnames:
+            dates = get_release_dates(baseurl, username, indexname, projectname)
+            if not dates:
+                print(
+                    "%s has no releases" % (baseurl + username + "/" + indexname),
+                    file=sys.stderr)
+                date = datetime.datetime.now()
+            else:
+                date = datetime.datetime(*max(dates))
+            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+                assert username and indexname
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -5,14 +5,11 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install OpenSSL
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 
-# Install OpenSSL
-ADD deps/build_openssl.sh /builder/build_openssl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
-
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,6 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library
-#
-set -e -x
-
-# perform clean build of core and install 
-export TOXWORKDIR=.docker-tox
-
-# build core library
-
-cargo build --release -p deltachat_ffi
-
-# configure access to a base python and 
-# to several python interpreters needed by tox below
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-#
-# run python tests
-#
-
-if [ -n "$TESTS" ]; then 
-
-    echo ----------------
-    echo run python tests
-    echo ----------------
-
-    pushd python 
-    # first run all tests ...
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
-    popd
-fi
-
-
-if [ -n "$DOCS" ]; then 
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
-fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse" workspace/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/run_all.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Build the Delta Chat Core Rust library, Python wheels and docs 
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# compile core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
+popd
+
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
+
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances 
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already. 
+unset DCC_PY_LIVECONFIG 
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)