Concourse CI pipeline to replace Circle CI

.circleci/config.yml

@@ -1,77 +0,0 @@
-version: 2.1
-executors:
-  default:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-  doxygen:
-    docker:
-      - image: hrektts/doxygen
-            
-jobs:
-  build_doxygen:
-    executor: doxygen
-    steps:
-      - checkout
-      - run: bash scripts/run-doxygen.sh
-      - run: mkdir -p workspace/c-docs
-      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - c-docs
-
-  remote_python_packaging:
-    machine: true
-    steps:
-      - checkout
-      # the following commands on success produces
-      # workspace/{wheelhouse,py-docs} as artefact directories
-      - run:
-          # building aarch64 packages under qemu is very slow
-          no_output_timeout: 60m
-          command: bash scripts/remote_python_packaging.sh
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
-
-  upload_docs_wheels:
-    machine: true
-    steps:
-      - checkout
-      - attach_workspace:
-          at: workspace
-      - run: ls -laR workspace
-      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
-
-workflows:
-  version: 2.1
-
-  test:
-    jobs:
-      - remote_python_packaging:
-          filters:
-            tags:
-              only: /.+/
-            branches:
-              ignore: /.*/
-
-      - build_doxygen:
-          filters:
-            tags:
-              only: /.+/
-            branches:
-              ignore: /.*/
-
-      - upload_docs_wheels:
-          requires:
-             - remote_python_packaging
-             - build_doxygen
-          filters:
-            tags:
-              only: /.+/
-            branches:
-              ignore: /.*/

scripts/ci_upload.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-if [ -z "$DEVPI_LOGIN" ] ; then 
-    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
-    exit 0
-fi
-
-set -xe
-
-PYDOCDIR=${1:?directory with python docs}
-WHEELHOUSEDIR=${2:?directory with pre-built wheels}
-DOXYDOCDIR=${3:?directory where doxygen docs to be found}
-SSHTARGET=ci@b1.delta.chat
-
-
-# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
-export BRANCH=${CIRCLE_BRANCH:master}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
-
-
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
-
-# C docs to c.delta.chat
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$DOXYDOCDIR/html/" \
-  delta@c.delta.chat:build-c/${BRANCH}
-
-echo -----------------------
-echo upload wheels 
-echo -----------------------
-
-# Bundle external shared libraries into the wheels
-
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  $WHEELHOUSEDIR \
-  $SSHTARGET:$BUILDDIR 
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    # we rely on the "venv" virtualenv on the remote account to exist 
-    source venv/bin/activate
-    cd $BUILDDIR
-
-    devpi use https://m.devpi.net
-    devpi login dc --password $DEVPI_LOGIN
-
-    N_BRANCH=${BRANCH//[\/]}
-
-    devpi use dc/\$N_BRANCH || {
-        devpi index -c \$N_BRANCH 
-        devpi use dc/\$N_BRANCH
-    }
-    devpi index \$N_BRANCH bases=/root/pypi
-    devpi upload wheelhouse/deltachat*
-
-    # remove devpi non-master dc indices if thy are too old
-    # this script was copied above 
-    python cleanup_devpi_indices.py
-_HERE

scripts/concourse/README.md

@@ -0,0 +1,21 @@
+# Concourse CI pipeline
+
+`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
+that builds C documentation, Python documentation, Python wheels for `x86_64`
+and `aarch64` and Python source packages, and uploads them.
+
+To setup the pipeline run
+```
+fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
+```
+where `secret.yml` contains the following secrets:
+```
+c.delta.chat:
+  private_key: |
+    -----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----
+devpi:
+  login: dc
+  password: ...
+```

scripts/concourse/docs_wheels.yml

@@ -0,0 +1,217 @@
+resources:
+- name: deltachat-core-rust
+  type: git
+  icon: github
+  source:
+    branch: master
+    uri: https://github.com/deltachat/deltachat-core-rust.git
+
+jobs:
+- name: doxygen
+  plan:
+  - get: deltachat-core-rust
+
+  # Build Doxygen documentation
+  - task: build-doxygen
+    config:
+      inputs:
+        - name: deltachat-core-rust
+      outputs:
+        - name: c-docs
+      image_resource:
+        source:
+          repository: hrektts/doxygen
+        type: registry-image
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          cd deltachat-core-rust
+          bash scripts/run-doxygen.sh
+          cd ..
+          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
+
+  - task: upload-c-docs
+    config:
+      inputs:
+        - name: c-docs
+      image_resource:
+        type: registry-image
+        source:
+          repository: alpine
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apk add --no-cache rsync openssh-client
+          mkdir -p ~/.ssh
+          chmod 700 ~/.ssh
+          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
+
+- name: python-x86_64
+  plan:
+  - get: deltachat-core-rust
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: vito/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
+        UNPACK_ROOTFS: "true"
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust
+          path: .
+      outputs:
+        - name: py-docs
+          path: ./python/doc/_build/
+        # Source packages
+        - name: py-dist
+          path: ./python/.docker-tox/dist/
+        # Binary wheels
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload python docs to py.delta.chat
+  - task: upload-py-docs
+    config:
+      inputs:
+        - name: py-docs
+      image_resource:
+        type: registry-image
+        source:
+          repository: alpine
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apk add --no-cache rsync openssh-client
+          mkdir -p ~/.ssh
+          chmod 700 ~/.ssh
+          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
+
+  # Upload x86_64 wheels and source packages
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+        - name: py-dist
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*manylinux201*
+          devpi upload py-dist/*
+
+- name: python-aarch64
+  plan:
+  - get: deltachat-core-rust
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: vito/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
+        UNPACK_ROOTFS: "true"
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload aarch64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*manylinux201*

scripts/remote_python_packaging.sh

@@ -1,67 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-set -xe
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-# we seem to need .git for setuptools_scm versioning 
-find .git >>.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
-
-set +x
-
-# we have to create a remote file for the remote-docker run 
-# so we can do a simple ssh command with a TTY
-# so that when our job dies, all container-runs are aborted.
-# sidenote: the circle-ci machinery will kill ongoing jobs
-# if there are new commits and we want to ensure that 
-# everything is terminated/cleaned up and we have no orphaned
-# useless still-running docker-containers consuming resources. 
-
-for arch in "" "-arm64"; do
-
-ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
-    set +x -e
-    shopt -s huponexit
-    cd ${BUILDDIR}${arch}
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
-    set -x
-
-    # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps${arch} scripts/run_all.sh
-
-_HERE
-
-done
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-echo "--- Building aarch64 wheels"
-ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
-
-echo "--- Building x86_64 wheels"
-ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
-
-mkdir -p workspace 
-
-# Wheels
-for arch in "" "-arm64"; do
-rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
-done
-
-# Source packages
-rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
-
-# Documentation
-rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs