diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 47b55539c..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2.1 -executors: - default: - docker: - - image: filecoin/rust:latest - working_directory: /mnt/crate - doxygen: - docker: - - image: hrektts/doxygen - -jobs: - build_doxygen: - executor: doxygen - steps: - - checkout - - run: bash scripts/run-doxygen.sh - - run: mkdir -p workspace/c-docs - - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/ - - persist_to_workspace: - root: workspace - paths: - - c-docs - - remote_python_packaging: - machine: true - steps: - - checkout - # the following commands on success produces - # workspace/{wheelhouse,py-docs} as artefact directories - - run: - # building aarch64 packages under qemu is very slow - no_output_timeout: 60m - command: bash scripts/remote_python_packaging.sh - - persist_to_workspace: - root: workspace - paths: - # - c-docs - - py-docs - - wheelhouse - - upload_docs_wheels: - machine: true - steps: - - checkout - - attach_workspace: - at: workspace - - run: ls -laR workspace - - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs - -workflows: - version: 2.1 - - test: - jobs: - - remote_python_packaging: - filters: - tags: - only: /.+/ - branches: - ignore: /.*/ - - - build_doxygen: - filters: - tags: - only: /.+/ - branches: - ignore: /.*/ - - - upload_docs_wheels: - requires: - - remote_python_packaging - - build_doxygen - filters: - tags: - only: /.+/ - branches: - ignore: /.*/ diff --git a/scripts/ci_upload.sh b/scripts/ci_upload.sh deleted file mode 100755 index f478bdeef..000000000 --- a/scripts/ci_upload.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -if [ -z "$DEVPI_LOGIN" ] ; then - echo "required: password for 'dc' user on https://m.devpi/net/dc index" - exit 0 -fi - -set -xe - -PYDOCDIR=${1:?directory with python docs} -WHEELHOUSEDIR=${2:?directory with pre-built wheels} -DOXYDOCDIR=${3:?directory where doxygen docs to be found} -SSHTARGET=ci@b1.delta.chat - - -# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. -export BRANCH=${CIRCLE_BRANCH:master} - -export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse - - -# python docs to py.delta.chat -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH} -rsync -avz \ - --delete \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - "$PYDOCDIR/html/" \ - delta@py.delta.chat:build/${BRANCH} - -# C docs to c.delta.chat -ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH} -rsync -avz \ - --delete \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - "$DOXYDOCDIR/html/" \ - delta@c.delta.chat:build-c/${BRANCH} - -echo ----------------------- -echo upload wheels -echo ----------------------- - -# Bundle external shared libraries into the wheels - -ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR -scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR -rsync -avz \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - $WHEELHOUSEDIR \ - $SSHTARGET:$BUILDDIR - -ssh $SSHTARGET <<_HERE - set +x -e - # make sure all processes exit when ssh dies - shopt -s huponexit - - # we rely on the "venv" virtualenv on the remote account to exist - source venv/bin/activate - cd $BUILDDIR - - devpi use https://m.devpi.net - devpi login dc --password $DEVPI_LOGIN - - N_BRANCH=${BRANCH//[\/]} - - devpi use dc/\$N_BRANCH || { - devpi index -c \$N_BRANCH - devpi use dc/\$N_BRANCH - } - devpi index \$N_BRANCH bases=/root/pypi - devpi upload wheelhouse/deltachat* - - # remove devpi non-master dc indices if thy are too old - # this script was copied above - python cleanup_devpi_indices.py -_HERE diff --git a/scripts/concourse/README.md b/scripts/concourse/README.md new file mode 100644 index 000000000..b022d9bc4 --- /dev/null +++ b/scripts/concourse/README.md @@ -0,0 +1,21 @@ +# Concourse CI pipeline + +`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/) +that builds C documentation, Python documentation, Python wheels for `x86_64` +and `aarch64` and Python source packages, and uploads them. + +To setup the pipeline run +``` +fly -t set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml +``` +where `secret.yml` contains the following secrets: +``` +c.delta.chat: + private_key: | + -----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY----- +devpi: + login: dc + password: ... +``` diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml new file mode 100644 index 000000000..0a1d53730 --- /dev/null +++ b/scripts/concourse/docs_wheels.yml @@ -0,0 +1,217 @@ +resources: +- name: deltachat-core-rust + type: git + icon: github + source: + branch: master + uri: https://github.com/deltachat/deltachat-core-rust.git + +jobs: +- name: doxygen + plan: + - get: deltachat-core-rust + + # Build Doxygen documentation + - task: build-doxygen + config: + inputs: + - name: deltachat-core-rust + outputs: + - name: c-docs + image_resource: + source: + repository: hrektts/doxygen + type: registry-image + platform: linux + run: + path: bash + args: + - -exc + - | + cd deltachat-core-rust + bash scripts/run-doxygen.sh + cd .. + cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/ + + - task: upload-c-docs + config: + inputs: + - name: c-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master + +- name: python-x86_64 + plan: + - get: deltachat-core-rust + + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + - name: deltachat-core-rust + image_resource: + source: + repository: vito/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/docker-coredeps + UNPACK_ROOTFS: "true" + platform: linux + caches: + - path: cache + run: + path: build + + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust + path: . + outputs: + - name: py-docs + path: ./python/doc/_build/ + # Source packages + - name: py-dist + path: ./python/.docker-tox/dist/ + # Binary wheels + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh + + # Upload python docs to py.delta.chat + - task: upload-py-docs + config: + inputs: + - name: py-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master + + # Upload x86_64 wheels and source packages + - task: upload-wheels + config: + inputs: + - name: py-wheels + - name: py-dist + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* + devpi upload py-dist/* + +- name: python-aarch64 + plan: + - get: deltachat-core-rust + + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + - name: deltachat-core-rust + image_resource: + source: + repository: vito/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64 + UNPACK_ROOTFS: "true" + platform: linux + caches: + - path: cache + run: + path: build + + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh + + # Upload aarch64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* diff --git a/scripts/remote_python_packaging.sh b/scripts/remote_python_packaging.sh deleted file mode 100755 index bbd86941f..000000000 --- a/scripts/remote_python_packaging.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -export BRANCH=${CIRCLE_BRANCH:-master} -export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust} -export SSHTARGET=${SSHTARGET-ci@b1.delta.chat} - -export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number} - -echo "--- Copying files to $SSHTARGET:$BUILDDIR" - -set -xe - -ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR" -git ls-files >.rsynclist -# we seem to need .git for setuptools_scm versioning -find .git >>.rsynclist -rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR" -rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64" - -set +x - -# we have to create a remote file for the remote-docker run -# so we can do a simple ssh command with a TTY -# so that when our job dies, all container-runs are aborted. -# sidenote: the circle-ci machinery will kill ongoing jobs -# if there are new commits and we want to ensure that -# everything is terminated/cleaned up and we have no orphaned -# useless still-running docker-containers consuming resources. - -for arch in "" "-arm64"; do - -ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE - set +x -e - shopt -s huponexit - cd ${BUILDDIR}${arch} - export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL - set -x - - # run everything else inside docker - docker run -e DCC_NEW_TMP_EMAIL \ - --rm -it -v \$(pwd):/mnt -w /mnt \ - deltachat/coredeps${arch} scripts/run_all.sh - -_HERE - -done - -echo "--- Running $CIRCLE_JOB remotely" - -echo "--- Building aarch64 wheels" -ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run" - -echo "--- Building x86_64 wheels" -ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run" - -mkdir -p workspace - -# Wheels -for arch in "" "-arm64"; do -rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/ -done - -# Source packages -rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/ - -# Documentation -rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs