mirror of
https://github.com/chatmail/core.git
synced 2026-04-17 13:36:30 +03:00
Concourse CI pipeline to replace Circle CI
This commit is contained in:
link2xt
@@ -1,77 +0,0 @@
|
||||
version: 2.1
|
||||
executors:
|
||||
default:
|
||||
docker:
|
||||
- image: filecoin/rust:latest
|
||||
working_directory: /mnt/crate
|
||||
doxygen:
|
||||
docker:
|
||||
- image: hrektts/doxygen
|
||||
|
||||
jobs:
|
||||
build_doxygen:
|
||||
executor: doxygen
|
||||
steps:
|
||||
- checkout
|
||||
- run: bash scripts/run-doxygen.sh
|
||||
- run: mkdir -p workspace/c-docs
|
||||
- run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
|
||||
- persist_to_workspace:
|
||||
root: workspace
|
||||
paths:
|
||||
- c-docs
|
||||
|
||||
remote_python_packaging:
|
||||
machine: true
|
||||
steps:
|
||||
- checkout
|
||||
# the following commands on success produces
|
||||
# workspace/{wheelhouse,py-docs} as artefact directories
|
||||
- run:
|
||||
# building aarch64 packages under qemu is very slow
|
||||
no_output_timeout: 60m
|
||||
command: bash scripts/remote_python_packaging.sh
|
||||
- persist_to_workspace:
|
||||
root: workspace
|
||||
paths:
|
||||
# - c-docs
|
||||
- py-docs
|
||||
- wheelhouse
|
||||
|
||||
upload_docs_wheels:
|
||||
machine: true
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: workspace
|
||||
- run: ls -laR workspace
|
||||
- run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
|
||||
|
||||
workflows:
|
||||
version: 2.1
|
||||
|
||||
test:
|
||||
jobs:
|
||||
- remote_python_packaging:
|
||||
filters:
|
||||
tags:
|
||||
only: /.+/
|
||||
branches:
|
||||
ignore: /.*/
|
||||
|
||||
- build_doxygen:
|
||||
filters:
|
||||
tags:
|
||||
only: /.+/
|
||||
branches:
|
||||
ignore: /.*/
|
||||
|
||||
- upload_docs_wheels:
|
||||
requires:
|
||||
- remote_python_packaging
|
||||
- build_doxygen
|
||||
filters:
|
||||
tags:
|
||||
only: /.+/
|
||||
branches:
|
||||
ignore: /.*/
|
||||
@@ -1,75 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ -z "$DEVPI_LOGIN" ] ; then
|
||||
echo "required: password for 'dc' user on https://m.devpi/net/dc index"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set -xe
|
||||
|
||||
PYDOCDIR=${1:?directory with python docs}
|
||||
WHEELHOUSEDIR=${2:?directory with pre-built wheels}
|
||||
DOXYDOCDIR=${3:?directory where doxygen docs to be found}
|
||||
SSHTARGET=ci@b1.delta.chat
|
||||
|
||||
|
||||
# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable.
|
||||
export BRANCH=${CIRCLE_BRANCH:master}
|
||||
|
||||
export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
|
||||
|
||||
|
||||
# python docs to py.delta.chat
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
|
||||
rsync -avz \
|
||||
--delete \
|
||||
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
|
||||
"$PYDOCDIR/html/" \
|
||||
delta@py.delta.chat:build/${BRANCH}
|
||||
|
||||
# C docs to c.delta.chat
|
||||
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
|
||||
rsync -avz \
|
||||
--delete \
|
||||
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
|
||||
"$DOXYDOCDIR/html/" \
|
||||
delta@c.delta.chat:build-c/${BRANCH}
|
||||
|
||||
echo -----------------------
|
||||
echo upload wheels
|
||||
echo -----------------------
|
||||
|
||||
# Bundle external shared libraries into the wheels
|
||||
|
||||
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR
|
||||
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR
|
||||
rsync -avz \
|
||||
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
|
||||
$WHEELHOUSEDIR \
|
||||
$SSHTARGET:$BUILDDIR
|
||||
|
||||
ssh $SSHTARGET <<_HERE
|
||||
set +x -e
|
||||
# make sure all processes exit when ssh dies
|
||||
shopt -s huponexit
|
||||
|
||||
# we rely on the "venv" virtualenv on the remote account to exist
|
||||
source venv/bin/activate
|
||||
cd $BUILDDIR
|
||||
|
||||
devpi use https://m.devpi.net
|
||||
devpi login dc --password $DEVPI_LOGIN
|
||||
|
||||
N_BRANCH=${BRANCH//[\/]}
|
||||
|
||||
devpi use dc/\$N_BRANCH || {
|
||||
devpi index -c \$N_BRANCH
|
||||
devpi use dc/\$N_BRANCH
|
||||
}
|
||||
devpi index \$N_BRANCH bases=/root/pypi
|
||||
devpi upload wheelhouse/deltachat*
|
||||
|
||||
# remove devpi non-master dc indices if thy are too old
|
||||
# this script was copied above
|
||||
python cleanup_devpi_indices.py
|
||||
_HERE
|
||||
21
scripts/concourse/README.md
Normal file
21
scripts/concourse/README.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# Concourse CI pipeline
|
||||
|
||||
`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
|
||||
that builds C documentation, Python documentation, Python wheels for `x86_64`
|
||||
and `aarch64` and Python source packages, and uploads them.
|
||||
|
||||
To setup the pipeline run
|
||||
```
|
||||
fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
|
||||
```
|
||||
where `secret.yml` contains the following secrets:
|
||||
```
|
||||
c.delta.chat:
|
||||
private_key: |
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
...
|
||||
-----END RSA PRIVATE KEY-----
|
||||
devpi:
|
||||
login: dc
|
||||
password: ...
|
||||
```
|
||||
217
scripts/concourse/docs_wheels.yml
Normal file
217
scripts/concourse/docs_wheels.yml
Normal file
@@ -0,0 +1,217 @@
|
||||
resources:
|
||||
- name: deltachat-core-rust
|
||||
type: git
|
||||
icon: github
|
||||
source:
|
||||
branch: master
|
||||
uri: https://github.com/deltachat/deltachat-core-rust.git
|
||||
|
||||
jobs:
|
||||
- name: doxygen
|
||||
plan:
|
||||
- get: deltachat-core-rust
|
||||
|
||||
# Build Doxygen documentation
|
||||
- task: build-doxygen
|
||||
config:
|
||||
inputs:
|
||||
- name: deltachat-core-rust
|
||||
outputs:
|
||||
- name: c-docs
|
||||
image_resource:
|
||||
source:
|
||||
repository: hrektts/doxygen
|
||||
type: registry-image
|
||||
platform: linux
|
||||
run:
|
||||
path: bash
|
||||
args:
|
||||
- -exc
|
||||
- |
|
||||
cd deltachat-core-rust
|
||||
bash scripts/run-doxygen.sh
|
||||
cd ..
|
||||
cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
|
||||
|
||||
- task: upload-c-docs
|
||||
config:
|
||||
inputs:
|
||||
- name: c-docs
|
||||
image_resource:
|
||||
type: registry-image
|
||||
source:
|
||||
repository: alpine
|
||||
platform: linux
|
||||
run:
|
||||
path: sh
|
||||
args:
|
||||
- -ec
|
||||
- |
|
||||
apk add --no-cache rsync openssh-client
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
|
||||
|
||||
- name: python-x86_64
|
||||
plan:
|
||||
- get: deltachat-core-rust
|
||||
|
||||
# Build manylinux image with additional dependencies
|
||||
- task: build-coredeps
|
||||
privileged: true
|
||||
config:
|
||||
inputs:
|
||||
- name: deltachat-core-rust
|
||||
image_resource:
|
||||
source:
|
||||
repository: vito/oci-build-task
|
||||
type: registry-image
|
||||
outputs:
|
||||
- name: coredeps-image
|
||||
path: image
|
||||
params:
|
||||
CONTEXT: deltachat-core-rust/scripts/docker-coredeps
|
||||
UNPACK_ROOTFS: "true"
|
||||
platform: linux
|
||||
caches:
|
||||
- path: cache
|
||||
run:
|
||||
path: build
|
||||
|
||||
# Use built image to build python wheels
|
||||
- task: build-wheels
|
||||
image: coredeps-image
|
||||
config:
|
||||
inputs:
|
||||
- name: deltachat-core-rust
|
||||
path: .
|
||||
outputs:
|
||||
- name: py-docs
|
||||
path: ./python/doc/_build/
|
||||
# Source packages
|
||||
- name: py-dist
|
||||
path: ./python/.docker-tox/dist/
|
||||
# Binary wheels
|
||||
- name: py-wheels
|
||||
path: ./python/.docker-tox/wheelhouse/
|
||||
platform: linux
|
||||
run:
|
||||
path: bash
|
||||
args:
|
||||
- -exc
|
||||
- |
|
||||
scripts/run_all.sh
|
||||
|
||||
# Upload python docs to py.delta.chat
|
||||
- task: upload-py-docs
|
||||
config:
|
||||
inputs:
|
||||
- name: py-docs
|
||||
image_resource:
|
||||
type: registry-image
|
||||
source:
|
||||
repository: alpine
|
||||
platform: linux
|
||||
run:
|
||||
path: sh
|
||||
args:
|
||||
- -ec
|
||||
- |
|
||||
apk add --no-cache rsync openssh-client
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
|
||||
|
||||
# Upload x86_64 wheels and source packages
|
||||
- task: upload-wheels
|
||||
config:
|
||||
inputs:
|
||||
- name: py-wheels
|
||||
- name: py-dist
|
||||
image_resource:
|
||||
type: registry-image
|
||||
source:
|
||||
repository: debian
|
||||
platform: linux
|
||||
run:
|
||||
path: sh
|
||||
args:
|
||||
- -ec
|
||||
- |
|
||||
apt-get update -y
|
||||
apt-get install -y --no-install-recommends python3-pip python3-setuptools
|
||||
pip3 install devpi
|
||||
devpi use https://m.devpi.net/dc/master
|
||||
devpi login ((devpi.login)) --password ((devpi.password))
|
||||
devpi upload py-wheels/*manylinux201*
|
||||
devpi upload py-dist/*
|
||||
|
||||
- name: python-aarch64
|
||||
plan:
|
||||
- get: deltachat-core-rust
|
||||
|
||||
# Build manylinux image with additional dependencies
|
||||
- task: build-coredeps
|
||||
privileged: true
|
||||
config:
|
||||
inputs:
|
||||
- name: deltachat-core-rust
|
||||
image_resource:
|
||||
source:
|
||||
repository: vito/oci-build-task
|
||||
type: registry-image
|
||||
outputs:
|
||||
- name: coredeps-image
|
||||
path: image
|
||||
params:
|
||||
CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
|
||||
UNPACK_ROOTFS: "true"
|
||||
platform: linux
|
||||
caches:
|
||||
- path: cache
|
||||
run:
|
||||
path: build
|
||||
|
||||
# Use built image to build python wheels
|
||||
- task: build-wheels
|
||||
image: coredeps-image
|
||||
config:
|
||||
inputs:
|
||||
- name: deltachat-core-rust
|
||||
path: .
|
||||
outputs:
|
||||
- name: py-wheels
|
||||
path: ./python/.docker-tox/wheelhouse/
|
||||
platform: linux
|
||||
run:
|
||||
path: bash
|
||||
args:
|
||||
- -exc
|
||||
- |
|
||||
scripts/run_all.sh
|
||||
|
||||
# Upload aarch64 wheels
|
||||
- task: upload-wheels
|
||||
config:
|
||||
inputs:
|
||||
- name: py-wheels
|
||||
image_resource:
|
||||
type: registry-image
|
||||
source:
|
||||
repository: debian
|
||||
platform: linux
|
||||
run:
|
||||
path: sh
|
||||
args:
|
||||
- -ec
|
||||
- |
|
||||
apt-get update -y
|
||||
apt-get install -y --no-install-recommends python3-pip python3-setuptools
|
||||
pip3 install devpi
|
||||
devpi use https://m.devpi.net/dc/master
|
||||
devpi login ((devpi.login)) --password ((devpi.password))
|
||||
devpi upload py-wheels/*manylinux201*
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
export BRANCH=${CIRCLE_BRANCH:-master}
|
||||
export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
|
||||
export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
|
||||
|
||||
export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
|
||||
|
||||
echo "--- Copying files to $SSHTARGET:$BUILDDIR"
|
||||
|
||||
set -xe
|
||||
|
||||
ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
|
||||
git ls-files >.rsynclist
|
||||
# we seem to need .git for setuptools_scm versioning
|
||||
find .git >>.rsynclist
|
||||
rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
|
||||
rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
|
||||
|
||||
set +x
|
||||
|
||||
# we have to create a remote file for the remote-docker run
|
||||
# so we can do a simple ssh command with a TTY
|
||||
# so that when our job dies, all container-runs are aborted.
|
||||
# sidenote: the circle-ci machinery will kill ongoing jobs
|
||||
# if there are new commits and we want to ensure that
|
||||
# everything is terminated/cleaned up and we have no orphaned
|
||||
# useless still-running docker-containers consuming resources.
|
||||
|
||||
for arch in "" "-arm64"; do
|
||||
|
||||
ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
|
||||
set +x -e
|
||||
shopt -s huponexit
|
||||
cd ${BUILDDIR}${arch}
|
||||
export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
|
||||
set -x
|
||||
|
||||
# run everything else inside docker
|
||||
docker run -e DCC_NEW_TMP_EMAIL \
|
||||
--rm -it -v \$(pwd):/mnt -w /mnt \
|
||||
deltachat/coredeps${arch} scripts/run_all.sh
|
||||
|
||||
_HERE
|
||||
|
||||
done
|
||||
|
||||
echo "--- Running $CIRCLE_JOB remotely"
|
||||
|
||||
echo "--- Building aarch64 wheels"
|
||||
ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
|
||||
|
||||
echo "--- Building x86_64 wheels"
|
||||
ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
|
||||
|
||||
mkdir -p workspace
|
||||
|
||||
# Wheels
|
||||
for arch in "" "-arm64"; do
|
||||
rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
|
||||
done
|
||||
|
||||
# Source packages
|
||||
rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
|
||||
|
||||
# Documentation
|
||||
rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs
|
||||
Reference in New Issue
Block a user