Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-17 13:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Concourse CI pipeline to replace Circle CI

Browse Source
This commit is contained in:
link2xt
2021-05-30 05:59:55 +03:00
parent a638259c36
commit cf8736da48
5 changed files with 238 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
scripts/ci_upload.sh
View File

@@ -1,75 +0,0 @@
#!/bin/bash
if [ -z "$DEVPI_LOGIN" ] ; then
echo "required: password for 'dc' user on https://m.devpi/net/dc index"
exit 0
fi
set -xe
PYDOCDIR=${1:?directory with python docs}
WHEELHOUSEDIR=${2:?directory with pre-built wheels}
DOXYDOCDIR=${3:?directory where doxygen docs to be found}
SSHTARGET=ci@b1.delta.chat
# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable.
export BRANCH=${CIRCLE_BRANCH:master}
export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
# python docs to py.delta.chat
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
rsync -avz \
--delete \
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
"$PYDOCDIR/html/" \
delta@py.delta.chat:build/${BRANCH}
# C docs to c.delta.chat
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
rsync -avz \
--delete \
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
"$DOXYDOCDIR/html/" \
delta@c.delta.chat:build-c/${BRANCH}
echo -----------------------
echo upload wheels
echo -----------------------
# Bundle external shared libraries into the wheels
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR
rsync -avz \
-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
$WHEELHOUSEDIR \
$SSHTARGET:$BUILDDIR
ssh $SSHTARGET <<_HERE
set +x -e
# make sure all processes exit when ssh dies
shopt -s huponexit
# we rely on the "venv" virtualenv on the remote account to exist
source venv/bin/activate
cd $BUILDDIR
devpi use https://m.devpi.net
devpi login dc --password $DEVPI_LOGIN
N_BRANCH=${BRANCH//[\/]}
devpi use dc/\$N_BRANCH || {
devpi index -c \$N_BRANCH
devpi use dc/\$N_BRANCH
}
devpi index \$N_BRANCH bases=/root/pypi
devpi upload wheelhouse/deltachat*
# remove devpi non-master dc indices if thy are too old
# this script was copied above
python cleanup_devpi_indices.py
_HERE

21
scripts/concourse/README.md Normal file
View File

@@ -0,0 +1,21 @@
# Concourse CI pipeline
`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
that builds C documentation, Python documentation, Python wheels for `x86_64`
and `aarch64` and Python source packages, and uploads them.
To setup the pipeline run
```
fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
```
where `secret.yml` contains the following secrets:
```
c.delta.chat:
private_key: |
-----BEGIN RSA PRIVATE KEY-----
...
-----END RSA PRIVATE KEY-----
devpi:
login: dc
password: ...
```

217
scripts/concourse/docs_wheels.yml Normal file
View File

@@ -0,0 +1,217 @@
resources:
- name: deltachat-core-rust
type: git
icon: github
source:
branch: master
uri: https://github.com/deltachat/deltachat-core-rust.git
jobs:
- name: doxygen
plan:
- get: deltachat-core-rust
# Build Doxygen documentation
- task: build-doxygen
config:
inputs:
- name: deltachat-core-rust
outputs:
- name: c-docs
image_resource:
source:
repository: hrektts/doxygen
type: registry-image
platform: linux
run:
path: bash
args:
- -exc
- |
cd deltachat-core-rust
bash scripts/run-doxygen.sh
cd ..
cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
- task: upload-c-docs
config:
inputs:
- name: c-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
- name: python-x86_64
plan:
- get: deltachat-core-rust
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
- name: deltachat-core-rust
image_resource:
source:
repository: vito/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/docker-coredeps
UNPACK_ROOTFS: "true"
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust
path: .
outputs:
- name: py-docs
path: ./python/doc/_build/
# Source packages
- name: py-dist
path: ./python/.docker-tox/dist/
# Binary wheels
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload python docs to py.delta.chat
- task: upload-py-docs
config:
inputs:
- name: py-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
# Upload x86_64 wheels and source packages
- task: upload-wheels
config:
inputs:
- name: py-wheels
- name: py-dist
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*
devpi upload py-dist/*
- name: python-aarch64
plan:
- get: deltachat-core-rust
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
- name: deltachat-core-rust
image_resource:
source:
repository: vito/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
UNPACK_ROOTFS: "true"
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload aarch64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*

67
scripts/remote_python_packaging.sh
View File

@@ -1,67 +0,0 @@
#!/bin/bash
export BRANCH=${CIRCLE_BRANCH:-master}
export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
echo "--- Copying files to $SSHTARGET:$BUILDDIR"
set -xe
ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
git ls-files >.rsynclist
# we seem to need .git for setuptools_scm versioning
find .git >>.rsynclist
rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
set +x
# we have to create a remote file for the remote-docker run
# so we can do a simple ssh command with a TTY
# so that when our job dies, all container-runs are aborted.
# sidenote: the circle-ci machinery will kill ongoing jobs
# if there are new commits and we want to ensure that
# everything is terminated/cleaned up and we have no orphaned
# useless still-running docker-containers consuming resources.
for arch in "" "-arm64"; do
ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
set +x -e
shopt -s huponexit
cd ${BUILDDIR}${arch}
export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
set -x
# run everything else inside docker
docker run -e DCC_NEW_TMP_EMAIL \
--rm -it -v \$(pwd):/mnt -w /mnt \
deltachat/coredeps${arch} scripts/run_all.sh
_HERE
done
echo "--- Running $CIRCLE_JOB remotely"
echo "--- Building aarch64 wheels"
ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
echo "--- Building x86_64 wheels"
ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
mkdir -p workspace
# Wheels
for arch in "" "-arm64"; do
rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
done
# Source packages
rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
# Documentation
rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs