From e9511ebfc3342e976c7d86041b4e970678939f5e Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 2 May 2022 19:40:16 +0200 Subject: [PATCH] first steps to integrate deltachat-node to core repository, adjust CI: adjust scripts to new location of deltachat-core-rust adjust dc-node readme to repo change mention old repository migrate github actions, try out if they work fix path to node docs in SSH github action passing mailadm token to node tests hopefully fixing the download paths for the artifacts fixing download paths, this time for real post upload link to details fix scp command forgot to remove platform_status dict fixing paths in the github action add github action to delete node preview builds when PR is closed move environment variable to yaml remove git trash github action to release to npm use different action which also works with branches for testing we don't want to publish to NPM through the CI see what lint issues windows has --- .github/workflows/node-delete-preview.yml | 32 +++++++++ .../workflows/node-docs.yml | 6 +- .../workflows/node-package.yml | 71 ++++++++++++------- .../workflows/node-tests.yml | 31 +++++--- .gitignore | 7 ++ node/.gitattributes | 1 - .../build-and-test-integration-linux.yml | 51 ------------- node/.gitignore | 13 ---- node/.gitmodules | 3 - node/README.md | 50 ++++++++++--- node/package.json | 3 +- node/scripts/generate-constants.js | 2 +- node/scripts/install.js | 5 -- node/scripts/postLinksToDetails.js | 46 ++++++++++++ node/scripts/postinstall.js | 2 +- node/scripts/rebuild-core.js | 2 +- 16 files changed, 202 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/node-delete-preview.yml rename node/.github/workflows/upload-docs.yaml => .github/workflows/node-docs.yml (83%) rename node/.github/workflows/make-package.yml => .github/workflows/node-package.yml (60%) rename node/.github/workflows/build-and-test.yml => .github/workflows/node-tests.yml (72%) delete mode 100644 node/.gitattributes delete mode 100644 node/.github/workflows/build-and-test-integration-linux.yml delete mode 100644 node/.gitignore delete mode 100644 node/.gitmodules create mode 100644 node/scripts/postLinksToDetails.js diff --git a/.github/workflows/node-delete-preview.yml b/.github/workflows/node-delete-preview.yml new file mode 100644 index 000000000..12d4fe47f --- /dev/null +++ b/.github/workflows/node-delete-preview.yml @@ -0,0 +1,32 @@ +# documentation: https://github.com/deltachat/sysadmin/tree/master/download.delta.chat +name: Delete node PR previews + +on: + pull_request: + types: [closed] + +jobs: + delete: + + runs-on: ubuntu-latest + + steps: + - name: Get Pullrequest ID + id: getid + run: | + export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH) + echo ::set-output name=prid::$PULLREQUEST_ID + - name: Renaming + run: | + # create empty file to copy it over the outdated deliverable on download.delta.chat + echo "This preview build is outdated and has been removed." > empty + cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz + - name: Replace builds with dummy files + uses: horochx/deploy-via-scp@v1.0.1 + with: + user: ${{ secrets.USERNAME }} + key: ${{ secrets.SSH_KEY }} + host: "download.delta.chat" + port: 22 + local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz" + remote: "/var/www/html/download/node/" diff --git a/node/.github/workflows/upload-docs.yaml b/.github/workflows/node-docs.yml similarity index 83% rename from node/.github/workflows/upload-docs.yaml rename to .github/workflows/node-docs.yml index bff047f45..ad614a885 100644 --- a/node/.github/workflows/upload-docs.yaml +++ b/.github/workflows/node-docs.yml @@ -1,9 +1,10 @@ -name: Generate & upload documentation +name: Generate & upload node.js documentation on: push: branches: - master + - integrate-node-into-repo jobs: generate: @@ -18,6 +19,7 @@ jobs: - name: npm install and generate documentation run: | + cd node npm i --ignore-scripts npx typedoc mv docs js @@ -29,5 +31,5 @@ jobs: key: ${{ secrets.KEY }} host: "delta.chat" port: 22 - local: "js" + local: "node/js" remote: "/var/www/html/" diff --git a/node/.github/workflows/make-package.yml b/.github/workflows/node-package.yml similarity index 60% rename from node/.github/workflows/make-package.yml rename to .github/workflows/node-package.yml index 0fe25f4ce..c5befbf63 100644 --- a/node/.github/workflows/make-package.yml +++ b/.github/workflows/node-package.yml @@ -1,5 +1,6 @@ -name: 'Make Package' -on: push +name: 'Build node.js bindings from PR' +on: + pull_request: jobs: prebuild: @@ -22,9 +23,6 @@ jobs: npm --version node --version - - name: Pull submodule - run: npm run submodule - - name: Cache node modules uses: actions/cache@v2 with: @@ -39,15 +37,18 @@ jobs: path: | ~/.cargo/registry/ ~/.cargo/git - deltachat-core-rust/target + target key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2 - name: Install dependencies & build if: steps.cache.outputs.cache-hit != 'true' - run: npm install --verbose + run: | + cd node + npm install --verbose - name: Build Prebuild run: | + cd node npm run prebuildify tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds . @@ -55,7 +56,7 @@ jobs: uses: actions/upload-artifact@v1 with: name: ${{ matrix.os }} - path: ${{ matrix.os }}.tar.gz + path: node/${{ matrix.os }}.tar.gz pack-module: needs: prebuild @@ -69,6 +70,10 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '16' + - name: Get Pullrequest ID + id: prepare + run: | + node -e "console.log('::set-output name=prid::' + '${{ github.ref }}'.split('/')[2])" - name: System info run: | rustc -vV @@ -76,49 +81,63 @@ jobs: cargo -vV npm --version node --version - - name: npm run submodule - run: npm run submodule - name: Download ubuntu prebuild uses: actions/download-artifact@v1 with: name: ubuntu-18.04 + path: node/ - name: Download macos prebuild uses: actions/download-artifact@v1 with: name: macos-latest + path: node/ - name: Download windows prebuild uses: actions/download-artifact@v1 with: name: windows-latest + path: node/ - shell: bash run: | + cd node ls -lah mkdir prebuilds - tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C prebuilds - tar -xvzf macos-latest/macos-latest.tar.gz -C prebuilds - tar -xvzf windows-latest/windows-latest.tar.gz -C prebuilds + tar -xvzf ubuntu-18.04.tar.gz -C prebuilds + tar -xvzf macos-latest.tar.gz -C prebuilds + tar -xvzf windows-latest.tar.gz -C prebuilds tree prebuilds - name: install dependencies without running scripts - run: npm install --ignore-scripts + run: | + cd node + npm install --ignore-scripts - name: build typescript part - run: npm run build:bindings:ts + run: | + cd node + npm run build:bindings:ts - name: package shell: bash run: | + cd node npm pack . ls -lah - mv $(find deltachat-node-*) deltachat-node.tgz + mv $(find deltachat-node-*) deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz - name: Upload Prebuild uses: actions/upload-artifact@v1 with: name: deltachat-node.tgz - path: deltachat-node.tgz - - name: Upload to release - if: contains(github.ref, 'refs/tags/') - uses: svenstaro/upload-release-action@v1-release - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: deltachat-node.tgz - asset_name: deltachat-node.tgz - tag: ${{ github.ref }} - overwrite: true + path: node/deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz + # Upload Step + - name: upload folder + id: upload + shell: bash + run: | + cd node + echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE + chmod 600 __TEMP_INPUT_KEY_FILE + scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/" + continue-on-error: true + - name: "Post links to details" + if: steps.upload.outcome == 'success' + run: node ./node/scripts/postLinksToDetails.js + env: + PR_ID: ${{ steps.prepare.outputs.prid }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/node/.github/workflows/build-and-test.yml b/.github/workflows/node-tests.yml similarity index 72% rename from node/.github/workflows/build-and-test.yml rename to .github/workflows/node-tests.yml index 6221f6076..51044e565 100644 --- a/node/.github/workflows/build-and-test.yml +++ b/.github/workflows/node-tests.yml @@ -1,5 +1,11 @@ -name: 'Build & Test' -on: push +name: 'Run node.js tests' +on: + pull_request: + push: + branches: + - master + - staging + - trying jobs: build-and-test: @@ -22,9 +28,6 @@ jobs: npm --version node --version - - name: Pull submodule - run: npm run submodule - - name: Cache node modules uses: actions/cache@v1 with: @@ -37,16 +40,26 @@ jobs: path: | ~/.cargo/registry/ ~/.cargo/git - deltachat-core-rust/target + target key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2 - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' - run: npm install --ignore-scripts --verbose + run: | + cd node + npm install --ignore-scripts --verbose - name: Build deltachat-core-rust & bindings if: steps.cache.outputs.cache-hit != 'true' - run: npm run build + run: | + cd node + npm run build - name: Test - run: npm run test + run: | + cd node + npm run lint-fix + git diff + npm run test + env: + DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} diff --git a/.gitignore b/.gitignore index ab2f50a0b..97d1bae04 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,10 @@ coverage/ python/accounts.txt python/all-testaccounts.txt tmp/ + +# from deltachat-node +node/node_modules/ +node/build/ +node/dist/ +node/prebuilds/ +node/.nyc_output/ diff --git a/node/.gitattributes b/node/.gitattributes deleted file mode 100644 index 937b84ad7..000000000 --- a/node/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.ts text eol=lf \ No newline at end of file diff --git a/node/.github/workflows/build-and-test-integration-linux.yml b/node/.github/workflows/build-and-test-integration-linux.yml deleted file mode 100644 index 909fad350..000000000 --- a/node/.github/workflows/build-and-test-integration-linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: 'Linux' -on: push - -jobs: - build-and-test: - name: "Build & Test integration" - runs-on: ubuntu-18.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: System info - run: | - rustc -vV - rustup -vV - cargo -vV - npm --version - node --version - - - name: Pull submodule - run: npm run submodule - - - name: Cache node modules - uses: actions/cache@v1 - with: - path: ${{ env.APPDATA }}/npm-cache # npm cache files are stored in `~/.npm` on Linux/macOS - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }} - - - name: Cache cargo index - uses: actions/cache@v1 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }} - - - name: Cache cargo build - uses: actions/cache@v1 - with: - path: deltachat-core-rust/target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} - - - name: Install dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: npm install --ignore-scripts --verbose - - - name: Build deltachat-core-rust & bindings - if: steps.cache.outputs.cache-hit != 'true' - run: npm run build - - - name: Tests (+ Integration tests) - run: npm run test - env: - DCC_NEW_TMP_EMAIL: ${{secrets.DCC_NEW_TMP_EMAIL}} diff --git a/node/.gitignore b/node/.gitignore deleted file mode 100644 index b6a5650bc..000000000 --- a/node/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -build/ -package-lock.json -node_modules/ -test.sqlite -db.sqlite -db.sqlite-journal -.nyc_output/ -coverage/ -prebuilds/ -*.tar.gz -dist/ -docs/ -.DS_Store diff --git a/node/.gitmodules b/node/.gitmodules deleted file mode 100644 index 0bb85c95e..000000000 --- a/node/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "deltachat-core-rust"] - path = deltachat-core-rust - url = ../deltachat-core-rust diff --git a/node/README.md b/node/README.md index f423732e7..06241a141 100644 --- a/node/README.md +++ b/node/README.md @@ -8,13 +8,13 @@ [![dependencies](https://david-dm.org/deltachat/deltachat-node.svg)](https://david-dm.org/deltachat/deltachat-node) [![JavaScript Style Guide](https://img.shields.io/badge/code_style-prettier-brightgreen.svg)](https://prettier.io) -**If you are upgrading:** please see [`UPGRADING.md`](UPGRADING.md). - `deltachat-node` primarily aims to offer two things: - A high level JavaScript api with syntactic sugar - A low level c binding api around [`deltachat-core-rust`][deltachat-core-rust] +This code used to live at [`deltachat-node`][deltachat-node]. + ## Table of Contents
Click to expand @@ -31,8 +31,9 @@ ## Install By default the installation will build try to use the bundled prebuilds in the -npm package. If this fails it falls back to compile the bundled -`deltachat-core-rust` from the submodule using `scripts/rebuild-core.js`. +npm package. If this fails it falls back to compile `deltachat-core-rust` from +this repository, using `scripts/rebuild-core.js`. + To install from npm use: ``` @@ -55,6 +56,42 @@ building from source or clone this repository and follow this steps: 1. `git clone https://github.com/deltachat/deltachat-node.git` 2. `cd deltachat-node` 3. `npm i` +4. `npm run build` + +### Use build-from-source in deltachat-desktop + +If you want to use the manually built node bindings in the desktop client (for +example), you can follow these instructions: + +First clone the +[deltachat-desktop](https://github.com/deltachat/deltachat-desktop) repository, +e.g. with `git clone https://github.com/deltachat/deltachat-desktop`. + +Then you need to make sure that this directory is referenced correctly in +deltachat-desktop's package.json. You need to change +`deltachat-desktop/package.json` like this: + +``` +diff --git i/package.json w/package.json +index d3fb3f23..b6ee6b7f 100644 +--- i/package.json ++++ w/package.json +@@ -83,7 +83,7 @@ + "application-config": "^1.0.1", + "classnames": "^2.3.1", + "debounce": "^1.2.0", +- "deltachat-node": "1.77.1", ++ "deltachat-node": "file:../deltachat-core-rust/node", + "emoji-js-clean": "^4.0.0", + "emoji-mart": "^3.0.1", + "emoji-regex": "^9.2.2", +``` + +Then, in the `deltachat-desktop` repository, run: + +1. `npm i` +2. `npm run build` +3. And `npm run start` to start the newly built client. ### Workaround to build for x86_64 on Apple's M1 @@ -173,7 +210,6 @@ We have the following scripts for building, testing and coverage: - `ǹpm run clean` Removes all built code - `npm run prebuildify` Builds prebuilt binary to `prebuilds/$PLATFORM-$ARCH`. Copies `deltachat.dll` from `deltachat-core-rust` for windows. - `npm run download-prebuilds` Downloads all prebuilt binaries from github before `npm publish`. -- `npm run submodule` Updates the `deltachat-core-rust` submodule. - `npm test` Runs `standard` and then the tests in `test/index.js`. - `npm run test-integration` Runs the integration tests. - `npm run hallmark` Runs `hallmark` on all markdown files. @@ -188,7 +224,7 @@ The following steps are needed to make a release: - Also adjust links to github prepare links at the end of the file 2. Bump version number in package.json -3. Commit the changed files, commit message should be similiar to `Prepare for v1.0.0-foo.number` +3. Commit the changed files, commit message should be similiar to `Prepare node-bindings npm release` 4. Tag the release with `git tag -a v1.0.0-foo.number` 5. Push to github with `git push origin master --tags` 6. Wait until `Make Package` github action is completed @@ -213,8 +249,6 @@ Licensed under `GPL-3.0-or-later`, see [LICENSE](./LICENSE) file for details. > You should have received a copy of the GNU General Public License > along with this program. If not, see . -[deltachat-core-rust]: https://github.com/deltachat/deltachat-core-rust - [appveyor-shield]: https://ci.appveyor.com/api/projects/status/t0narp672wpbl6pd?svg=true [appveyor]: https://ci.appveyor.com/project/ralphtheninja/deltachat-node-d4bf8 diff --git a/node/package.json b/node/package.json index 11b03c731..a89e6f10a 100644 --- a/node/package.json +++ b/node/package.json @@ -9,7 +9,7 @@ "coverage-html-report": "rm -rf coverage/ && nyc report --reporter=html && xdg-open coverage/index.html", "install": "node scripts/install.js", "install:prebuilds": "node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"", - "clean": "rm -rf ./dist ./build ./prebuilds ./deltachat-core-rust/target", + "clean": "rm -rf ./dist ./build ./prebuilds ./node_modules ../target", "build": "npm run build:core && npm run build:bindings", "build:core": "npm run build:core:rust && npm run build:core:constants", "build:core:rust": "node scripts/rebuild-core.js", @@ -21,7 +21,6 @@ "build:bindings:ts": "tsc", "prebuildify": "prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"", "download-prebuilds": "prebuildify-ci download", - "submodule": "git submodule update --recursive --init", "test": "npm run test:lint && npm run test:mocha", "test:mocha": "mocha -r esm test/test.js --growl --reporter=spec", "test:lint": "npm run lint", diff --git a/node/scripts/generate-constants.js b/node/scripts/generate-constants.js index fc3de2d97..72dc7b613 100755 --- a/node/scripts/generate-constants.js +++ b/node/scripts/generate-constants.js @@ -8,7 +8,7 @@ const data = [] const regex = /^#define\s+(\w+)\s+(\w+)/i const header = path.resolve( __dirname, - '../deltachat-core-rust/deltachat-ffi/deltachat.h' + '../../deltachat-ffi/deltachat.h' ) console.log('Generating constants...') diff --git a/node/scripts/install.js b/node/scripts/install.js index c1cdca2e5..3f320be4a 100644 --- a/node/scripts/install.js +++ b/node/scripts/install.js @@ -7,11 +7,6 @@ const run = (cmd) => { execSync(cmd, {stdio: 'inherit'}) } -if (!existsSync(join(__dirname, '..', 'deltachat-core-rust'))) { - console.log('[i] deltachat-node/deltachat-core-rust doesn\'t exist, fetching submodule. If this fails, make sure git is installed') - run('npm run submodule') -} - // Build bindings if (process.env.USE_SYSTEM_LIBDELTACHAT === 'true') { console.log('[i] USE_SYSTEM_LIBDELTACHAT is true, rebuilding c bindings and using pkg-config to retrieve lib paths and cflags of libdeltachat') diff --git a/node/scripts/postLinksToDetails.js b/node/scripts/postLinksToDetails.js new file mode 100644 index 000000000..e8a2ad43d --- /dev/null +++ b/node/scripts/postLinksToDetails.js @@ -0,0 +1,46 @@ +const { readFileSync } = require('fs') + +const sha = JSON.parse( + readFileSync(process.env['GITHUB_EVENT_PATH'], 'utf8') +).pull_request.head.sha + +const base_url = + 'https://download.delta.chat/node/deltachat-node-' + +const GITHUB_API_URL = + 'https://api.github.com/repos/deltachat/deltachat-core-rust/statuses/' + sha + +const prId = process.env['PR_ID'] +const GITHUB_TOKEN = process.env['GITHUB_TOKEN'] + +const STATUS_DATA = { + state: 'success', + description: '⏩ Click on "Details" to download →', + context: 'Download the node-bindings.tar.gz', + target_url: base_url + prId + '.tar.gz', +} + +const http = require('https') + +const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'User-Agent': 'github-action ci for deltachat deskop', + authorization: 'Bearer ' + GITHUB_TOKEN, + }, +} + +const req = http.request(GITHUB_API_URL, options, function(res) { + var chunks = [] + res.on('data', function(chunk) { + chunks.push(chunk) + }) + res.on('end', function() { + var body = Buffer.concat(chunks) + console.log(body.toString()) + }) +}) + +req.write(JSON.stringify(STATUS_DATA)) +req.end() diff --git a/node/scripts/postinstall.js b/node/scripts/postinstall.js index 6253469d1..15eb241ad 100644 --- a/node/scripts/postinstall.js +++ b/node/scripts/postinstall.js @@ -9,7 +9,7 @@ if (process.platform !== 'win32') { const from = path.resolve( __dirname, '..', - 'deltachat-core-rust', + '..', 'target', 'release', 'deltachat.dll' diff --git a/node/scripts/rebuild-core.js b/node/scripts/rebuild-core.js index b29ebf2ac..80dedcce5 100644 --- a/node/scripts/rebuild-core.js +++ b/node/scripts/rebuild-core.js @@ -1,7 +1,7 @@ const path = require('path') const { spawn } = require('./common') const opts = { - cwd: path.resolve(__dirname, '../deltachat-core-rust'), + cwd: path.resolve(__dirname, '../..'), stdio: 'inherit' }