first steps to integrate deltachat-node to core repository, adjust CI:

adjust scripts to new location of deltachat-core-rust
adjust dc-node readme to repo change
mention old repository
migrate github actions, try out if they work
fix path to node docs in SSH github action
passing mailadm token to node tests
hopefully fixing the download paths for the artifacts
fixing download paths, this time for real
post upload link to details
fix scp command
forgot to remove platform_status dict
fixing paths in the github action
add github action to delete node preview builds when PR is closed
move environment variable to yaml
remove git trash
github action to release to npm
use different action which also works with branches for testing
we don't want to publish to NPM through the CI
see what lint issues windows has

.github/workflows/node-delete-preview.yml

@@ -0,0 +1,32 @@
+# documentation: https://github.com/deltachat/sysadmin/tree/master/download.delta.chat
+name: Delete node PR previews
+
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  delete:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Get Pullrequest ID
+      id: getid
+      run: |
+        export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
+        echo ::set-output name=prid::$PULLREQUEST_ID
+    - name: Renaming
+      run: |
+        # create empty file to copy it over the outdated deliverable on download.delta.chat
+        echo "This preview build is outdated and has been removed." > empty
+        cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
+    - name: Replace builds with dummy files
+      uses: horochx/deploy-via-scp@v1.0.1
+      with:
+        user: ${{ secrets.USERNAME }}
+        key: ${{ secrets.SSH_KEY }}
+        host: "download.delta.chat"
+        port: 22
+        local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
+        remote: "/var/www/html/download/node/"

.github/workflows/node-docs.yml

@@ -1,9 +1,10 @@
-name: Generate & upload documentation
+name: Generate & upload node.js documentation
 
 on:
   push:
     branches:
       - master
+      - integrate-node-into-repo
 
 jobs:
   generate:
@@ -18,6 +19,7 @@ jobs:
 
     - name: npm install and generate documentation
       run: |
+        cd node
         npm i --ignore-scripts
         npx typedoc
         mv docs js
@@ -29,5 +31,5 @@ jobs:
         key: ${{ secrets.KEY }}
         host: "delta.chat"
         port: 22
-        local: "js"
+        local: "node/js"
         remote: "/var/www/html/"

.github/workflows/node-package.yml

@@ -1,5 +1,6 @@
-name: 'Make Package'
+name: 'Build node.js bindings from PR'
-on: push
+on:
+  pull_request:
 
 jobs:
   prebuild:
@@ -22,9 +23,6 @@ jobs:
           npm --version
           node --version
 
-      - name: Pull submodule
-        run: npm run submodule
-
       - name: Cache node modules
         uses: actions/cache@v2
         with:
@@ -39,15 +37,18 @@ jobs:
           path: |
             ~/.cargo/registry/
             ~/.cargo/git
-            deltachat-core-rust/target
+            target
           key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2
 
       - name: Install dependencies & build
         if: steps.cache.outputs.cache-hit != 'true'
-        run: npm install --verbose
+        run: |
+          cd node
+          npm install --verbose
 
       - name: Build Prebuild
         run: |
+          cd node
           npm run prebuildify
           tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds .
 
@@ -55,7 +56,7 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: ${{ matrix.os }}
-          path: ${{ matrix.os }}.tar.gz
+          path: node/${{ matrix.os }}.tar.gz
 
   pack-module:
     needs: prebuild
@@ -69,6 +70,10 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
+      - name: Get Pullrequest ID
+        id: prepare
+        run: |
+          node -e "console.log('::set-output name=prid::' + '${{ github.ref }}'.split('/')[2])"
       - name: System info
         run: |
           rustc -vV
@@ -76,49 +81,63 @@ jobs:
           cargo -vV
           npm --version
           node --version
-      - name: npm run submodule
-        run: npm run submodule
       - name: Download ubuntu prebuild
         uses: actions/download-artifact@v1
         with:
           name: ubuntu-18.04
+          path: node/
       - name: Download macos prebuild
         uses: actions/download-artifact@v1
         with:
           name: macos-latest
+          path: node/
       - name: Download windows prebuild
         uses: actions/download-artifact@v1
         with:
           name: windows-latest
+          path: node/
       - shell: bash
         run: |
+          cd node
           ls -lah
           mkdir prebuilds
-          tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C prebuilds
+          tar -xvzf ubuntu-18.04.tar.gz -C prebuilds
-          tar -xvzf macos-latest/macos-latest.tar.gz -C prebuilds
+          tar -xvzf macos-latest.tar.gz -C prebuilds
-          tar -xvzf windows-latest/windows-latest.tar.gz -C prebuilds
+          tar -xvzf windows-latest.tar.gz -C prebuilds
           tree prebuilds
       - name: install dependencies without running scripts
-        run: npm install --ignore-scripts
+        run: |
+          cd node
+          npm install --ignore-scripts
       - name: build typescript part
-        run: npm run build:bindings:ts
+        run: |
+          cd node
+          npm run build:bindings:ts
       - name: package
         shell: bash
         run: |
+          cd node
           npm pack .
           ls -lah
-          mv $(find deltachat-node-*) deltachat-node.tgz
+          mv $(find deltachat-node-*) deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz
       - name: Upload Prebuild
         uses: actions/upload-artifact@v1
         with:
           name: deltachat-node.tgz
-          path: deltachat-node.tgz
+          path: node/deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz
-      - name: Upload to release
+      # Upload Step
-        if: contains(github.ref, 'refs/tags/')
+      - name: upload folder
-        uses: svenstaro/upload-release-action@v1-release
+        id: upload
-        with:
+        shell: bash
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
+        run: |
-          file: deltachat-node.tgz
+          cd node
-          asset_name: deltachat-node.tgz
+          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
-          tag: ${{ github.ref }}
+          chmod 600 __TEMP_INPUT_KEY_FILE
-          overwrite: true
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"
+        continue-on-error: true
+      - name: "Post links to details"
+        if: steps.upload.outcome == 'success'
+        run: node ./node/scripts/postLinksToDetails.js
+        env:
+         PR_ID: ${{ steps.prepare.outputs.prid }}
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/node-tests.yml

@@ -1,5 +1,11 @@
-name: 'Build & Test'
+name: 'Run node.js tests'
-on: push
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
 
 jobs:
   build-and-test:
@@ -22,9 +28,6 @@ jobs:
           npm --version
           node --version
 
-      - name: Pull submodule
-        run: npm run submodule
-
       - name: Cache node modules
         uses: actions/cache@v1
         with:
@@ -37,16 +40,26 @@ jobs:
           path: |
             ~/.cargo/registry/
             ~/.cargo/git
-            deltachat-core-rust/target
+            target
           key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2
 
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
-        run: npm install --ignore-scripts --verbose
+        run: |
+          cd node
+          npm install --ignore-scripts --verbose
 
       - name: Build deltachat-core-rust & bindings
         if: steps.cache.outputs.cache-hit != 'true'
-        run: npm run build
+        run: |
+          cd node
+          npm run build
 
       - name: Test
-        run: npm run test
+        run: |
+          cd node
+          npm run lint-fix
+          git diff
+          npm run test
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}

.gitignore

@@ -33,3 +33,10 @@ coverage/
 python/accounts.txt
 python/all-testaccounts.txt
 tmp/
+
+# from deltachat-node
+node/node_modules/
+node/build/
+node/dist/
+node/prebuilds/
+node/.nyc_output/

node/.gitattributes

@@ -1 +0,0 @@
-*.ts text eol=lf

node/.github/workflows/build-and-test-integration-linux.yml

@@ -1,51 +0,0 @@
-name: 'Linux'
-on: push
-
-jobs:
-  build-and-test:
-    name: "Build & Test integration"
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: System info
-        run: |
-          rustc -vV
-          rustup -vV
-          cargo -vV
-          npm --version
-          node --version
-
-      - name: Pull submodule
-        run: npm run submodule
-
-      - name: Cache node modules
-        uses: actions/cache@v1
-        with:
-          path: ${{ env.APPDATA }}/npm-cache # npm cache files are stored in `~/.npm` on Linux/macOS
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}
-
-      - name: Cache cargo index
-        uses: actions/cache@v1
-        with:
-          path: ~/.cargo/git
-          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Cache cargo build
-        uses: actions/cache@v1
-        with:
-          path: deltachat-core-rust/target
-          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: npm install --ignore-scripts --verbose
-
-      - name: Build deltachat-core-rust & bindings
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: npm run build
-
-      - name: Tests (+ Integration tests)
-        run: npm run test
-        env:
-          DCC_NEW_TMP_EMAIL: ${{secrets.DCC_NEW_TMP_EMAIL}}

node/.gitignore

@@ -1,13 +0,0 @@
-build/
-package-lock.json
-node_modules/
-test.sqlite
-db.sqlite
-db.sqlite-journal
-.nyc_output/
-coverage/
-prebuilds/
-*.tar.gz
-dist/
-docs/
-.DS_Store

node/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "deltachat-core-rust"]
-	path = deltachat-core-rust
-	url = ../deltachat-core-rust

node/README.md

@@ -8,13 +8,13 @@
 [![dependencies](https://david-dm.org/deltachat/deltachat-node.svg)](https://david-dm.org/deltachat/deltachat-node)
 [![JavaScript Style Guide](https://img.shields.io/badge/code_style-prettier-brightgreen.svg)](https://prettier.io)
 
-**If you are upgrading:** please see [`UPGRADING.md`](UPGRADING.md).
-
 `deltachat-node` primarily aims to offer two things:
 
 - A high level JavaScript api with syntactic sugar
 - A low level c binding api around  [`deltachat-core-rust`][deltachat-core-rust]
 
+This code used to live at [`deltachat-node`][deltachat-node].
+
 ## Table of Contents
 
 <details><summary>Click to expand</summary>
@@ -31,8 +31,9 @@
 ## Install
 
 By default the installation will build try to use the bundled prebuilds in the
-npm package. If this fails it falls back to compile the bundled
+npm package. If this fails it falls back to compile `deltachat-core-rust` from
-`deltachat-core-rust` from the submodule using `scripts/rebuild-core.js`.
+this repository, using `scripts/rebuild-core.js`.
+
 To install from npm use:
 
 ```
@@ -55,6 +56,42 @@ building from source or clone this repository and follow this steps:
 1. `git clone https://github.com/deltachat/deltachat-node.git`
 2. `cd deltachat-node`
 3. `npm i`
+4. `npm run build`
+
+### Use build-from-source in deltachat-desktop
+
+If you want to use the manually built node bindings in the desktop client (for
+example), you can follow these instructions:
+
+First clone the
+[deltachat-desktop](https://github.com/deltachat/deltachat-desktop) repository,
+e.g. with `git clone https://github.com/deltachat/deltachat-desktop`.
+
+Then you need to make sure that this directory is referenced correctly in
+deltachat-desktop's package.json. You need to change
+`deltachat-desktop/package.json` like this:
+
+```
+diff --git i/package.json w/package.json
+index d3fb3f23..b6ee6b7f 100644
+--- i/package.json
++++ w/package.json
+@@ -83,7 +83,7 @@
+     "application-config": "^1.0.1",
+     "classnames": "^2.3.1",
+     "debounce": "^1.2.0",
+-    "deltachat-node": "1.77.1",
++    "deltachat-node": "file:../deltachat-core-rust/node",
+     "emoji-js-clean": "^4.0.0",
+     "emoji-mart": "^3.0.1",
+     "emoji-regex": "^9.2.2",
+```
+
+Then, in the `deltachat-desktop` repository, run:
+
+1. `npm i`
+2. `npm run build`
+3. And `npm run start` to start the newly built client.
 
 ### Workaround to build for x86_64 on Apple's M1
 
@@ -173,7 +210,6 @@ We have the following scripts for building, testing and coverage:
 - `ǹpm run clean` Removes all built code
 - `npm run prebuildify` Builds prebuilt binary to `prebuilds/$PLATFORM-$ARCH`. Copies `deltachat.dll` from `deltachat-core-rust` for windows.
 - `npm run download-prebuilds` Downloads all prebuilt binaries from github before `npm publish`.
-- `npm run submodule` Updates the `deltachat-core-rust` submodule.
 - `npm test` Runs `standard` and then the tests in `test/index.js`.
 - `npm run test-integration` Runs the integration tests.
 - `npm run hallmark` Runs `hallmark` on all markdown files.
@@ -188,7 +224,7 @@ The following steps are needed to make a release:
 - Also adjust links to github prepare links at the end of the file
 
 2. Bump version number in package.json
-3. Commit the changed files, commit message should be similiar to `Prepare for v1.0.0-foo.number`
+3. Commit the changed files, commit message should be similiar to `Prepare node-bindings npm release`
 4. Tag the release with `git tag -a v1.0.0-foo.number`
 5. Push to github with `git push origin master --tags`
 6. Wait until `Make Package` github action is completed
@@ -213,8 +249,6 @@ Licensed under `GPL-3.0-or-later`, see [LICENSE](./LICENSE) file for details.
 >    You should have received a copy of the GNU General Public License
 >    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-[deltachat-core-rust]: https://github.com/deltachat/deltachat-core-rust
-
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/t0narp672wpbl6pd?svg=true
 
 [appveyor]: https://ci.appveyor.com/project/ralphtheninja/deltachat-node-d4bf8

node/package.json

@@ -9,7 +9,7 @@
     "coverage-html-report": "rm -rf coverage/ && nyc report --reporter=html && xdg-open coverage/index.html",
     "install": "node scripts/install.js",
     "install:prebuilds": "node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"",
-    "clean": "rm -rf ./dist ./build ./prebuilds ./deltachat-core-rust/target",
+    "clean": "rm -rf ./dist ./build ./prebuilds ./node_modules ../target",
     "build": "npm run build:core && npm run build:bindings",
     "build:core": "npm run build:core:rust && npm run build:core:constants",
     "build:core:rust": "node scripts/rebuild-core.js",
@@ -21,7 +21,6 @@
     "build:bindings:ts": "tsc",
     "prebuildify": "prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "download-prebuilds": "prebuildify-ci download",
-    "submodule": "git submodule update --recursive --init",
     "test": "npm run test:lint && npm run test:mocha",
     "test:mocha": "mocha -r esm test/test.js --growl --reporter=spec",
     "test:lint": "npm run lint",

node/scripts/generate-constants.js

@@ -8,7 +8,7 @@ const data = []
 const regex = /^#define\s+(\w+)\s+(\w+)/i
 const header = path.resolve(
   __dirname,
-  '../deltachat-core-rust/deltachat-ffi/deltachat.h'
+  '../../deltachat-ffi/deltachat.h'
 )
 
 console.log('Generating constants...')

node/scripts/install.js

@@ -7,11 +7,6 @@ const run = (cmd) => {
   execSync(cmd, {stdio: 'inherit'})
 }
 
-if (!existsSync(join(__dirname, '..', 'deltachat-core-rust'))) {
-  console.log('[i] deltachat-node/deltachat-core-rust doesn\'t exist, fetching submodule. If this fails, make sure git is installed')
-  run('npm run submodule')
-}
-
 // Build bindings
 if (process.env.USE_SYSTEM_LIBDELTACHAT === 'true') {
   console.log('[i] USE_SYSTEM_LIBDELTACHAT is true, rebuilding c bindings and using pkg-config to retrieve lib paths and cflags of libdeltachat')

node/scripts/postLinksToDetails.js

@@ -0,0 +1,46 @@
+const { readFileSync } = require('fs')
+
+const sha = JSON.parse(
+  readFileSync(process.env['GITHUB_EVENT_PATH'], 'utf8')
+).pull_request.head.sha
+
+const base_url =
+  'https://download.delta.chat/node/deltachat-node-'
+
+const GITHUB_API_URL =
+  'https://api.github.com/repos/deltachat/deltachat-core-rust/statuses/' + sha
+
+const prId = process.env['PR_ID']
+const GITHUB_TOKEN = process.env['GITHUB_TOKEN']
+
+const STATUS_DATA = {
+  state: 'success',
+  description: '⏩ Click on "Details" to download →',
+  context: 'Download the node-bindings.tar.gz',
+  target_url: base_url + prId + '.tar.gz',
+}
+
+const http = require('https')
+
+const options = {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'User-Agent': 'github-action ci for deltachat deskop',
+    authorization: 'Bearer ' + GITHUB_TOKEN,
+  },
+}
+
+const req = http.request(GITHUB_API_URL, options, function(res) {
+  var chunks = []
+  res.on('data', function(chunk) {
+    chunks.push(chunk)
+  })
+  res.on('end', function() {
+    var body = Buffer.concat(chunks)
+    console.log(body.toString())
+  })
+})
+
+req.write(JSON.stringify(STATUS_DATA))
+req.end()

node/scripts/postinstall.js

@@ -9,7 +9,7 @@ if (process.platform !== 'win32') {
 const from = path.resolve(
   __dirname,
   '..',
-  'deltachat-core-rust',
+  '..',
   'target',
   'release',
   'deltachat.dll'

node/scripts/rebuild-core.js

@@ -1,7 +1,7 @@
 const path = require('path')
 const { spawn } = require('./common')
 const opts = {
-  cwd: path.resolve(__dirname, '../deltachat-core-rust'),
+  cwd: path.resolve(__dirname, '../..'),
   stdio: 'inherit'
 }