Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-18 05:56:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into flub/send-backup

Browse Source
This commit is contained in:
Floris Bruynooghe
2023-02-20 17:39:52 +01:00
parent b920db12c7 eaa2ef5a44
commit a1e19e2c41
36 changed files with 641 additions and 649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.github/mergeable.yml vendored
View File

@@ -5,22 +5,22 @@ mergeable:
validate:
- do: or
validate:
- do: description
must_include:
regex: '#skip-changelog'
- do: and
validate:
- do: dependent
changed:
file: 'src/**'
required: ['CHANGELOG.md']
- do: dependent
changed:
file: 'deltachat-ffi/src/**'
required: ['CHANGELOG.md']
- do: description
must_include:
regex: "#skip-changelog"
- do: and
validate:
- do: dependent
changed:
file: "src/**"
required: ["CHANGELOG.md"]
- do: dependent
changed:
file: "deltachat-ffi/src/**"
required: ["CHANGELOG.md"]
fail:
- do: checks
status: 'action_required'
status: "action_required"
payload:
title: Changelog might need an update
summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

136
.github/workflows/ci.yml vendored
View File

@@ -8,28 +8,22 @@ on:
env:
RUSTFLAGS: -Dwarnings
jobs:
fmt:
name: Rustfmt
lint:
name: Rustfmt and Clippy
runs-on: ubuntu-latest
env:
RUSTUP_TOOLCHAIN: 1.67.1
steps:
- uses: actions/checkout@v3
- run: cargo fmt --all -- --check
run_clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install clippy
run: rustup toolchain install 1.67.1 --component clippy
- name: Install rustfmt and clippy
run: rustup toolchain install $RUSTUP_TOOLCHAIN --component rustfmt --component clippy
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Run rustfmt
run: cargo fmt --all -- --check
- name: Run clippy
env:
RUSTUP_TOOLCHAIN: 1.67.1
run: scripts/clippy.sh
docs:
@@ -69,73 +63,73 @@ jobs:
python: 3.7
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@master
- uses: actions/checkout@master
- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install ${{ matrix.rust }}
- run: rustup override set ${{ matrix.rust }}
- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install ${{ matrix.rust }}
- run: rustup override set ${{ matrix.rust }}
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Check
run: cargo check --workspace --bins --examples --tests --benches
- name: Check
run: cargo check --workspace --bins --examples --tests --benches
- name: Tests
run: cargo test --workspace
- name: Tests
run: cargo test --workspace
- name: Test cargo vendor
run: cargo vendor
- name: Test cargo vendor
run: cargo vendor
- name: Install python
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install python
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install tox
if: ${{ matrix.python }}
run: pip install tox
- name: Install tox
if: ${{ matrix.python }}
run: pip install tox
- name: Build C library
if: ${{ matrix.python }}
run: cargo build -p deltachat_ffi --features jsonrpc
- name: Build C library
if: ${{ matrix.python }}
run: cargo build -p deltachat_ffi --features jsonrpc
- name: Run python tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e lint,mypy,doc,py3
- name: Run python tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e lint,mypy,doc,py3
- name: Build deltachat-rpc-server
if: ${{ matrix.python }}
run: cargo build -p deltachat-rpc-server
- name: Build deltachat-rpc-server
if: ${{ matrix.python }}
run: cargo build -p deltachat-rpc-server
- name: Add deltachat-rpc-server to path
if: ${{ matrix.python }}
run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
- name: Add deltachat-rpc-server to path
if: ${{ matrix.python }}
run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
- name: Run deltachat-rpc-client tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
working-directory: deltachat-rpc-client
run: tox -e py3,lint
- name: Run deltachat-rpc-client tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
working-directory: deltachat-rpc-client
run: tox -e py3,lint
- name: Install pypy
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: 'pypy${{ matrix.python }}'
- name: Install pypy
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: "pypy${{ matrix.python }}"
- name: Run pypy tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e pypy3
- name: Run pypy tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e pypy3

17
.github/workflows/jsonrpc-client-npm-package.yml vendored
View File

@@ -1,15 +1,14 @@
name: 'jsonrpc js client build'
name: "jsonrpc js client build"
on:
pull_request:
push:
tags:
- '*'
- '!py-*'
- "*"
- "!py-*"
jobs:
pack-module:
name: 'Package @deltachat/jsonrpc-client and upload to download.delta.chat'
name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat"
runs-on: ubuntu-18.04
steps:
- name: Install tree
@@ -18,7 +17,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16'
node-version: "16"
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
@@ -69,9 +68,9 @@ jobs:
if: steps.upload-preview.outcome == 'success'
run: node ./node/scripts/postLinksToDetails.js
env:
URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
# Upload to download.delta.chat/node/
- name: Upload deltachat-jsonrpc-client build to download.delta.chat/node/
if: ${{ steps.tag.outputs.tag }}

4
.github/workflows/jsonrpc.yml vendored
View File

@@ -35,6 +35,10 @@ jobs:
npm run test
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
- name: make sure websocket server version still builds
run: |
cd deltachat-jsonrpc
cargo build --bin deltachat-jsonrpc-server --features webserver
- name: Run linter
run: |
cd deltachat-jsonrpc/typescript

39
.github/workflows/node-delete-preview.yml vendored
View File

@@ -7,26 +7,25 @@ on:
jobs:
delete:
runs-on: ubuntu-latest
steps:
- name: Get Pullrequest ID
id: getid
run: |
export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
- name: Renaming
run: |
# create empty file to copy it over the outdated deliverable on download.delta.chat
echo "This preview build is outdated and has been removed." > empty
cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
- name: Replace builds with dummy files
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.SSH_KEY }}
host: "download.delta.chat"
port: 22
local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
remote: "/var/www/html/download/node/preview/"
- name: Get Pullrequest ID
id: getid
run: |
export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
- name: Renaming
run: |
# create empty file to copy it over the outdated deliverable on download.delta.chat
echo "This preview build is outdated and has been removed." > empty
cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
- name: Replace builds with dummy files
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.SSH_KEY }}
host: "download.delta.chat"
port: 22
local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
remote: "/var/www/html/download/node/preview/"

40
.github/workflows/node-docs.yml vendored
View File

@@ -9,26 +9,26 @@ jobs:
generate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Use Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Use Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: npm install and generate documentation
run: |
cd node
npm i --ignore-scripts
npx typedoc
mv docs js
- name: npm install and generate documentation
run: |
cd node
npm i --ignore-scripts
npx typedoc
mv docs js
- name: Upload
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.KEY }}
host: "delta.chat"
port: 22
local: "node/js"
remote: "/var/www/html/"
- name: Upload
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.KEY }}
host: "delta.chat"
port: 22
local: "node/js"
remote: "/var/www/html/"

15
.github/workflows/node-package.yml vendored
View File

@@ -1,11 +1,10 @@
name: 'node.js build'
name: "node.js build"
on:
pull_request:
push:
tags:
- '*'
- '!py-*'
- "*"
- "!py-*"
jobs:
prebuild:
@@ -19,7 +18,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16'
node-version: "16"
- name: System info
run: |
rustc -vV
@@ -74,7 +73,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v2
with:
node-version: '16'
node-version: "16"
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
@@ -152,8 +151,8 @@ jobs:
if: steps.upload-preview.outcome == 'success'
run: node ./node/scripts/postLinksToDetails.js
env:
URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Upload to download.delta.chat/node/
- name: Upload deltachat-node build to download.delta.chat/node/
if: ${{ steps.tag.outputs.tag }}

8
.github/workflows/node-tests.yml vendored
View File

@@ -1,4 +1,4 @@
name: 'node.js tests'
name: "node.js tests"
on:
pull_request:
push:
@@ -19,7 +19,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16'
node-version: "16"
- name: System info
run: |
rustc -vV
@@ -59,7 +59,7 @@ jobs:
npm run test
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"
- name: Run tests on Windows, except lint
timeout-minutes: 10
if: runner.os == 'Windows'
@@ -68,4 +68,4 @@ jobs:
npm run test:mocha
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

16
.github/workflows/repl.yml vendored
View File

@@ -11,13 +11,13 @@ jobs:
name: Build REPL example
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Build
run: cargo build -p deltachat-repl --features vendored
- name: Build
run: cargo build -p deltachat-repl --features vendored
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: repl.exe
path: 'target/debug/deltachat-repl.exe'
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: repl.exe
path: "target/debug/deltachat-repl.exe"

26
.github/workflows/upload-docs.yml vendored
View File

@@ -8,20 +8,18 @@ on:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat --no-deps
- name: Upload to rs.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/rs/"
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat --no-deps
- name: Upload to rs.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/rs/"

26
.github/workflows/upload-ffi-docs.yml vendored
View File

@@ -8,20 +8,18 @@ on:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat_ffi --no-deps
- name: Upload to cffi.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/cffi/"
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat_ffi --no-deps
- name: Upload to cffi.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/cffi/"

22
CHANGELOG.md
View File

@@ -3,20 +3,34 @@
## Unreleased
### Changes
- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
- python: mark bindings as supporting typing according to PEP 561 #4045
- retry filesystem operations during account migration #4043
- use transaction in `Contact::add_or_lookup()` #4059
- ability to send backup over network and QR code to setup second device #4007
### Fixes
- Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063
### API-Changes
## 1.109.0
### Changes
- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
### Fixes
- deltachat-rpc-server: do not block stdin while processing the request. #4041
deltachat-rpc-server now reads the next request as soon as previous request handler is spawned.
- enable `auto_vacuum` on all SQL connections #2955
- Enable `auto_vacuum` on all SQL connections. #2955
- Replace `r2d2` connection pool with an own implementation. #4050 #4053 #4043 #4061
This change improves reliability
by closing all database connections immediately when the context is closed.
### API-Changes
- Remove `MimeMessage::from_bytes()` public interface. #4033
- BREAKING Types: jsonrpc: `get_messages` now returns a map with `MessageLoadResult` instead of failing completely if one of the requested messages could not be loaded. #4038
- Add `dc_msg_set_subject()`. C-FFI #4057
- Mark python bindings as supporting typing according to PEP 561 #4045
## 1.108.0

188
Cargo.lock generated
View File

@@ -306,7 +306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
dependencies = [
"async-trait",
"axum-core 0.2.9",
"axum-core",
"base64 0.13.1",
"bitflags",
"bytes",
@@ -315,7 +315,7 @@ dependencies = [
"http-body",
"hyper",
"itoa",
"matchit 0.5.0",
"matchit",
"memchr",
"mime",
"percent-encoding",
@@ -326,43 +326,7 @@ dependencies = [
"sha-1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.17.2",
"tower",
"tower-http",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc"
dependencies = [
"async-trait",
"axum-core 0.3.2",
"base64 0.20.0",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit 0.7.0",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sha1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.18.0",
"tokio-tungstenite",
"tower",
"tower-http",
"tower-layer",
@@ -385,23 +349,6 @@ dependencies = [
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "backtrace"
version = "0.3.67"
@@ -441,12 +388,6 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
[[package]]
name = "base64"
version = "0.21.0"
@@ -1055,7 +996,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
[[package]]
name = "deltachat"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -1069,6 +1010,7 @@ dependencies = [
"bitflags",
"chrono",
"criterion",
"crossbeam-queue",
"deltachat_derive",
"email",
"encoded-words",
@@ -1096,8 +1038,6 @@ dependencies = [
"proptest",
"qrcodegen",
"quick-xml",
"r2d2",
"r2d2_sqlite",
"rand 0.8.5",
"ratelimit",
"regex",
@@ -1130,11 +1070,11 @@ dependencies = [
[[package]]
name = "deltachat-jsonrpc"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"async-channel",
"axum 0.6.4",
"axum",
"deltachat",
"env_logger 0.10.0",
"futures",
@@ -1152,7 +1092,7 @@ dependencies = [
[[package]]
name = "deltachat-repl"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -1167,7 +1107,7 @@ dependencies = [
[[package]]
name = "deltachat-rpc-server"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"deltachat-jsonrpc",
@@ -1190,7 +1130,7 @@ dependencies = [
[[package]]
name = "deltachat_ffi"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"deltachat",
@@ -1924,15 +1864,6 @@ version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -1944,11 +1875,11 @@ dependencies = [
[[package]]
name = "hashlink"
version = "0.7.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
dependencies = [
"hashbrown 0.11.2",
"hashbrown",
]
[[package]]
@@ -2190,7 +2121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"hashbrown",
]
[[package]]
@@ -2349,9 +2280,9 @@ checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565"
[[package]]
name = "libsqlite3-sys"
version = "0.24.2"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
dependencies = [
"cc",
"openssl-sys",
@@ -2443,12 +2374,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
[[package]]
name = "matchit"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
[[package]]
name = "md-5"
version = "0.10.5"
@@ -3219,27 +3144,6 @@ version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20f14e071918cbeefc5edc986a7aa92c425dae244e003a35e1cdddb5ca39b5cb"
[[package]]
name = "r2d2"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
dependencies = [
"log",
"parking_lot",
"scheduled-thread-pool",
]
[[package]]
name = "r2d2_sqlite"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fdc8e4da70586127893be32b7adf21326a4c6b1aba907611edf467d13ffe895"
dependencies = [
"r2d2",
"rusqlite",
]
[[package]]
name = "radix_trie"
version = "0.2.1"
@@ -3530,16 +3434,15 @@ dependencies = [
[[package]]
name = "rusqlite"
version = "0.27.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"memchr",
"smallvec",
]
@@ -3700,15 +3603,6 @@ dependencies = [
"windows-sys 0.36.1",
]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf"
dependencies = [
"parking_lot",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
@@ -3845,15 +3739,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "184c643044780f7ceb59104cef98a5a6f12cb2288a7bc701ab93a362b49fd47d"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.1"
@@ -4366,19 +4251,7 @@ dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.17.3",
]
[[package]]
name = "tokio-tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.18.0",
"tungstenite",
]
[[package]]
@@ -4628,25 +4501,6 @@ dependencies = [
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788"
dependencies = [
"base64 0.13.1",
"byteorder",
"bytes",
"http",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror",
"url",
"utf-8",
]
[[package]]
name = "twofish"
version = "0.7.1"
@@ -4704,7 +4558,7 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137"
dependencies = [
"hashbrown 0.12.3",
"hashbrown",
"regex",
]
@@ -5166,7 +5020,7 @@ dependencies = [
"async-channel",
"async-mutex",
"async-trait",
"axum 0.5.17",
"axum",
"futures",
"futures-util",
"log",

37
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat"
version = "1.108.0"
version = "1.109.0"
edition = "2021"
license = "MPL-2.0"
rust-version = "1.63"
@@ -29,21 +29,24 @@ format-flowed = { path = "./format-flowed" }
ratelimit = { path = "./deltachat-ratelimit" }
anyhow = "1"
async-channel = "1.8.0"
async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
async-smtp = { version = "0.8", default-features = false, features = ["runtime-tokio"] }
trust-dns-resolver = "0.22"
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
backtrace = "0.3"
base64 = "0.21"
bitflags = "1.3"
chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
crossbeam-queue = "0.3"
email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
escaper = "0.1"
fast-socks5 = "0.8"
futures = "0.3"
futures-lite = "1.12.0"
hex = "0.4.0"
humansize = "2"
image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
kamadak-exif = "0.5"
lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
@@ -57,12 +60,12 @@ once_cell = "1.17.0"
percent-encoding = "2.2"
pgp = { version = "0.9", default-features = false }
pretty_env_logger = { version = "0.4", optional = true }
qrcodegen = "1.7.0"
quick-xml = "0.27"
r2d2 = "0.8"
r2d2_sqlite = "0.20"
rand = "0.8"
regex = "1.7"
rusqlite = { version = "0.27", features = ["sqlcipher"] }
reqwest = { version = "0.11.14", features = ["json"] }
rusqlite = { version = "0.28", features = ["sqlcipher"] }
rust-hsluv = "0.1"
sanitize-filename = "0.4"
sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false }
@@ -73,21 +76,17 @@ sha2 = "0.10"
smallvec = "1"
strum = "0.24"
strum_macros = "0.24"
thiserror = "1"
toml = "0.7"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
fast-socks5 = "0.8"
humansize = "2"
qrcodegen = "1.7.0"
tagger = "4.3.4"
textwrap = "0.16.0"
async-channel = "1.8.0"
futures-lite = "1.12.0"
tokio-stream = { version = "0.1.11", features = ["fs"] }
thiserror = "1"
tokio-io-timeout = "1.2.0"
reqwest = { version = "0.11.14", features = ["json"] }
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
tokio-stream = { version = "0.1.11", features = ["fs"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
toml = "0.7"
trust-dns-resolver = "0.22"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
[dev-dependencies]
ansi_term = "0.12.0"

2
deltachat-ffi/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat_ffi"
version = "1.108.0"
version = "1.109.0"
description = "Deltachat FFI"
edition = "2018"
readme = "README.md"

12
deltachat-ffi/deltachat.h
View File

@@ -4443,6 +4443,18 @@ void dc_msg_set_text (dc_msg_t* msg, const char* text);
void dc_msg_set_html (dc_msg_t* msg, const char* html);
/**
* Sets the email's subject. If it's empty, a default subject
* will be used (e.g. `Message from Alice` or `Re: <last subject>`).
* This does not alter any information in the database.
*
* @memberof dc_msg_t
* @param msg The message object.
* @param subject The new subject.
*/
void dc_msg_set_subject (dc_msg_t* msg, const char* subject);
/**
* Set different sender name for a message.
* This overrides the name set by the dc_set_config()-option `displayname`.

20
deltachat-ffi/src/lib.rs
View File

@@ -2004,12 +2004,10 @@ pub unsafe extern "C" fn dc_create_contact(
let ctx = &*context;
let name = to_string_lossy(name);
block_on(async move {
Contact::create(ctx, &name, &to_string_lossy(addr))
.await
.map(|id| id.to_u32())
.unwrap_or(0)
})
block_on(Contact::create(ctx, &name, &to_string_lossy(addr)))
.log_err(ctx, "Cannot create contact")
.map(|id| id.to_u32())
.unwrap_or(0)
}
#[no_mangle]
@@ -3599,6 +3597,16 @@ pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::
ffi_msg.message.set_html(to_opt_string_lossy(html))
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_subject(msg: *mut dc_msg_t, subject: *const libc::c_char) {
if msg.is_null() {
eprintln!("ignoring careless call to dc_msg_get_subject()");
return;
}
let ffi_msg = &mut *msg;
ffi_msg.message.set_subject(to_string_lossy(subject));
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_override_sender_name(
msg: *mut dc_msg_t,

4
deltachat-jsonrpc/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-jsonrpc"
version = "1.108.0"
version = "1.109.0"
description = "DeltaChat JSON-RPC API"
edition = "2021"
default-run = "deltachat-jsonrpc-server"
@@ -28,7 +28,7 @@ sanitize-filename = "0.4"
walkdir = "2.3.2"
# optional dependencies
axum = { version = "0.6.4", optional = true, features = ["ws"] }
axum = { version = "0.5.9", optional = true, features = ["ws"] }
env_logger = { version = "0.10.0", optional = true }
[dev-dependencies]

2
deltachat-jsonrpc/typescript/package.json
View File

@@ -48,5 +48,5 @@
},
"type": "module",
"types": "dist/deltachat.d.ts",
"version": "1.108.0"
"version": "1.109.0"
}

4
deltachat-repl/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-repl"
version = "1.108.0"
version = "1.109.0"
edition = "2021"
[dependencies]
@@ -10,7 +10,7 @@ deltachat = { path = "..", features = ["internals"]}
dirs = "4"
log = "0.4.16"
pretty_env_logger = "0.4"
rusqlite = "0.27"
rusqlite = "0.28"
rustyline = "10"
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }

2
deltachat-rpc-server/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-rpc-server"
version = "1.108.0"
version = "1.109.0"
description = "DeltaChat JSON-RPC server"
edition = "2021"
readme = "README.md"

2
package.json
View File

@@ -60,5 +60,5 @@
"test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
},
"types": "node/dist/index.d.ts",
"version": "1.108.0"
"version": "1.109.0"
}

8
python/src/deltachat/account.py
View File

@@ -6,7 +6,7 @@ from array import array
from contextlib import contextmanager
from email.utils import parseaddr
from threading import Event
from typing import Any, Dict, Generator, List, Optional, Union
from typing import Any, Dict, Generator, List, Optional, Union, TYPE_CHECKING
from . import const, hookspec
from .capi import ffi, lib
@@ -22,6 +22,9 @@ from .cutil import (
from .message import Message
from .tracker import ConfigureTracker, ImexTracker
if TYPE_CHECKING:
from .events import FFIEventTracker
class MissingCredentials(ValueError):
"""Account is missing `addr` and `mail_pw` config values."""
@@ -60,6 +63,9 @@ class Account:
MissingCredentials = MissingCredentials
_logid: str
_evtracker: "FFIEventTracker"
def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None:
from .events import EventThread

4
python/src/deltachat/direct_imap.py
View File

@@ -207,14 +207,14 @@ class IdleManager:
return res
def wait_for_new_message(self, timeout=None) -> bytes:
while 1:
while True:
for item in self.check(timeout=timeout):
if b"EXISTS" in item or b"RECENT" in item:
return item
def wait_for_seen(self, timeout=None) -> int:
"""Return first message with SEEN flag from a running idle-stream."""
while 1:
while True:
for item in self.check(timeout=timeout):
if FETCH in item:
self.log(str(item))

16
python/src/deltachat/events.py
View File

@@ -108,7 +108,7 @@ class FFIEventTracker:
return ev
def iter_events(self, timeout=None, check_error=True):
while 1:
while True:
yield self.get(timeout=timeout, check_error=check_error)
def get_matching(self, event_name_regex, check_error=True, timeout=None):
@@ -119,14 +119,14 @@ class FFIEventTracker:
def get_info_contains(self, regex: str) -> FFIEvent:
rex = re.compile(regex)
while 1:
while True:
ev = self.get_matching("DC_EVENT_INFO")
if rex.search(ev.data2):
return ev
def get_info_regex_groups(self, regex, check_error=True):
rex = re.compile(regex)
while 1:
while True:
ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
m = rex.match(ev.data2)
if m is not None:
@@ -137,7 +137,7 @@ class FFIEventTracker:
This only works reliably if the connectivity doesn't change
again too quickly, otherwise we might miss it.
"""
while 1:
while True:
if self.account.get_connectivity() == connectivity:
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -146,7 +146,7 @@ class FFIEventTracker:
"""Wait until the connectivity changes to `expected_next`.
Fails the test if it changes to something else.
"""
while 1:
while True:
current = self.account.get_connectivity()
if current == expected_next:
return
@@ -156,7 +156,7 @@ class FFIEventTracker:
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
def wait_for_all_work_done(self):
while 1:
while True:
if self.account.all_work_done():
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -164,7 +164,7 @@ class FFIEventTracker:
def ensure_event_not_queued(self, event_name_regex):
__tracebackhide__ = True
rex = re.compile(f"(?:{event_name_regex}).*")
while 1:
while True:
try:
ev = self._event_queue.get(False)
except Empty:
@@ -173,7 +173,7 @@ class FFIEventTracker:
assert not rex.match(ev.name), f"event found {ev}"
def wait_securejoin_inviter_progress(self, target):
while 1:
while True:
event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
if event.data2 >= target:
print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)

32
python/src/deltachat/testplugin.py
View File

@@ -275,6 +275,8 @@ class ACSetup:
CONFIGURED = "CONFIGURED"
IDLEREADY = "IDLEREADY"
_configured_events: Queue
def __init__(self, testprocess, init_time):
self._configured_events = Queue()
self._account2state = {}
@@ -307,7 +309,7 @@ class ACSetup:
def wait_one_configured(self, account):
"""wait until this account has successfully configured."""
if self._account2state[account] == self.CONFIGURING:
while 1:
while True:
acc = self._pop_config_success()
if acc == account:
break
@@ -376,8 +378,13 @@ class ACSetup:
class ACFactory:
"""Account factory"""
init_time: float
_finalizers: List[Callable[[], None]]
_accounts: List[Account]
_acsetup: ACSetup
_preconfigured_keys: List[str]
def __init__(self, request, testprocess, tmpdir, data) -> None:
self.init_time = time.time()
@@ -429,14 +436,15 @@ class ACFactory:
assert "addr" in configdict and "mail_pw" in configdict
return configdict
def _get_cached_account(self, addr):
def _get_cached_account(self, addr) -> Optional[Account]:
if addr in self.testprocess._addr2files:
return self._getaccount(addr)
return None
def get_unconfigured_account(self, closed=False):
def get_unconfigured_account(self, closed=False) -> Account:
return self._getaccount(closed=closed)
def _getaccount(self, try_cache_addr=None, closed=False):
def _getaccount(self, try_cache_addr=None, closed=False) -> Account:
logid = f"ac{len(self._accounts) + 1}"
# we need to use fixed database basename for maybe_cache_* functions to work
path = self.tmpdir.mkdir(logid).join("dc.db")
@@ -450,10 +458,10 @@ class ACFactory:
self._accounts.append(ac)
return ac
def set_logging_default(self, logging):
def set_logging_default(self, logging) -> None:
self._logging = bool(logging)
def remove_preconfigured_keys(self):
def remove_preconfigured_keys(self) -> None:
self._preconfigured_keys = []
def _preconfigure_key(self, account, addr):
@@ -491,7 +499,7 @@ class ACFactory:
self._acsetup.init_logging(ac)
return ac
def new_online_configuring_account(self, cloned_from=None, cache=False, **kwargs):
def new_online_configuring_account(self, cloned_from=None, cache=False, **kwargs) -> Account:
if cloned_from is None:
configdict = self.get_next_liveconfig()
else:
@@ -513,7 +521,7 @@ class ACFactory:
self._acsetup.start_configure(ac)
return ac
def prepare_account_from_liveconfig(self, configdict):
def prepare_account_from_liveconfig(self, configdict) -> Account:
ac = self.get_unconfigured_account()
assert "addr" in configdict and "mail_pw" in configdict, configdict
configdict.setdefault("bcc_self", False)
@@ -523,11 +531,11 @@ class ACFactory:
self._preconfigure_key(ac, configdict["addr"])
return ac
def wait_configured(self, account):
def wait_configured(self, account) -> None:
"""Wait until the specified account has successfully completed configure."""
self._acsetup.wait_one_configured(account)
def bring_accounts_online(self):
def bring_accounts_online(self) -> None:
print("bringing accounts online")
self._acsetup.bring_online()
print("all accounts online")
@@ -630,7 +638,7 @@ class BotProcess:
def _run_stdout_thread(self) -> None:
try:
while 1:
while True:
line = self.popen.stdout.readline()
if not line:
break
@@ -651,7 +659,7 @@ class BotProcess:
for next_pattern in patterns:
print("+++FNMATCH:", next_pattern)
ignored = []
while 1:
while True:
line = self.stdout_queue.get()
if line is None:
if ignored:

2
python/src/deltachat/tracker.py
View File

@@ -85,7 +85,7 @@ class ConfigureTracker:
self._imap_finished.wait()
def wait_progress(self, data1=None):
while 1:
while True:
evdata = self._progress.get()
if data1 is None or evdata == data1:
break

3
python/tox.ini
View File

@@ -8,7 +8,7 @@ envlist =
[testenv]
commands =
pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
pytest -n6 --extra-info -v -rsXx --ignored --strict-tls {posargs: tests examples}
pip wheel . -w {toxworkdir}/wheelhouse --no-deps
setenv =
# Avoid stack overflow when Rust core is built without optimizations.
@@ -21,7 +21,6 @@ passenv =
RUSTC_WRAPPER
deps =
pytest
pytest-rerunfailures
pytest-timeout
pytest-xdist
pdbpp

36
src/accounts.rs
View File

@@ -1,7 +1,6 @@
//! # Account manager module.
use std::collections::BTreeMap;
use std::future::Future;
use std::path::{Path, PathBuf};
use anyhow::{ensure, Context as _, Result};
@@ -151,7 +150,7 @@ impl Accounts {
if let Some(cfg) = self.config.get_account(id) {
let account_path = self.dir.join(cfg.dir);
try_many_times(|| fs::remove_dir_all(&account_path))
fs::remove_dir_all(&account_path)
.await
.context("failed to remove account data")?;
}
@@ -187,10 +186,10 @@ impl Accounts {
fs::create_dir_all(self.dir.join(&account_config.dir))
.await
.context("failed to create dir")?;
try_many_times(|| fs::rename(&dbfile, &new_dbfile))
fs::rename(&dbfile, &new_dbfile)
.await
.context("failed to rename dbfile")?;
try_many_times(|| fs::rename(&blobdir, &new_blobdir))
fs::rename(&blobdir, &new_blobdir)
.await
.context("failed to rename blobdir")?;
if walfile.exists() {
@@ -215,7 +214,7 @@ impl Accounts {
}
Err(err) => {
let account_path = std::path::PathBuf::from(&account_config.dir);
try_many_times(|| fs::remove_dir_all(&account_path))
fs::remove_dir_all(&account_path)
.await
.context("failed to remove account data")?;
self.config.remove_account(account_config.id).await?;
@@ -472,33 +471,6 @@ impl Config {
}
}
/// Spend up to 1 minute trying to do the operation.
///
/// Files may remain locked up to 30 seconds due to r2d2 bug:
/// <https://github.com/sfackler/r2d2/issues/99>
async fn try_many_times<F, Fut, T>(f: F) -> std::result::Result<(), T>
where
F: Fn() -> Fut,
Fut: Future<Output = std::result::Result<(), T>>,
{
let mut counter = 0;
loop {
counter += 1;
if let Err(err) = f().await {
if counter > 60 {
return Err(err);
}
// Wait 1 second and try again.
tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
} else {
break;
}
}
Ok(())
}
/// Configuration of a single account.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
struct AccountConfig {

2
src/chat.rs
View File

@@ -3397,7 +3397,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
msg.param.remove(Param::WebxdcSummaryTimestamp);
msg.in_reply_to = None;
// do not leak data as group names; a default subject is generated by mimfactory
// do not leak data as group names; a default subject is generated by mimefactory
msg.subject = "".to_string();
let new_msg_id: MsgId;

235
src/contact.rs
View File

@@ -13,6 +13,7 @@ use async_channel::{self as channel, Receiver, Sender};
use deltachat_derive::{FromSql, ToSql};
use once_cell::sync::Lazy;
use regex::Regex;
use rusqlite::OptionalExtension;
use serde::{Deserialize, Serialize};
use tokio::task;
use tokio::time::{timeout, Duration};
@@ -520,8 +521,6 @@ impl Contact {
/// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
///
/// Returns the contact_id and a `Modifier` value indicating if a modification occurred.
///
/// Returns None if the contact with such address cannot exist.
pub(crate) async fn add_or_lookup(
context: &Context,
name: &str,
@@ -566,14 +565,12 @@ impl Contact {
);
let mut update_addr = false;
let mut row_id = 0;
if let Some((id, row_name, row_addr, row_origin, row_authname)) = context
.sql
.query_row_optional(
"SELECT id, name, addr, origin, authname \
FROM contacts WHERE addr=? COLLATE NOCASE;",
paramsv![addr.to_string()],
let row_id = context.sql.transaction(|transaction| {
let row = transaction.query_row(
"SELECT id, name, addr, origin, authname
FROM contacts WHERE addr=? COLLATE NOCASE",
[addr.to_string()],
|row| {
let row_id: isize = row.get(0)?;
let row_name: String = row.get(1)?;
@@ -582,120 +579,130 @@ impl Contact {
let row_authname: String = row.get(4)?;
Ok((row_id, row_name, row_addr, row_origin, row_authname))
},
)
.await?
{
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
}).optional()?;
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
let row_id;
if let Some((id, row_name, row_addr, row_origin, row_authname)) = row {
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
context
.sql
.execute(
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
if update_authname {
name.to_string()
} else {
row_authname
},
row_id
],
)
.await
.ok();
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<i32> = context.sql.query_get_value(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
paramsv![Chattype::Single, isize::try_from(row_id)?]
).await?;
if let Some(chat_id) = chat_id {
let contact = Contact::get_by_id(context, ContactId::new(row_id)).await?;
let chat_name = contact.get_display_name();
match context
.sql
.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?3",
paramsv![chat_name, chat_id, chat_name],
)
.await
{
Err(err) => warn!(context, "Can't update chat name: {}", err),
Ok(count) => {
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(ChatId::new(
chat_id.try_into()?,
)));
}
transaction
.execute(
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
if update_authname {
name.to_string()
} else {
row_authname
},
row_id
],
)?;
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<ChatId> = transaction.query_row(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
params![Chattype::Single, isize::try_from(row_id)?],
|row| {
let chat_id: ChatId = row.get(0)?;
Ok(chat_id)
}
).optional()?;
if let Some(chat_id) = chat_id {
let contact_id = ContactId::new(row_id);
let (addr, name, authname) =
transaction.query_row(
"SELECT addr, name, authname
FROM contacts
WHERE id=?",
params![contact_id],
|row| {
let addr: String = row.get(0)?;
let name: String = row.get(1)?;
let authname: String = row.get(2)?;
Ok((addr, name, authname))
})?;
let chat_name = if !name.is_empty() {
name
} else if !authname.is_empty() {
authname
} else {
addr
};
let count = transaction.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?1",
params![chat_name, chat_id])?;
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(chat_id));
}
}
}
sth_modified = Modifier::Modified;
}
sth_modified = Modifier::Modified;
}
} else {
let update_name = manual;
let update_authname = !manual;
if let Ok(new_row_id) = context
.sql
.insert(
"INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
paramsv![
if update_name {
name.to_string()
} else {
"".to_string()
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
],
)
.await
{
row_id = u32::try_from(new_row_id)?;
sth_modified = Modifier::Created;
info!(context, "added contact id={} addr={}", row_id, &addr);
} else {
error!(context, "Cannot add contact.");
let update_name = manual;
let update_authname = !manual;
transaction
.execute(
"INSERT INTO contacts (name, addr, origin, authname)
VALUES (?, ?, ?, ?);",
params![
if update_name {
name.to_string()
} else {
"".to_string()
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
],
)?;
sth_modified = Modifier::Created;
row_id = u32::try_from(transaction.last_insert_rowid())?;
info!(context, "added contact id={} addr={}", row_id, &addr);
}
}
Ok(row_id)
}).await?;
Ok((ContactId::new(row_id), sth_modified))
}

42
src/imex.rs
View File

@@ -512,6 +512,9 @@ fn get_next_backup_path(folder: &Path, backup_time: i64) -> Result<(PathBuf, Pat
bail!("could not create backup file, disk full?");
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
// get a fine backup file name (the name includes the date so that multiple backup instances are possible)
let now = time();
@@ -737,9 +740,9 @@ where
Ok(())
}
/// Exports the database to *file*, encrypted using *passphrase*.
/// Exports the database to *dest*, encrypted using *passphrase*.
///
/// The directory of *file* must already exist, if *file* itself exists it will be
/// The directory of *dest* must already exist, if *dest* itself exists it will be
/// overwritten.
///
/// This also verifies that IO is not running during the export.
@@ -750,20 +753,31 @@ async fn export_database(context: &Context, dest: &Path, passphrase: String) ->
);
let now = time().try_into().context("32-bit UNIX time overflow")?;
// TODO: Maybe introduce camino crate for UTF-8 paths where we need them.
let dest = dest
.to_str()
.with_context(|| format!("path {} is not valid unicode", dest.display()))?;
context.sql.set_raw_config_int("backup_time", now).await?;
sql::housekeeping(context).await.ok_or_log(context);
context
.sql
.execute("VACUUM;", paramsv![])
.await
.map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
.ok();
context
.sql
.export(dest, passphrase)
.await
.with_context(|| format!("failed to backup database to {}", dest.display()))?;
Ok(())
let conn = context.sql.get_conn().await?;
tokio::task::block_in_place(move || {
conn.execute("VACUUM;", params![])
.map_err(|err| warn!(context, "Vacuum failed, exporting anyway {err}"))
.ok();
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![dest, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
}
#[cfg(test)]

8
src/message.rs
View File

@@ -265,6 +265,8 @@ pub struct Message {
pub(crate) text: Option<String>,
/// Message subject.
///
/// If empty, a default subject will be generated when sending.
pub(crate) subject: String,
/// `Message-ID` header value.
@@ -795,6 +797,12 @@ impl Message {
self.text = text;
}
/// Sets the email's subject. If it's empty, a default subject
/// will be used (e.g. `Message from Alice` or `Re: <last subject>`).
pub fn set_subject(&mut self, subject: String) {
self.subject = subject;
}
/// Sets the file associated with a message.
///
/// This function does not use the file or check if it exists,

16
src/mimefactory.rs
View File

@@ -1612,6 +1612,22 @@ mod tests {
assert_eq!(maybe_encode_words("äöü"), "=?utf-8?b?w6TDtsO8?=");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_manually_set_subject() -> Result<()> {
let t = TestContext::new_alice().await;
let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
let mut msg = Message::new(Viewtype::Text);
msg.set_subject("Subjeeeeect".to_string());
let sent_msg = t.send_msg(chat.id, &mut msg).await;
let payload = sent_msg.payload();
assert_eq!(payload.match_indices("Subject: Subjeeeeect").count(), 1);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_subject_from_mua() {
// 1.: Receive a mail from an MUA

134
src/sql.rs
View File

@@ -4,10 +4,9 @@ use std::collections::{HashMap, HashSet};
use std::convert::TryFrom;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use anyhow::{bail, Context as _, Result};
use rusqlite::{config::DbConfig, Connection, OpenFlags};
use rusqlite::{self, config::DbConfig, Connection, OpenFlags, TransactionBehavior};
use tokio::sync::RwLock;
use crate::blob::BlobObject;
@@ -48,6 +47,9 @@ pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator<Item = &d
}
mod migrations;
mod pool;
use pool::{Pool, PooledConnection};
/// A wrapper around the underlying Sqlite3 object.
#[derive(Debug)]
@@ -56,7 +58,7 @@ pub struct Sql {
pub(crate) dbfile: PathBuf,
/// SQL connection pool.
pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
pool: RwLock<Option<Pool>>,
/// None if the database is not open, true if it is open with passphrase and false if it is
/// open without a passphrase.
@@ -122,31 +124,6 @@ impl Sql {
// drop closes the connection
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
pub(crate) async fn export(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
.to_str()
.with_context(|| format!("path {path:?} is not valid unicode"))?;
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
}
/// Imports the database from a separate file with the given passphrase.
pub(crate) async fn import(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
@@ -192,50 +169,15 @@ impl Sql {
})
}
fn new_pool(
dbfile: &Path,
passphrase: String,
) -> Result<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>> {
let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE);
open_flags.insert(OpenFlags::SQLITE_OPEN_CREATE);
/// Creates a new connection pool.
fn new_pool(dbfile: &Path, passphrase: String) -> Result<Pool> {
let mut connections = Vec::new();
for _ in 0..3 {
let connection = new_connection(dbfile, &passphrase)?;
connections.push(connection);
}
// this actually creates min_idle database handles just now.
// therefore, with_init() must not try to modify the database as otherwise
// we easily get busy-errors (eg. table-creation, journal_mode etc. should be done on only one handle)
let mgr = r2d2_sqlite::SqliteConnectionManager::file(dbfile)
.with_flags(open_flags)
.with_init(move |c| {
c.execute_batch(&format!(
"PRAGMA cipher_memory_security = OFF; -- Too slow on Android
PRAGMA secure_delete=on;
PRAGMA busy_timeout = {};
PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
PRAGMA foreign_keys=on;
",
Duration::from_secs(10).as_millis()
))?;
c.pragma_update(None, "key", passphrase.clone())?;
// Try to enable auto_vacuum. This will only be
// applied if the database is new or after successful
// VACUUM, which usually happens before backup export.
// When auto_vacuum is INCREMENTAL, it is possible to
// use PRAGMA incremental_vacuum to return unused
// database pages to the filesystem.
c.pragma_update(None, "auto_vacuum", "INCREMENTAL".to_string())?;
c.pragma_update(None, "journal_mode", "WAL".to_string())?;
// Default synchronous=FULL is much slower. NORMAL is sufficient for WAL mode.
c.pragma_update(None, "synchronous", "NORMAL".to_string())?;
Ok(())
});
let pool = r2d2::Pool::builder()
.min_idle(Some(2))
.max_size(10)
.connection_timeout(Duration::from_secs(60))
.build(mgr)
.context("Can't build SQL connection pool")?;
let pool = Pool::new(connections);
Ok(pool)
}
@@ -393,12 +335,10 @@ impl Sql {
}
/// Allocates a connection from the connection pool and returns it.
pub async fn get_conn(
&self,
) -> Result<r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>> {
pub(crate) async fn get_conn(&self) -> Result<PooledConnection> {
let lock = self.pool.read().await;
let pool = lock.as_ref().context("no SQL connection")?;
let conn = pool.get()?;
let conn = pool.get().await?;
Ok(conn)
}
@@ -437,6 +377,12 @@ impl Sql {
///
/// If the function returns an error, the transaction will be rolled back. If it does not return an
/// error, the transaction will be committed.
///
/// Transactions started use IMMEDIATE behavior
/// rather than default DEFERRED behavior
/// to avoid "database is busy" errors
/// which may happen when DEFERRED transaction
/// is attempted to be promoted to a write transaction.
pub async fn transaction<G, H>(&self, callback: G) -> Result<H>
where
H: Send + 'static,
@@ -444,7 +390,7 @@ impl Sql {
{
let mut conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let mut transaction = conn.transaction()?;
let mut transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let ret = callback(&mut transaction);
match ret {
@@ -642,6 +588,42 @@ impl Sql {
}
}
/// Creates a new SQLite connection.
///
/// `path` is the database path.
///
/// `passphrase` is the SQLCipher database passphrase.
/// Empty string if database is not encrypted.
fn new_connection(path: &Path, passphrase: &str) -> Result<Connection> {
let mut flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE);
flags.insert(OpenFlags::SQLITE_OPEN_CREATE);
let conn = Connection::open_with_flags(path, flags)?;
conn.execute_batch(
"PRAGMA cipher_memory_security = OFF; -- Too slow on Android
PRAGMA secure_delete=on;
PRAGMA busy_timeout = 60000; -- 60 seconds
PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
PRAGMA foreign_keys=on;
",
)?;
conn.pragma_update(None, "key", passphrase)?;
// Try to enable auto_vacuum. This will only be
// applied if the database is new or after successful
// VACUUM, which usually happens before backup export.
// When auto_vacuum is INCREMENTAL, it is possible to
// use PRAGMA incremental_vacuum to return unused
// database pages to the filesystem.
conn.pragma_update(None, "auto_vacuum", "INCREMENTAL".to_string())?;
conn.pragma_update(None, "journal_mode", "WAL".to_string())?;
// Default synchronous=FULL is much slower. NORMAL is sufficient for WAL mode.
conn.pragma_update(None, "synchronous", "NORMAL".to_string())?;
Ok(conn)
}
/// Cleanup the account to restore some storage and optimize the database.
pub async fn housekeeping(context: &Context) -> Result<()> {
if let Err(err) = remove_unused_files(context).await {

102
src/sql/pool.rs Normal file
View File

@@ -0,0 +1,102 @@
//! Connection pool.
use std::ops::{Deref, DerefMut};
use std::sync::{Arc, Weak};
use anyhow::{Context, Result};
use crossbeam_queue::ArrayQueue;
use rusqlite::Connection;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
/// Inner connection pool.
#[derive(Debug)]
struct InnerPool {
/// Available connections.
connections: ArrayQueue<Connection>,
/// Counts the number of available connections.
semaphore: Arc<Semaphore>,
}
impl InnerPool {
/// Puts a connection into the pool.
///
/// The connection could be new or returned back.
fn put(&self, connection: Connection) {
self.connections.force_push(connection);
}
}
/// Pooled connection.
pub struct PooledConnection {
/// Weak reference to the pool used to return the connection back.
pool: Weak<InnerPool>,
/// Only `None` right after moving the connection back to the pool.
conn: Option<Connection>,
/// Semaphore permit, dropped after returning the connection to the pool.
_permit: OwnedSemaphorePermit,
}
impl Drop for PooledConnection {
fn drop(&mut self) {
// Put the connection back unless the pool is already dropped.
if let Some(pool) = self.pool.upgrade() {
if let Some(conn) = self.conn.take() {
pool.put(conn);
}
}
}
}
impl Deref for PooledConnection {
type Target = Connection;
fn deref(&self) -> &Connection {
self.conn.as_ref().unwrap()
}
}
impl DerefMut for PooledConnection {
fn deref_mut(&mut self) -> &mut Connection {
self.conn.as_mut().unwrap()
}
}
/// Connection pool.
#[derive(Clone, Debug)]
pub struct Pool {
/// Reference to the actual connection pool.
inner: Arc<InnerPool>,
}
impl Pool {
/// Creates a new connection pool.
pub fn new(connections: Vec<Connection>) -> Self {
let inner = Arc::new(InnerPool {
connections: ArrayQueue::new(connections.len()),
semaphore: Arc::new(Semaphore::new(connections.len())),
});
for connection in connections {
inner.connections.force_push(connection);
}
Pool { inner }
}
/// Retrieves a connection from the pool.
pub async fn get(&self) -> Result<PooledConnection> {
let permit = self.inner.semaphore.clone().acquire_owned().await?;
let conn = self
.inner
.connections
.pop()
.context("got a permit when there are no connections in the pool")?;
let conn = PooledConnection {
pool: Arc::downgrade(&self.inner),
conn: Some(conn),
_permit: permit,
};
Ok(conn)
}
}