Compare commits

..

2 Commits

Author SHA1 Message Date
link2xt
90087bde39 sql: release memory when returning connection to the pool 2023-02-17 22:09:24 +00:00
link2xt
bfd3c1763d Replace r2d2 with an own connection pool
New connection pool does not use threads
and does not remove idle connections
or create new connections at runtime.
2023-02-17 21:12:17 +00:00
96 changed files with 2028 additions and 2842 deletions

28
.github/mergeable.yml vendored
View File

@@ -5,22 +5,22 @@ mergeable:
validate:
- do: or
validate:
- do: description
must_include:
regex: "#skip-changelog"
- do: and
validate:
- do: dependent
changed:
file: "src/**"
required: ["CHANGELOG.md"]
- do: dependent
changed:
file: "deltachat-ffi/src/**"
required: ["CHANGELOG.md"]
- do: description
must_include:
regex: '#skip-changelog'
- do: and
validate:
- do: dependent
changed:
file: 'src/**'
required: ['CHANGELOG.md']
- do: dependent
changed:
file: 'deltachat-ffi/src/**'
required: ['CHANGELOG.md']
fail:
- do: checks
status: "action_required"
status: 'action_required'
payload:
title: Changelog might need an update
summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

View File

@@ -1,11 +1,5 @@
name: Rust CI
# Cancel previously started workflow runs
# when the branch is updated.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
push:
@@ -14,32 +8,30 @@ on:
env:
RUSTFLAGS: -Dwarnings
jobs:
lint:
name: Rustfmt and Clippy
fmt:
name: Rustfmt
runs-on: ubuntu-latest
env:
RUSTUP_TOOLCHAIN: 1.67.1
steps:
- uses: actions/checkout@v3
- name: Install rustfmt and clippy
run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
- run: cargo fmt --all -- --check
run_clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install clippy
run: rustup toolchain install 1.67.1 --component clippy
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Run rustfmt
run: cargo fmt --all -- --check
- name: Run clippy
env:
RUSTUP_TOOLCHAIN: 1.67.1
run: scripts/clippy.sh
foo:
name: bar
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: EmbarkStudios/cargo-deny-action@v1
arguments: --all-features --workspace
docs:
name: Rust doc comments
runs-on: ubuntu-latest
@@ -77,73 +69,73 @@ jobs:
python: 3.7
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@master
- uses: actions/checkout@master
- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install --profile minimal ${{ matrix.rust }}
- run: rustup override set ${{ matrix.rust }}
- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install ${{ matrix.rust }}
- run: rustup override set ${{ matrix.rust }}
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Check
run: cargo check --workspace --bins --examples --tests --benches
- name: Check
run: cargo check --workspace --bins --examples --tests --benches
- name: Tests
run: cargo test --workspace
- name: Tests
run: cargo test --workspace
- name: Test cargo vendor
run: cargo vendor
- name: Test cargo vendor
run: cargo vendor
- name: Install python
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install python
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install tox
if: ${{ matrix.python }}
run: pip install tox
- name: Install tox
if: ${{ matrix.python }}
run: pip install tox
- name: Build C library
if: ${{ matrix.python }}
run: cargo build -p deltachat_ffi --features jsonrpc
- name: Build C library
if: ${{ matrix.python }}
run: cargo build -p deltachat_ffi --features jsonrpc
- name: Run python tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e lint,mypy,doc,py3
- name: Run python tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e lint,mypy,doc,py3
- name: Build deltachat-rpc-server
if: ${{ matrix.python }}
run: cargo build -p deltachat-rpc-server
- name: Build deltachat-rpc-server
if: ${{ matrix.python }}
run: cargo build -p deltachat-rpc-server
- name: Add deltachat-rpc-server to path
if: ${{ matrix.python }}
run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
- name: Add deltachat-rpc-server to path
if: ${{ matrix.python }}
run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
- name: Run deltachat-rpc-client tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
working-directory: deltachat-rpc-client
run: tox -e py3,lint
- name: Run deltachat-rpc-client tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
working-directory: deltachat-rpc-client
run: tox -e py3,lint
- name: Install pypy
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: "pypy${{ matrix.python }}"
- name: Install pypy
if: ${{ matrix.python }}
uses: actions/setup-python@v4
with:
python-version: 'pypy${{ matrix.python }}'
- name: Run pypy tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e pypy3
- name: Run pypy tests
if: ${{ matrix.python }}
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
DCC_RS_TARGET: debug
DCC_RS_DEV: ${{ github.workspace }}
working-directory: python
run: tox -e pypy3

View File

@@ -1,129 +0,0 @@
# Manually triggered action to build deltachat-rpc-server binaries.
name: Build deltachat-rpc-server binaries
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_dispatch:
jobs:
# Build a version statically linked against musl libc
# to avoid problems with glibc version incompatibility.
build_static_linux:
name: Build deltachat-rpc-server for Linux
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Setup rust target
run: rustup target add x86_64-unknown-linux-musl
- name: Install musl-gcc
run: sudo apt install musl-tools
- name: Build
env:
RUSTFLAGS: "-C link-arg=-s"
run: cargo build --release --target x86_64-unknown-linux-musl -p deltachat-rpc-server --features vendored
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-x86_64
path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server
if-no-files-found: error
build_aarch64_linux:
name: Cross-compile deltachat-rpc-server for aarch64 Linux
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Build
run: sh scripts/aarch64-unknown-linux-musl.sh
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-aarch64
path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server
if-no-files-found: error
build_android:
name: Cross-compile deltachat-rpc-server for Android (armeabi-v7a, arm64-v8a, x86 and x86_64)
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: nttld/setup-ndk@v1
id: setup-ndk
with:
ndk-version: r21d
- name: Build
env:
ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
run: sh scripts/android-rpc-server.sh
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-android-armv7
path: target/armv7-linux-androideabi/release/deltachat-rpc-server
if-no-files-found: error
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-android-aarch64
path: target/aarch64-linux-android/release/deltachat-rpc-server
if-no-files-found: error
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-android-i686
path: target/i686-linux-android/release/deltachat-rpc-server
if-no-files-found: error
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-android-x86_64
path: target/x86_64-linux-android/release/deltachat-rpc-server
if-no-files-found: error
build_windows:
name: Build deltachat-rpc-server for Windows
strategy:
fail-fast: false
matrix:
include:
- os: windows-latest
artifact: win32.exe
path: deltachat-rpc-server.exe
target: i686-pc-windows-msvc
- os: windows-latest
artifact: win64.exe
path: deltachat-rpc-server.exe
target: x86_64-pc-windows-msvc
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup rust target
run: rustup target add ${{ matrix.target }}
- name: Build
run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: deltachat-rpc-server-${{ matrix.artifact }}
path: target/${{ matrix.target}}/release/${{ matrix.path }}
if-no-files-found: error

View File

@@ -1,15 +1,16 @@
name: "jsonrpc js client build"
name: 'jsonrpc js client build'
on:
pull_request:
push:
tags:
- "*"
- "!py-*"
- '*'
- '!py-*'
jobs:
pack-module:
name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat"
runs-on: ubuntu-20.04
name: 'Package @deltachat/jsonrpc-client and upload to download.delta.chat'
runs-on: ubuntu-18.04
steps:
- name: Install tree
run: sudo apt install tree
@@ -17,7 +18,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16"
node-version: '16'
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
@@ -68,9 +69,9 @@ jobs:
if: steps.upload-preview.outcome == 'success'
run: node ./node/scripts/postLinksToDetails.js
env:
URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
# Upload to download.delta.chat/node/
- name: Upload deltachat-jsonrpc-client build to download.delta.chat/node/
if: ${{ steps.tag.outputs.tag }}

View File

@@ -35,10 +35,6 @@ jobs:
npm run test
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
- name: make sure websocket server version still builds
run: |
cd deltachat-jsonrpc
cargo build --bin deltachat-jsonrpc-server --features webserver
- name: Run linter
run: |
cd deltachat-jsonrpc/typescript

View File

@@ -7,25 +7,26 @@ on:
jobs:
delete:
runs-on: ubuntu-latest
steps:
- name: Get Pullrequest ID
id: getid
run: |
export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
- name: Renaming
run: |
# create empty file to copy it over the outdated deliverable on download.delta.chat
echo "This preview build is outdated and has been removed." > empty
cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
- name: Replace builds with dummy files
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.SSH_KEY }}
host: "download.delta.chat"
port: 22
local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
remote: "/var/www/html/download/node/preview/"
- name: Get Pullrequest ID
id: getid
run: |
export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
- name: Renaming
run: |
# create empty file to copy it over the outdated deliverable on download.delta.chat
echo "This preview build is outdated and has been removed." > empty
cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
- name: Replace builds with dummy files
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.SSH_KEY }}
host: "download.delta.chat"
port: 22
local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
remote: "/var/www/html/download/node/preview/"

View File

@@ -9,26 +9,26 @@ jobs:
generate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Use Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: Use Node.js 16.x
uses: actions/setup-node@v3
with:
node-version: 16.x
- name: npm install and generate documentation
run: |
cd node
npm i --ignore-scripts
npx typedoc
mv docs js
- name: npm install and generate documentation
run: |
cd node
npm i --ignore-scripts
npx typedoc
mv docs js
- name: Upload
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.KEY }}
host: "delta.chat"
port: 22
local: "node/js"
remote: "/var/www/html/"
- name: Upload
uses: horochx/deploy-via-scp@v1.0.1
with:
user: ${{ secrets.USERNAME }}
key: ${{ secrets.KEY }}
host: "delta.chat"
port: 22
local: "node/js"
remote: "/var/www/html/"

View File

@@ -1,10 +1,11 @@
name: "node.js build"
name: 'node.js build'
on:
pull_request:
push:
tags:
- "*"
- "!py-*"
- '*'
- '!py-*'
jobs:
prebuild:
@@ -12,13 +13,13 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
os: [ubuntu-18.04, macos-latest, windows-latest]
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16"
node-version: '16'
- name: System info
run: |
rustc -vV
@@ -65,7 +66,7 @@ jobs:
pack-module:
needs: prebuild
name: Package deltachat-node and upload to download.delta.chat
runs-on: ubuntu-20.04
runs-on: ubuntu-18.04
steps:
- name: Install tree
run: sudo apt install tree
@@ -73,7 +74,7 @@ jobs:
uses: actions/checkout@v3
- uses: actions/setup-node@v2
with:
node-version: "16"
node-version: '16'
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
@@ -99,7 +100,7 @@ jobs:
- name: Download Ubuntu prebuild
uses: actions/download-artifact@v1
with:
name: ubuntu-20.04
name: ubuntu-18.04
- name: Download macOS prebuild
uses: actions/download-artifact@v1
with:
@@ -111,11 +112,11 @@ jobs:
- shell: bash
run: |
mkdir node/prebuilds
tar -xvzf ubuntu-20.04/ubuntu-20.04.tar.gz -C node/prebuilds
tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C node/prebuilds
tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
tree node/prebuilds
rm -rf ubuntu-20.04 macos-latest windows-latest
rm -rf ubuntu-18.04 macos-latest windows-latest
- name: Install dependencies without running scripts
run: |
npm install --ignore-scripts
@@ -151,8 +152,8 @@ jobs:
if: steps.upload-preview.outcome == 'success'
run: node ./node/scripts/postLinksToDetails.js
env:
URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Upload to download.delta.chat/node/
- name: Upload deltachat-node build to download.delta.chat/node/
if: ${{ steps.tag.outputs.tag }}

View File

@@ -1,16 +1,11 @@
name: "node.js tests"
# Cancel previously started workflow runs
# when the branch is updated.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
name: 'node.js tests'
on:
pull_request:
push:
branches:
- master
- staging
- trying
jobs:
tests:
@@ -18,13 +13,13 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
os: [ubuntu-18.04, macos-latest, windows-latest]
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16"
node-version: '16'
- name: System info
run: |
rustc -vV
@@ -64,7 +59,7 @@ jobs:
npm run test
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"
NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
- name: Run tests on Windows, except lint
timeout-minutes: 10
if: runner.os == 'Windows'
@@ -73,4 +68,4 @@ jobs:
npm run test:mocha
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"
NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'

View File

@@ -11,13 +11,13 @@ jobs:
name: Build REPL example
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Build
run: cargo build -p deltachat-repl --features vendored
- name: Build
run: cargo build -p deltachat-repl --features vendored
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: repl.exe
path: "target/debug/deltachat-repl.exe"
- name: Upload binary
uses: actions/upload-artifact@v3
with:
name: repl.exe
path: 'target/debug/deltachat-repl.exe'

View File

@@ -8,18 +8,20 @@ on:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat --no-deps
- name: Upload to rs.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/rs/"
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat --no-deps
- name: Upload to rs.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/rs/"

View File

@@ -8,18 +8,20 @@ on:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat_ffi --no-deps
- name: Upload to cffi.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/cffi/"
- uses: actions/checkout@v3
- name: Build the documentation with cargo
run: |
cargo doc --package deltachat_ffi --no-deps
- name: Upload to cffi.delta.chat
uses: up9cloud/action-rsync@v1.3
env:
USER: ${{ secrets.USERNAME }}
KEY: ${{ secrets.KEY }}
HOST: "delta.chat"
SOURCE: "target/doc"
TARGET: "/var/www/html/cffi/"

View File

@@ -2,57 +2,21 @@
## Unreleased
### Changes
- Make smeared timestamp generation non-async. #4075
- Set minimum TLS version to 1.2. #4096
### Fixes
- Do not block async task executor while decrypting the messages. #4079
### API-Changes
## 1.110.0
### Changes
- use transaction in `Contact::add_or_lookup()` #4059
- Organize the connection pool as a stack rather than a queue to ensure that
connection page cache is reused more often.
This speeds up tests by 28%, real usage will have lower speedup. #4065
- Use transaction in `update_blocked_mailinglist_contacts`. #4058
- Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055
- Updated provider database.
- Disable DKIM-Checks again #4076
- Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089
- mimeparser: handle headers from the signed part of unencrypted signed message #4013
### Fixes
- Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063
- Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted.
Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972
- Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067
- Update timestamps in `param` columns with transactions. #4083
## 1.109.0
### Changes
- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
- python: mark bindings as supporting typing according to PEP 561 #4045
- retry filesystem operations during account migration #4043
- replace `r2d2` and `r2d2_sqlite` dependencies with an own connection pool #4050 #4053
### Fixes
- deltachat-rpc-server: do not block stdin while processing the request. #4041
deltachat-rpc-server now reads the next request as soon as previous request handler is spawned.
- Enable `auto_vacuum` on all SQL connections. #2955
- Replace `r2d2` connection pool with an own implementation. #4050 #4053 #4043 #4061
This change improves reliability
by closing all database connections immediately when the context is closed.
- enable `auto_vacuum` on all SQL connections #2955
### API-Changes
- Remove `MimeMessage::from_bytes()` public interface. #4033
- BREAKING Types: jsonrpc: `get_messages` now returns a map with `MessageLoadResult` instead of failing completely if one of the requested messages could not be loaded. #4038
- Add `dc_msg_set_subject()`. C-FFI #4057
- Mark python bindings as supporting typing according to PEP 561 #4045
## 1.108.0
@@ -63,7 +27,6 @@
- Prefer TLS over STARTTLS during autoconfiguration #4021
- Use SOCKS5 configuration for HTTP requests #4017
- Show non-deltachat emails by default for new installations #4019
- Re-enabled SMTP pipelining after disabling it in #4006
### Fixes
- Fix Securejoin for multiple devices on a joining side #3982
@@ -451,7 +414,7 @@
- Auto accept contact requests if `Config::Bot` is set for a client #3567
- Don't prepend the subject to chat messages in mailinglists
- fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585
- Reject webxdc-updates from contacts who are not group members #3568
- Reject webxcd-updates from contacts who are not group members #3568
## 1.93.0

128
Cargo.lock generated
View File

@@ -239,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
dependencies = [
"async-trait",
"axum-core",
"axum-core 0.2.9",
"base64 0.13.1",
"bitflags",
"bytes",
@@ -248,7 +248,7 @@ dependencies = [
"http-body",
"hyper",
"itoa",
"matchit",
"matchit 0.5.0",
"memchr",
"mime",
"percent-encoding",
@@ -259,7 +259,43 @@ dependencies = [
"sha-1",
"sync_wrapper",
"tokio",
"tokio-tungstenite",
"tokio-tungstenite 0.17.2",
"tower",
"tower-http",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc"
dependencies = [
"async-trait",
"axum-core 0.3.2",
"base64 0.20.0",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit 0.7.0",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sha1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.18.0",
"tower",
"tower-http",
"tower-layer",
@@ -282,6 +318,23 @@ dependencies = [
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "backtrace"
version = "0.3.67"
@@ -315,6 +368,12 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
[[package]]
name = "base64"
version = "0.21.0"
@@ -834,7 +893,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
[[package]]
name = "deltachat"
version = "1.110.0"
version = "1.108.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -864,6 +923,7 @@ dependencies = [
"libc",
"log",
"mailparse",
"native-tls",
"num-derive",
"num-traits",
"num_cpus",
@@ -905,11 +965,11 @@ dependencies = [
[[package]]
name = "deltachat-jsonrpc"
version = "1.110.0"
version = "1.108.0"
dependencies = [
"anyhow",
"async-channel",
"axum",
"axum 0.6.4",
"deltachat",
"env_logger 0.10.0",
"futures",
@@ -927,7 +987,7 @@ dependencies = [
[[package]]
name = "deltachat-repl"
version = "1.110.0"
version = "1.108.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -942,7 +1002,7 @@ dependencies = [
[[package]]
name = "deltachat-rpc-server"
version = "1.110.0"
version = "1.108.0"
dependencies = [
"anyhow",
"deltachat-jsonrpc",
@@ -965,7 +1025,7 @@ dependencies = [
[[package]]
name = "deltachat_ffi"
version = "1.110.0"
version = "1.108.0"
dependencies = [
"anyhow",
"deltachat",
@@ -2098,6 +2158,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
[[package]]
name = "matchit"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
[[package]]
name = "md-5"
version = "0.10.5"
@@ -3154,6 +3220,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "184c643044780f7ceb59104cef98a5a6f12cb2288a7bc701ab93a362b49fd47d"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.1"
@@ -3563,7 +3638,19 @@ dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite",
"tungstenite 0.17.3",
]
[[package]]
name = "tokio-tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.18.0",
]
[[package]]
@@ -3773,6 +3860,25 @@ dependencies = [
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788"
dependencies = [
"base64 0.13.1",
"byteorder",
"bytes",
"http",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror",
"url",
"utf-8",
]
[[package]]
name = "twofish"
version = "0.7.1"
@@ -4233,7 +4339,7 @@ dependencies = [
"async-channel",
"async-mutex",
"async-trait",
"axum",
"axum 0.5.17",
"futures",
"futures-util",
"log",

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat"
version = "1.110.0"
version = "1.108.0"
edition = "2021"
license = "MPL-2.0"
rust-version = "1.63"
@@ -29,11 +29,12 @@ format-flowed = { path = "./format-flowed" }
ratelimit = { path = "./deltachat-ratelimit" }
anyhow = "1"
async-channel = "1.8.0"
async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
async-smtp = { version = "0.8", default-features = false, features = ["runtime-tokio"] }
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
trust-dns-resolver = "0.22"
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
backtrace = "0.3"
base64 = "0.21"
bitflags = "1.3"
@@ -41,30 +42,26 @@ chrono = { version = "0.4", default-features=false, features = ["clock", "std"]
email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
escaper = "0.1"
fast-socks5 = "0.8"
futures = "0.3"
futures-lite = "1.12.0"
hex = "0.4.0"
humansize = "2"
image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
kamadak-exif = "0.5"
lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
libc = "0.2"
mailparse = "0.14"
native-tls = "0.2"
num_cpus = "1.15"
num-derive = "0.3"
num-traits = "0.2"
once_cell = "1.17.0"
percent-encoding = "2.2"
parking_lot = "0.12"
percent-encoding = "2.2"
pgp = { version = "0.9", default-features = false }
pretty_env_logger = { version = "0.4", optional = true }
qrcodegen = "1.7.0"
quick-xml = "0.27"
rand = "0.8"
regex = "1.7"
reqwest = { version = "0.11.14", features = ["json"] }
rusqlite = { version = "0.28", features = ["sqlcipher"] }
rusqlite = { version = "0.28", features = ["sqlcipher", "release_memory"] }
rust-hsluv = "0.1"
sanitize-filename = "0.4"
serde_json = "1.0"
@@ -74,17 +71,21 @@ sha2 = "0.10"
smallvec = "1"
strum = "0.24"
strum_macros = "0.24"
tagger = "4.3.4"
textwrap = "0.16.0"
thiserror = "1"
tokio-io-timeout = "1.2.0"
tokio-stream = { version = "0.1.11", features = ["fs"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
toml = "0.7"
trust-dns-resolver = "0.22"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
fast-socks5 = "0.8"
humansize = "2"
qrcodegen = "1.7.0"
tagger = "4.3.4"
textwrap = "0.16.0"
async-channel = "1.8.0"
futures-lite = "1.12.0"
tokio-stream = { version = "0.1.11", features = ["fs"] }
tokio-io-timeout = "1.2.0"
reqwest = { version = "0.11.14", features = ["json"] }
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
[dev-dependencies]
ansi_term = "0.12.0"

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat_ffi"
version = "1.110.0"
version = "1.108.0"
description = "Deltachat FFI"
edition = "2018"
readme = "README.md"

View File

@@ -4327,18 +4327,6 @@ void dc_msg_set_text (dc_msg_t* msg, const char* text);
void dc_msg_set_html (dc_msg_t* msg, const char* html);
/**
* Sets the email's subject. If it's empty, a default subject
* will be used (e.g. `Message from Alice` or `Re: <last subject>`).
* This does not alter any information in the database.
*
* @memberof dc_msg_t
* @param msg The message object.
* @param subject The new subject.
*/
void dc_msg_set_subject (dc_msg_t* msg, const char* subject);
/**
* Set different sender name for a message.
* This overrides the name set by the dc_set_config()-option `displayname`.

View File

@@ -1,4 +1,4 @@
#![warn(unused, clippy::all, clippy::missing_docs_in_private_items)]
#![warn(unused, clippy::all)]
#![allow(
non_camel_case_types,
non_snake_case,
@@ -2003,10 +2003,12 @@ pub unsafe extern "C" fn dc_create_contact(
let ctx = &*context;
let name = to_string_lossy(name);
block_on(Contact::create(ctx, &name, &to_string_lossy(addr)))
.log_err(ctx, "Cannot create contact")
.map(|id| id.to_u32())
.unwrap_or(0)
block_on(async move {
Contact::create(ctx, &name, &to_string_lossy(addr))
.await
.map(|id| id.to_u32())
.unwrap_or(0)
})
}
#[no_mangle]
@@ -3596,16 +3598,6 @@ pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::
ffi_msg.message.set_html(to_opt_string_lossy(html))
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_subject(msg: *mut dc_msg_t, subject: *const libc::c_char) {
if msg.is_null() {
eprintln!("ignoring careless call to dc_msg_get_subject()");
return;
}
let ffi_msg = &mut *msg;
ffi_msg.message.set_subject(to_string_lossy(subject));
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_override_sender_name(
msg: *mut dc_msg_t,

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-jsonrpc"
version = "1.110.0"
version = "1.108.0"
description = "DeltaChat JSON-RPC API"
edition = "2021"
default-run = "deltachat-jsonrpc-server"
@@ -28,7 +28,7 @@ sanitize-filename = "0.4"
walkdir = "2.3.2"
# optional dependencies
axum = { version = "0.5.9", optional = true, features = ["ws"] }
axum = { version = "0.6.4", optional = true, features = ["ws"] }
env_logger = { version = "0.10.0", optional = true }
[dev-dependencies]
@@ -36,6 +36,5 @@ tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] }
[features]
default = ["vendored"]
default = []
webserver = ["env_logger", "axum", "tokio/full", "yerpc/support-axum"]
vendored = ["deltachat/vendored"]

View File

@@ -3,7 +3,7 @@
"dependencies": {
"@deltachat/tiny-emitter": "3.0.0",
"isomorphic-ws": "^4.0.1",
"yerpc": "^0.4.3"
"yerpc": "^0.3.3"
},
"devDependencies": {
"@types/chai": "^4.2.21",
@@ -14,7 +14,7 @@
"c8": "^7.10.0",
"chai": "^4.3.4",
"chai-as-promised": "^7.1.1",
"esbuild": "^0.17.9",
"esbuild": "^0.14.11",
"http-server": "^14.1.1",
"mocha": "^9.1.1",
"node-fetch": "^2.6.1",
@@ -24,19 +24,12 @@
"typescript": "^4.5.5",
"ws": "^8.5.0"
},
"exports": {
".": {
"import": "./dist/deltachat.js",
"require": "./dist/deltachat.cjs"
}
},
"license": "MPL-2.0",
"main": "dist/deltachat.js",
"name": "@deltachat/jsonrpc-client",
"scripts": {
"build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
"build": "run-s generate-bindings extract-constants build:tsc build:bundle",
"build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
"build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs",
"build:tsc": "tsc",
"docs": "typedoc --out docs deltachat.ts",
"example": "run-s build example:build example:start",
@@ -55,5 +48,5 @@
},
"type": "module",
"types": "dist/deltachat.d.ts",
"version": "1.110.0"
"version": "1.108.0"
}

View File

@@ -1,7 +1,6 @@
[package]
name = "deltachat-repl"
version = "1.110.0"
license = "MPL-2.0"
version = "1.108.0"
edition = "2021"
[dependencies]

View File

@@ -25,35 +25,7 @@ deltachat_rpc_client = [
line-length = 120
[tool.ruff]
select = [
"E", "W", # pycodestyle
"F", # Pyflakes
"N", # pep8-naming
"I", # isort
"ARG", # flake8-unused-arguments
"B", # flake8-bugbear
"C4", # flake8-comprehensions
"COM", # flake8-commas
"DTZ", # flake8-datetimez
"ICN", # flake8-import-conventions
"ISC", # flake8-implicit-str-concat
"PIE", # flake8-pie
"PT", # flake8-pytest-style
"RET", # flake8-return
"SIM", # flake8-simplify
"TCH", # flake8-type-checking
"TID", # flake8-tidy-imports
"YTT", # flake8-2020
"ERA", # eradicate
"PLC", # Pylint Convention
"PLE", # Pylint Error
"PLW", # Pylint Warning
"RUF006" # asyncio-dangling-task
]
select = ["E", "F", "W", "N", "YTT", "B", "C4", "ISC", "ICN", "PT", "RET", "SIM", "TID", "ARG", "DTZ", "ERA", "PLC", "PLE", "PLW", "PIE", "COM"]
line-length = 120
[tool.isort]

View File

@@ -1,15 +1,15 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
from dataclasses import dataclass
from ._utils import AttrDict
from .chat import Chat
from .const import ChatlistFlag, ContactFlag, SpecialContactId
from .contact import Contact
from .message import Message
from .rpc import Rpc
if TYPE_CHECKING:
from .deltachat import DeltaChat
from .rpc import Rpc
@dataclass
@@ -20,7 +20,7 @@ class Account:
id: int
@property
def _rpc(self) -> "Rpc":
def _rpc(self) -> Rpc:
return self.manager.rpc
async def wait_for_event(self) -> AttrDict:

View File

@@ -1,17 +1,16 @@
import calendar
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
from dataclasses import dataclass
from ._utils import AttrDict
from .const import ChatVisibility
from .contact import Contact
from .message import Message
from .rpc import Rpc
if TYPE_CHECKING:
from datetime import datetime
from .account import Account
from .rpc import Rpc
@dataclass
@@ -22,7 +21,7 @@ class Chat:
id: int
@property
def _rpc(self) -> "Rpc":
def _rpc(self) -> Rpc:
return self.account._rpc
async def delete(self) -> None:
@@ -182,23 +181,19 @@ class Chat:
"""Add contacts to this group."""
for cnt in contact:
if isinstance(cnt, str):
contact_id = (await self.account.create_contact(cnt)).id
cnt = (await self.account.create_contact(cnt)).id
elif not isinstance(cnt, int):
contact_id = cnt.id
else:
contact_id = cnt
await self._rpc.add_contact_to_chat(self.account.id, self.id, contact_id)
cnt = cnt.id
await self._rpc.add_contact_to_chat(self.account.id, self.id, cnt)
async def remove_contact(self, *contact: Union[int, str, Contact]) -> None:
"""Remove members from this group."""
for cnt in contact:
if isinstance(cnt, str):
contact_id = (await self.account.create_contact(cnt)).id
cnt = (await self.account.create_contact(cnt)).id
elif not isinstance(cnt, int):
contact_id = cnt.id
else:
contact_id = cnt
await self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id)
cnt = cnt.id
await self._rpc.remove_contact_from_chat(self.account.id, self.id, cnt)
async def get_contacts(self) -> List[Contact]:
"""Get the contacts belonging to this chat.
@@ -222,8 +217,8 @@ class Chat:
async def get_locations(
self,
contact: Optional[Contact] = None,
timestamp_from: Optional["datetime"] = None,
timestamp_to: Optional["datetime"] = None,
timestamp_from: Optional[datetime] = None,
timestamp_to: Optional[datetime] = None,
) -> List[AttrDict]:
"""Get list of location snapshots for the given contact in the given timespan."""
time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0
@@ -234,9 +229,9 @@ class Chat:
locations = []
contacts: Dict[int, Contact] = {}
for loc in result:
location = AttrDict(loc)
location["chat"] = self
location["contact"] = contacts.setdefault(location.contact_id, Contact(self.account, location.contact_id))
location["message"] = Message(self.account, location.msg_id)
locations.append(location)
loc = AttrDict(loc)
loc["chat"] = self
loc["contact"] = contacts.setdefault(loc.contact_id, Contact(self.account, loc.contact_id))
loc["message"] = Message(self.account, loc.msg_id)
locations.append(loc)
return locations

View File

@@ -2,7 +2,6 @@
import inspect
import logging
from typing import (
TYPE_CHECKING,
Callable,
Coroutine,
Dict,
@@ -14,6 +13,8 @@ from typing import (
Union,
)
from deltachat_rpc_client.account import Account
from ._utils import (
AttrDict,
parse_system_add_remove,
@@ -30,16 +31,13 @@ from .events import (
RawEvent,
)
if TYPE_CHECKING:
from deltachat_rpc_client.account import Account
class Client:
"""Simple Delta Chat client that listen to events of a single account."""
def __init__(
self,
account: "Account",
account: Account,
hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None,
logger: Optional[logging.Logger] = None,
) -> None:

View File

@@ -1,12 +1,12 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING
from dataclasses import dataclass
from ._utils import AttrDict
from .rpc import Rpc
if TYPE_CHECKING:
from .account import Account
from .chat import Chat
from .rpc import Rpc
@dataclass
@@ -21,7 +21,7 @@ class Contact:
id: int
@property
def _rpc(self) -> "Rpc":
def _rpc(self) -> Rpc:
return self.account._rpc
async def block(self) -> None:

View File

@@ -1,10 +1,8 @@
from typing import TYPE_CHECKING, Dict, List
from typing import Dict, List
from ._utils import AttrDict
from .account import Account
if TYPE_CHECKING:
from .rpc import Rpc
from .rpc import Rpc
class DeltaChat:
@@ -13,7 +11,7 @@ class DeltaChat:
This is the root of the object oriented API.
"""
def __init__(self, rpc: "Rpc") -> None:
def __init__(self, rpc: Rpc) -> None:
self.rpc = rpc
async def add_account(self) -> Account:

View File

@@ -2,13 +2,11 @@
import inspect
import re
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union
from typing import Callable, Iterable, Iterator, Optional, Set, Tuple, Union
from ._utils import AttrDict
from .const import EventType
if TYPE_CHECKING:
from ._utils import AttrDict
def _tuple_of(obj, type_: type) -> tuple:
if not obj:
@@ -82,7 +80,7 @@ class RawEvent(EventFilter):
return (self.types, self.func) == (other.types, other.func)
return False
async def filter(self, event: "AttrDict") -> bool:
async def filter(self, event: AttrDict) -> bool:
if self.types and event.type not in self.types:
return False
return await self._call_func(event)
@@ -120,7 +118,7 @@ class NewMessage(EventFilter):
command: Optional[str] = None,
is_bot: Optional[bool] = False,
is_info: Optional[bool] = None,
func: Optional[Callable[["AttrDict"], bool]] = None,
func: Optional[Callable[[AttrDict], bool]] = None,
) -> None:
super().__init__(func=func)
self.is_bot = is_bot
@@ -159,7 +157,7 @@ class NewMessage(EventFilter):
)
return False
async def filter(self, event: "AttrDict") -> bool:
async def filter(self, event: AttrDict) -> bool:
if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot:
return False
if self.is_info is not None and self.is_info != event.message_snapshot.is_info:
@@ -201,7 +199,7 @@ class MemberListChanged(EventFilter):
return (self.added, self.func) == (other.added, other.func)
return False
async def filter(self, event: "AttrDict") -> bool:
async def filter(self, event: AttrDict) -> bool:
if self.added is not None and self.added != event.member_added:
return False
return await self._call_func(event)
@@ -233,7 +231,7 @@ class GroupImageChanged(EventFilter):
return (self.deleted, self.func) == (other.deleted, other.func)
return False
async def filter(self, event: "AttrDict") -> bool:
async def filter(self, event: AttrDict) -> bool:
if self.deleted is not None and self.deleted != event.image_deleted:
return False
return await self._call_func(event)
@@ -258,7 +256,7 @@ class GroupNameChanged(EventFilter):
return self.func == other.func
return False
async def filter(self, event: "AttrDict") -> bool:
async def filter(self, event: AttrDict) -> bool:
return await self._call_func(event)

View File

@@ -1,13 +1,13 @@
import json
from dataclasses import dataclass
from typing import TYPE_CHECKING, Union
from dataclasses import dataclass
from ._utils import AttrDict
from .contact import Contact
from .rpc import Rpc
if TYPE_CHECKING:
from .account import Account
from .rpc import Rpc
@dataclass
@@ -18,7 +18,7 @@ class Message:
id: int
@property
def _rpc(self) -> "Rpc":
def _rpc(self) -> Rpc:
return self.account._rpc
async def send_reaction(self, *reaction: str):

View File

@@ -1,8 +1,8 @@
import asyncio
import json
import os
from typing import AsyncGenerator, List, Optional
import asyncio
import aiohttp
import pytest_asyncio

View File

@@ -1,7 +1,8 @@
import asyncio
from unittest.mock import MagicMock
import pytest
import asyncio
from deltachat_rpc_client import EventType, events
from deltachat_rpc_client.rpc import JsonRpcError

View File

@@ -1,4 +1,5 @@
import pytest
from deltachat_rpc_client import EventType

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-rpc-server"
version = "1.110.0"
version = "1.108.0"
description = "DeltaChat JSON-RPC server"
edition = "2021"
readme = "README.md"
@@ -13,7 +13,7 @@ categories = ["cryptography", "std", "email"]
name = "deltachat-rpc-server"
[dependencies]
deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
deltachat-jsonrpc = { path = "../deltachat-jsonrpc" }
anyhow = "1"
env_logger = { version = "0.10.0" }
@@ -23,7 +23,3 @@ serde_json = "1.0.91"
serde = { version = "1.0", features = ["derive"] }
tokio = { version = "1.25.0", features = ["io-std"] }
yerpc = { version = "0.4.0", features = ["anyhow_expose"] }
[features]
default = ["vendored"]
vendored = ["deltachat-jsonrpc/vendored"]

View File

@@ -1,16 +0,0 @@
[advisories]
ignore = [
"RUSTSEC-2020-0071",
]
[licenses]
allow = [
"0BSD",
"Apache-2.0",
"BSD-2-Clause",
"BSD-3-Clause",
"CC0-1.0",
"MIT",
"BSL-1.0", # Boost Software License 1.0
"Unicode-DFS-2016",
]

View File

@@ -60,5 +60,5 @@
"test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
},
"types": "node/dist/index.d.ts",
"version": "1.110.0"
"version": "1.108.0"
}

View File

@@ -21,7 +21,6 @@ classifiers = [
dependencies = [
"cffi>=1.0.0",
"imap-tools",
"importlib_metadata;python_version<'3.8'",
"pluggy",
"requests",
]
@@ -44,8 +43,8 @@ deltachat = [
[tool.setuptools_scm]
root = ".."
tag_regex = '^(?P<prefix>v)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
git_describe_command = "git describe --dirty --tags --long --match v*.*"
tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
git_describe_command = "git describe --dirty --tags --long --match py-*.*"
[tool.black]
line-length = 120

View File

@@ -1,9 +1,6 @@
import sys
if sys.version_info >= (3, 8):
from importlib.metadata import PackageNotFoundError, version
else:
from importlib_metadata import PackageNotFoundError, version
from pkg_resources import DistributionNotFound, get_distribution
from . import capi, events, hookspec # noqa
from .account import Account, get_core_info # noqa
@@ -14,8 +11,8 @@ from .hookspec import account_hookimpl, global_hookimpl # noqa
from .message import Message # noqa
try:
__version__ = version(__name__)
except PackageNotFoundError:
__version__ = get_distribution(__name__).version
except DistributionNotFound:
# package is not installed
__version__ = "0.0.0.dev0-unknown"

View File

@@ -207,14 +207,14 @@ class IdleManager:
return res
def wait_for_new_message(self, timeout=None) -> bytes:
while True:
while 1:
for item in self.check(timeout=timeout):
if b"EXISTS" in item or b"RECENT" in item:
return item
def wait_for_seen(self, timeout=None) -> int:
"""Return first message with SEEN flag from a running idle-stream."""
while True:
while 1:
for item in self.check(timeout=timeout):
if FETCH in item:
self.log(str(item))

View File

@@ -108,7 +108,7 @@ class FFIEventTracker:
return ev
def iter_events(self, timeout=None, check_error=True):
while True:
while 1:
yield self.get(timeout=timeout, check_error=check_error)
def get_matching(self, event_name_regex, check_error=True, timeout=None):
@@ -119,14 +119,14 @@ class FFIEventTracker:
def get_info_contains(self, regex: str) -> FFIEvent:
rex = re.compile(regex)
while True:
while 1:
ev = self.get_matching("DC_EVENT_INFO")
if rex.search(ev.data2):
return ev
def get_info_regex_groups(self, regex, check_error=True):
rex = re.compile(regex)
while True:
while 1:
ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
m = rex.match(ev.data2)
if m is not None:
@@ -137,7 +137,7 @@ class FFIEventTracker:
This only works reliably if the connectivity doesn't change
again too quickly, otherwise we might miss it.
"""
while True:
while 1:
if self.account.get_connectivity() == connectivity:
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -146,7 +146,7 @@ class FFIEventTracker:
"""Wait until the connectivity changes to `expected_next`.
Fails the test if it changes to something else.
"""
while True:
while 1:
current = self.account.get_connectivity()
if current == expected_next:
return
@@ -156,7 +156,7 @@ class FFIEventTracker:
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
def wait_for_all_work_done(self):
while True:
while 1:
if self.account.all_work_done():
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -164,7 +164,7 @@ class FFIEventTracker:
def ensure_event_not_queued(self, event_name_regex):
__tracebackhide__ = True
rex = re.compile(f"(?:{event_name_regex}).*")
while True:
while 1:
try:
ev = self._event_queue.get(False)
except Empty:
@@ -173,7 +173,7 @@ class FFIEventTracker:
assert not rex.match(ev.name), f"event found {ev}"
def wait_securejoin_inviter_progress(self, target):
while True:
while 1:
event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
if event.data2 >= target:
print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)

View File

@@ -309,7 +309,7 @@ class ACSetup:
def wait_one_configured(self, account):
"""wait until this account has successfully configured."""
if self._account2state[account] == self.CONFIGURING:
while True:
while 1:
acc = self._pop_config_success()
if acc == account:
break
@@ -638,7 +638,7 @@ class BotProcess:
def _run_stdout_thread(self) -> None:
try:
while True:
while 1:
line = self.popen.stdout.readline()
if not line:
break
@@ -659,7 +659,7 @@ class BotProcess:
for next_pattern in patterns:
print("+++FNMATCH:", next_pattern)
ignored = []
while True:
while 1:
line = self.stdout_queue.get()
if line is None:
if ignored:

View File

@@ -85,7 +85,7 @@ class ConfigureTracker:
self._imap_finished.wait()
def wait_progress(self, data1=None):
while True:
while 1:
evdata = self._progress.get()
if data1 is None or evdata == data1:
break

View File

@@ -1954,7 +1954,6 @@ def test_immediate_autodelete(acfactory, lp):
assert msg.text == "hello"
lp.sec("ac2: wait for close/expunge on autodelete")
ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
ac2._evtracker.get_info_contains("close/expunge succeeded")
lp.sec("ac2: check that message was autodeleted on server")
@@ -1996,34 +1995,6 @@ def test_delete_multiple_messages(acfactory, lp):
assert len(ac2.direct_imap.get_all_messages()) == 1
def test_trash_multiple_messages(acfactory, lp):
ac1, ac2 = acfactory.get_online_accounts(2)
ac2.set_config("delete_to_trash", "1")
chat12 = acfactory.get_accepted_chat(ac1, ac2)
lp.sec("ac1: sending 3 messages")
texts = ["first", "second", "third"]
for text in texts:
chat12.send_text(text)
lp.sec("ac2: waiting for all messages on the other side")
to_delete = []
for text in texts:
msg = ac2._evtracker.wait_next_incoming_message()
assert msg.text in texts
if text != "second":
to_delete.append(msg)
lp.sec("ac2: deleting all messages except second")
assert len(to_delete) == len(texts) - 1
ac2.delete_messages(to_delete)
ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
lp.sec("ac2: test that only one message is left")
ac2.direct_imap.select_config_folder("inbox")
assert len(ac2.direct_imap.get_all_messages()) == 1
def test_configure_error_msgs_wrong_pw(acfactory):
configdict = acfactory.get_next_liveconfig()
ac1 = acfactory.get_unconfigured_account()

View File

@@ -8,7 +8,7 @@ envlist =
[testenv]
commands =
pytest -n6 --extra-info -v -rsXx --ignored --strict-tls {posargs: tests examples}
pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
pip wheel . -w {toxworkdir}/wheelhouse --no-deps
setenv =
# Avoid stack overflow when Rust core is built without optimizations.
@@ -21,6 +21,7 @@ passenv =
RUSTC_WRAPPER
deps =
pytest
pytest-rerunfailures
pytest-timeout
pytest-xdist
pdbpp

View File

@@ -20,8 +20,6 @@ and an own build machine.
- `run_all.sh` builds Python wheels
- `aarch64-unknown-linux-musl.sh` cross-compiles static `deltachat-rpc-server` for aarch64
## Triggering runs on the build machine locally (fast!)
There is experimental support for triggering a remote Python or Rust test run

View File

@@ -1,18 +0,0 @@
#!/bin/sh
#
# Build statically linked deltachat-rpc-server for aarch64-unknown-linux-musl.
set -x
set -e
# Download Zig
rm -fr zig-linux-x86_64-0.10.1 zig-linux-x86_64-0.10.1.tar.xz
wget https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz
tar xf zig-linux-x86_64-0.10.1.tar.xz
export PATH="$PATH:$PWD/zig-linux-x86_64-0.10.1"
cargo install cargo-zigbuild
rustup target add aarch64-unknown-linux-musl
cargo zigbuild --release --target aarch64-unknown-linux-musl -p deltachat-rpc-server --features vendored

View File

@@ -1,44 +0,0 @@
#!/bin/sh
# Build deltachat-rpc-server for Android.
set -e
test -n "$ANDROID_NDK_ROOT" || exit 1
RUSTUP_TOOLCHAIN="1.64.0"
rustup install "$RUSTUP_TOOLCHAIN"
rustup target add armv7-linux-androideabi aarch64-linux-android i686-linux-android x86_64-linux-android --toolchain "$RUSTUP_TOOLCHAIN"
KERNEL="$(uname -s | tr '[:upper:]' '[:lower:]')"
ARCH="$(uname -m)"
NDK_HOST_TAG="$KERNEL-$ARCH"
TOOLCHAIN="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$NDK_HOST_TAG"
export PATH="$PATH:$TOOLCHAIN/bin/"
PACKAGE="deltachat-rpc-server"
export CARGO_PROFILE_RELEASE_LTO=on
CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang" \
CFLAGS=-D__ANDROID_API__=16 \
TARGET_CC=armv7a-linux-androideabi16-clang \
TARGET_AR=llvm-ar \
cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target armv7-linux-androideabi -p $PACKAGE
CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/aarch64-linux-android21-clang" \
CFLAGS=-D__ANDROID_API__=21 \
TARGET_CC=aarch64-linux-android21-clang \
TARGET_AR=llvm-ar \
cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target aarch64-linux-android -p $PACKAGE
CARGO_TARGET_I686_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/i686-linux-android16-clang" \
CFLAGS=-D__ANDROID_API__=16 \
TARGET_CC=i686-linux-android16-clang \
TARGET_AR=llvm-ar \
cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target i686-linux-android -p $PACKAGE
CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/x86_64-linux-android21-clang" \
CFLAGS=-D__ANDROID_API__=21 \
TARGET_CC=x86_64-linux-android21-clang \
TARGET_AR=llvm-ar \
cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target x86_64-linux-android -p $PACKAGE

View File

@@ -1,370 +1,370 @@
resources:
- name: deltachat-core-rust
type: git
icon: github
source:
branch: master
uri: https://github.com/deltachat/deltachat-core-rust.git
- name: deltachat-core-rust
type: git
icon: github
source:
branch: master
uri: https://github.com/deltachat/deltachat-core-rust.git
- name: deltachat-core-rust-release
type: git
icon: github
source:
branch: master
uri: https://github.com/deltachat/deltachat-core-rust.git
tag_filter: "v*"
- name: deltachat-core-rust-release
type: git
icon: github
source:
branch: master
uri: https://github.com/deltachat/deltachat-core-rust.git
tag_filter: "py-*"
jobs:
- name: doxygen
plan:
- get: deltachat-core-rust
trigger: true
- name: doxygen
plan:
- get: deltachat-core-rust
trigger: true
# Build Doxygen documentation
- task: build-doxygen
config:
inputs:
- name: deltachat-core-rust
outputs:
- name: c-docs
image_resource:
source:
repository: alpine
type: registry-image
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache doxygen git
cd deltachat-core-rust
scripts/run-doxygen.sh
cd ..
cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
# Build Doxygen documentation
- task: build-doxygen
config:
inputs:
- name: deltachat-core-rust
outputs:
- name: c-docs
image_resource:
source:
repository: alpine
type: registry-image
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache doxygen git
cd deltachat-core-rust
scripts/run-doxygen.sh
cd ..
cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
- task: upload-c-docs
config:
inputs:
- name: c-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
- task: upload-c-docs
config:
inputs:
- name: c-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
- name: python-x86_64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
- name: python-x86_64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64
platform: linux
caches:
- path: cache
run:
path: build
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-docs
path: ./python/doc/_build/
# Binary wheels
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-docs
path: ./python/doc/_build/
# Binary wheels
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload python docs to py.delta.chat
- task: upload-py-docs
config:
inputs:
- name: py-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
# Upload python docs to py.delta.chat
- task: upload-py-docs
config:
inputs:
- name: py-docs
image_resource:
type: registry-image
source:
repository: alpine
platform: linux
run:
path: sh
args:
- -ec
- |
apk add --no-cache rsync openssh-client
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
# Upload x86_64 wheels and source packages
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*
# Upload x86_64 wheels and source packages
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*
- name: python-aarch64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
- name: python-aarch64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64
platform: linux
caches:
- path: cache
run:
path: build
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload aarch64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*
# Upload aarch64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*manylinux201*
- name: python-musl-x86_64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
- name: python-musl-x86_64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64
platform: linux
caches:
- path: cache
run:
path: build
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload musl x86_64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*musllinux_1_1_x86_64*
# Upload musl x86_64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*musllinux_1_1_x86_64*
- name: python-musl-aarch64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
- name: python-musl-aarch64
plan:
- get: deltachat-core-rust
- get: deltachat-core-rust-release
trigger: true
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64
platform: linux
caches:
- path: cache
run:
path: build
# Build manylinux image with additional dependencies
- task: build-coredeps
privileged: true
config:
inputs:
# Building the latest, not tagged coredeps
- name: deltachat-core-rust
image_resource:
source:
repository: concourse/oci-build-task
type: registry-image
outputs:
- name: coredeps-image
path: image
params:
CONTEXT: deltachat-core-rust/scripts/coredeps
UNPACK_ROOTFS: "true"
BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64
platform: linux
caches:
- path: cache
run:
path: build
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Use built image to build python wheels
- task: build-wheels
image: coredeps-image
config:
inputs:
- name: deltachat-core-rust-release
path: .
outputs:
- name: py-wheels
path: ./python/.docker-tox/wheelhouse/
platform: linux
run:
path: bash
args:
- -exc
- |
scripts/run_all.sh
# Upload musl aarch64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*musllinux_1_1_aarch64*
# Upload musl aarch64 wheels
- task: upload-wheels
config:
inputs:
- name: py-wheels
image_resource:
type: registry-image
source:
repository: debian
platform: linux
run:
path: sh
args:
- -ec
- |
apt-get update -y
apt-get install -y --no-install-recommends python3-pip python3-setuptools
pip3 install devpi
devpi use https://m.devpi.net/dc/master
devpi login ((devpi.login)) --password ((devpi.password))
devpi upload py-wheels/*musllinux_1_1_aarch64*

View File

@@ -115,8 +115,10 @@ def main():
print("after commit, on master make sure to: ")
print("")
print(f" git tag -a v{newversion}")
print(f" git push origin v{newversion}")
print(f" git tag -a {newversion}")
print(f" git push origin {newversion}")
print(f" git tag -a py-{newversion}")
print(f" git push origin py-{newversion}")
print("")

View File

@@ -1,6 +1,7 @@
//! # Account manager module.
use std::collections::BTreeMap;
use std::future::Future;
use std::path::{Path, PathBuf};
use anyhow::{ensure, Context as _, Result};
@@ -150,7 +151,7 @@ impl Accounts {
if let Some(cfg) = self.config.get_account(id) {
let account_path = self.dir.join(cfg.dir);
fs::remove_dir_all(&account_path)
try_many_times(|| fs::remove_dir_all(&account_path))
.await
.context("failed to remove account data")?;
}
@@ -186,10 +187,10 @@ impl Accounts {
fs::create_dir_all(self.dir.join(&account_config.dir))
.await
.context("failed to create dir")?;
fs::rename(&dbfile, &new_dbfile)
try_many_times(|| fs::rename(&dbfile, &new_dbfile))
.await
.context("failed to rename dbfile")?;
fs::rename(&blobdir, &new_blobdir)
try_many_times(|| fs::rename(&blobdir, &new_blobdir))
.await
.context("failed to rename blobdir")?;
if walfile.exists() {
@@ -214,7 +215,7 @@ impl Accounts {
}
Err(err) => {
let account_path = std::path::PathBuf::from(&account_config.dir);
fs::remove_dir_all(&account_path)
try_many_times(|| fs::remove_dir_all(&account_path))
.await
.context("failed to remove account data")?;
self.config.remove_account(account_config.id).await?;
@@ -471,18 +472,42 @@ impl Config {
}
}
/// Spend up to 1 minute trying to do the operation.
///
/// Files may remain locked up to 30 seconds due to r2d2 bug:
/// <https://github.com/sfackler/r2d2/issues/99>
async fn try_many_times<F, Fut, T>(f: F) -> std::result::Result<(), T>
where
F: Fn() -> Fut,
Fut: Future<Output = std::result::Result<(), T>>,
{
let mut counter = 0;
loop {
counter += 1;
if let Err(err) = f().await {
if counter > 60 {
return Err(err);
}
// Wait 1 second and try again.
tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
} else {
break;
}
}
Ok(())
}
/// Configuration of a single account.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
struct AccountConfig {
/// Unique id.
pub id: u32,
/// Root directory for all data for this account.
///
/// The path is relative to the account manager directory.
pub dir: std::path::PathBuf,
/// Universally unique account identifier.
pub uuid: Uuid,
}

View File

@@ -644,7 +644,6 @@ Authentication-Results: dkim=";
.unwrap();
}
#[ignore = "Disallowing keychanges is disabled for now"]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_handle_authres_fails() -> Result<()> {
let mut tcm = TestContextManager::new();
@@ -822,8 +821,7 @@ Authentication-Results: dkim=";
.insert_str(0, "Authentication-Results: example.net; dkim=fail\n");
let rcvd = bob.recv_msg(&sent).await;
// Disallowing keychanges is disabled for now:
// assert!(rcvd.error.unwrap().contains("DKIM failed"));
assert!(rcvd.error.unwrap().contains("DKIM failed"));
// The message info should contain a warning:
assert!(message::get_msg_info(&bob, rcvd.id)
.await

View File

@@ -276,7 +276,7 @@ impl ChatId {
grpname,
grpid,
create_blocked,
create_smeared_timestamp(context),
create_smeared_timestamp(context).await,
create_protected,
param.unwrap_or_default(),
],
@@ -482,7 +482,7 @@ impl ChatId {
self,
&msg_text,
cmd,
create_smeared_timestamp(context),
create_smeared_timestamp(context).await,
None,
None,
None,
@@ -907,8 +907,7 @@ impl ChatId {
async fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> Result<Option<T>>
where
F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
T: Send + 'static,
F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
{
let sql = &context.sql;
let query = format!(
@@ -1881,10 +1880,7 @@ pub(crate) async fn update_special_chat_names(context: &Context) -> Result<()> {
/// [`Deref`]: std::ops::Deref
#[derive(Debug)]
pub(crate) struct ChatIdBlocked {
/// Chat ID.
pub id: ChatId,
/// Whether the chat is blocked, unblocked or a contact request.
pub blocked: Blocked,
}
@@ -1956,6 +1952,7 @@ impl ChatIdBlocked {
_ => (),
}
let created_timestamp = create_smeared_timestamp(context).await;
let chat_id = context
.sql
.transaction(move |transaction| {
@@ -1968,7 +1965,7 @@ impl ChatIdBlocked {
chat_name,
params.to_string(),
create_blocked as u8,
create_smeared_timestamp(context)
created_timestamp,
],
)?;
let chat_id = ChatId::new(
@@ -2116,7 +2113,7 @@ async fn prepare_msg_common(
context,
msg,
update_msg_id,
create_smeared_timestamp(context),
create_smeared_timestamp(context).await,
)
.await?;
msg.chat_id = chat_id;
@@ -2841,7 +2838,7 @@ pub async fn create_group_chat(
Chattype::Group,
chat_name,
grpid,
create_smeared_timestamp(context),
create_smeared_timestamp(context).await,
],
)
.await?;
@@ -2899,7 +2896,7 @@ pub async fn create_broadcast_list(context: &Context) -> Result<ChatId> {
Chattype::Broadcast,
chat_name,
grpid,
create_smeared_timestamp(context),
create_smeared_timestamp(context).await,
],
)
.await?;
@@ -3360,7 +3357,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
if let Some(reason) = chat.why_cant_send(context).await? {
bail!("cannot send to {}: {}", chat_id, reason);
}
curr_timestamp = create_smeared_timestamps(context, msg_ids.len());
curr_timestamp = create_smeared_timestamps(context, msg_ids.len()).await;
let ids = context
.sql
.query_map(
@@ -3400,7 +3397,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
msg.param.remove(Param::WebxdcSummaryTimestamp);
msg.in_reply_to = None;
// do not leak data as group names; a default subject is generated by mimefactory
// do not leak data as group names; a default subject is generated by mimfactory
msg.subject = "".to_string();
let new_msg_id: MsgId;
@@ -3562,7 +3559,7 @@ pub async fn add_device_msg_with_importance(
msg.try_calc_and_set_dimensions(context).await.ok();
prepare_msg_blob(context, msg).await?;
let timestamp_sent = create_smeared_timestamp(context);
let timestamp_sent = create_smeared_timestamp(context).await;
// makes sure, the added message is the last one,
// even if the date is wrong (useful esp. when warning about bad dates)
@@ -4090,6 +4087,7 @@ mod tests {
send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
let add1 = alice.pop_sent_msg().await;
add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
@@ -4108,18 +4106,29 @@ mod tests {
remove_contact_from_chat(&alice, alice_chat_id, daisy_id).await?;
let remove2 = alice.pop_sent_msg().await;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2);
// Bob receives the add and deletion messages out of order
let bob = TestContext::new_bob().await;
bob.recv_msg(&add1).await;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
bob.recv_msg(&add3).await;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
let bob_chat_id = bob.recv_msg(&add2).await.chat_id;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 4);
bob.recv_msg(&remove2).await;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
bob.recv_msg(&remove1).await;
tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 2);
Ok(())

View File

@@ -1,7 +1,5 @@
//! # Key-value configuration management.
use std::str::FromStr;
use anyhow::{ensure, Context as _, Result};
use strum::{EnumProperty, IntoEnumIterator};
use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
@@ -175,10 +173,6 @@ pub enum Config {
#[strum(props(default = "0"))]
DeleteDeviceAfter,
/// Move messages to the Trash folder instead of marking them "\Deleted". Overrides
/// `ProviderOptions::delete_to_trash`.
DeleteToTrash,
/// Save raw MIME messages with headers in the database if true.
SaveMimeHeaders,
@@ -233,9 +227,6 @@ pub enum Config {
/// Configured "Sent" folder.
ConfiguredSentboxFolder,
/// Configured "Trash" folder.
ConfiguredTrashFolder,
/// Unix timestamp of the last successful configuration.
ConfiguredTimestamp,
@@ -300,9 +291,6 @@ pub enum Config {
/// See `crate::authres::update_authservid_candidates`.
AuthservIdCandidates,
/// Make all outgoing messages with Autocrypt header "multipart/signed".
SignUnencrypted,
/// Let the core save all events to the database.
/// This value is used internally to remember the MsgId of the logging xdc
#[strum(props(default = "0"))]
@@ -339,37 +327,30 @@ impl Context {
}
}
/// Returns Some(T) if a value for the given key exists and was successfully parsed.
/// Returns None if could not parse.
pub async fn get_config_parsed<T: FromStr>(&self, key: Config) -> Result<Option<T>> {
self.get_config(key)
.await
.map(|s: Option<String>| s.and_then(|s| s.parse().ok()))
}
/// Returns 32-bit signed integer configuration value for the given key.
pub async fn get_config_int(&self, key: Config) -> Result<i32> {
Ok(self.get_config_parsed(key).await?.unwrap_or_default())
self.get_config(key)
.await
.map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
}
/// Returns 64-bit signed integer configuration value for the given key.
pub async fn get_config_i64(&self, key: Config) -> Result<i64> {
Ok(self.get_config_parsed(key).await?.unwrap_or_default())
self.get_config(key)
.await
.map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
}
/// Returns 64-bit unsigned integer configuration value for the given key.
pub async fn get_config_u64(&self, key: Config) -> Result<u64> {
Ok(self.get_config_parsed(key).await?.unwrap_or_default())
}
/// Returns boolean configuration value (if any) for the given key.
pub async fn get_config_bool_opt(&self, key: Config) -> Result<Option<bool>> {
Ok(self.get_config_parsed::<i32>(key).await?.map(|x| x != 0))
self.get_config(key)
.await
.map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
}
/// Returns boolean configuration value for the given key.
pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
Ok(self.get_config_bool_opt(key).await?.unwrap_or_default())
Ok(self.get_config_int(key).await? != 0)
}
/// Returns true if movebox ("DeltaChat" folder) should be watched.
@@ -569,6 +550,7 @@ fn get_config_keys_string() -> String {
#[cfg(test)]
mod tests {
use std::str::FromStr;
use std::string::ToString;
use num_traits::FromPrimitive;

View File

@@ -249,7 +249,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
}
}
},
strict_tls: Some(provider.opt.strict_tls),
strict_tls: Some(provider.strict_tls),
})
.collect();
@@ -338,7 +338,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
.collect();
let provider_strict_tls = param
.provider
.map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls);
.map_or(socks5_config.is_some(), |provider| provider.strict_tls);
let smtp_config_task = task::spawn(async move {
let mut smtp_configured = false;
@@ -646,14 +646,10 @@ async fn try_smtp_one_param(
}
}
/// Failure to connect and login with email client configuration.
#[derive(Debug, thiserror::Error)]
#[error("Trying {config}…\nError: {msg}")]
pub struct ConfigurationError {
/// Tried configuration description.
config: String,
/// Error message.
msg: String,
}

View File

@@ -190,18 +190,18 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
/// How many existing messages shall be fetched after configuration.
pub(crate) const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
// max. width/height of an avatar
pub(crate) const BALANCED_AVATAR_SIZE: u32 = 256;
pub(crate) const WORSE_AVATAR_SIZE: u32 = 128;
pub const BALANCED_AVATAR_SIZE: u32 = 256;
pub const WORSE_AVATAR_SIZE: u32 = 128;
// max. width/height of images
pub const BALANCED_IMAGE_SIZE: u32 = 1280;
pub const WORSE_IMAGE_SIZE: u32 = 640;
// this value can be increased if the folder configuration is changed and must be redone on next program start
pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
#[cfg(test)]
mod tests {

View File

@@ -13,7 +13,6 @@ use async_channel::{self as channel, Receiver, Sender};
use deltachat_derive::{FromSql, ToSql};
use once_cell::sync::Lazy;
use regex::Regex;
use rusqlite::OptionalExtension;
use serde::{Deserialize, Serialize};
use tokio::task;
use tokio::time::{timeout, Duration};
@@ -521,6 +520,8 @@ impl Contact {
/// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
///
/// Returns the contact_id and a `Modifier` value indicating if a modification occurred.
///
/// Returns None if the contact with such address cannot exist.
pub(crate) async fn add_or_lookup(
context: &Context,
name: &str,
@@ -565,12 +566,14 @@ impl Contact {
);
let mut update_addr = false;
let mut row_id = 0;
let row_id = context.sql.transaction(|transaction| {
let row = transaction.query_row(
"SELECT id, name, addr, origin, authname
FROM contacts WHERE addr=? COLLATE NOCASE",
[addr.to_string()],
if let Some((id, row_name, row_addr, row_origin, row_authname)) = context
.sql
.query_row_optional(
"SELECT id, name, addr, origin, authname \
FROM contacts WHERE addr=? COLLATE NOCASE;",
paramsv![addr.to_string()],
|row| {
let row_id: isize = row.get(0)?;
let row_name: String = row.get(1)?;
@@ -579,130 +582,120 @@ impl Contact {
let row_authname: String = row.get(4)?;
Ok((row_id, row_name, row_addr, row_origin, row_authname))
}).optional()?;
},
)
.await?
{
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
let row_id;
if let Some((id, row_name, row_addr, row_origin, row_authname)) = row {
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
transaction
.execute(
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
if update_authname {
name.to_string()
} else {
row_authname
},
row_id
],
)?;
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<ChatId> = transaction.query_row(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
params![Chattype::Single, isize::try_from(row_id)?],
|row| {
let chat_id: ChatId = row.get(0)?;
Ok(chat_id)
}
).optional()?;
if let Some(chat_id) = chat_id {
let contact_id = ContactId::new(row_id);
let (addr, name, authname) =
transaction.query_row(
"SELECT addr, name, authname
FROM contacts
WHERE id=?",
params![contact_id],
|row| {
let addr: String = row.get(0)?;
let name: String = row.get(1)?;
let authname: String = row.get(2)?;
Ok((addr, name, authname))
})?;
let chat_name = if !name.is_empty() {
name
} else if !authname.is_empty() {
authname
} else {
addr
};
let count = transaction.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?1",
params![chat_name, chat_id])?;
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(chat_id));
}
}
}
sth_modified = Modifier::Modified;
}
} else {
let update_name = manual;
let update_authname = !manual;
transaction
context
.sql
.execute(
"INSERT INTO contacts (name, addr, origin, authname)
VALUES (?, ?, ?, ?);",
params![
if update_name {
name.to_string()
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
"".to_string()
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
row_authname
},
row_id
],
)?;
)
.await
.ok();
sth_modified = Modifier::Created;
row_id = u32::try_from(transaction.last_insert_rowid())?;
info!(context, "added contact id={} addr={}", row_id, &addr);
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<i32> = context.sql.query_get_value(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
paramsv![Chattype::Single, isize::try_from(row_id)?]
).await?;
if let Some(chat_id) = chat_id {
let contact = Contact::get_by_id(context, ContactId::new(row_id)).await?;
let chat_name = contact.get_display_name();
match context
.sql
.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?3",
paramsv![chat_name, chat_id, chat_name],
)
.await
{
Err(err) => warn!(context, "Can't update chat name: {}", err),
Ok(count) => {
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(ChatId::new(
chat_id.try_into()?,
)));
}
}
}
}
}
sth_modified = Modifier::Modified;
}
Ok(row_id)
}).await?;
} else {
let update_name = manual;
let update_authname = !manual;
if let Ok(new_row_id) = context
.sql
.insert(
"INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
paramsv![
if update_name {
name.to_string()
} else {
"".to_string()
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
],
)
.await
{
row_id = u32::try_from(new_row_id)?;
sth_modified = Modifier::Created;
info!(context, "added contact id={} addr={}", row_id, &addr);
} else {
error!(context, "Cannot add contact.");
}
}
Ok((ContactId::new(row_id), sth_modified))
}
@@ -869,45 +862,47 @@ impl Contact {
Ok(ret)
}
/// Adds blocked mailinglists as contacts
/// to allow unblocking them as if they are contacts
/// (this way, only one unblock-ffi is needed and only one set of ui-functions,
/// from the users perspective,
/// there is not much difference in an email- and a mailinglist-address)
// add blocked mailinglists as contacts
// to allow unblocking them as if they are contacts
// (this way, only one unblock-ffi is needed and only one set of ui-functions,
// from the users perspective,
// there is not much difference in an email- and a mailinglist-address)
async fn update_blocked_mailinglist_contacts(context: &Context) -> Result<()> {
context
let blocked_mailinglists = context
.sql
.transaction(move |transaction| {
let mut stmt = transaction
.prepare("SELECT name, grpid FROM chats WHERE type=? AND blocked=?")?;
let rows = stmt.query_map(params![Chattype::Mailinglist, Blocked::Yes], |row| {
let name: String = row.get(0)?;
let grpid: String = row.get(1)?;
Ok((name, grpid))
})?;
let blocked_mailinglists = rows.collect::<std::result::Result<Vec<_>, _>>()?;
for (name, grpid) in blocked_mailinglists {
let count = transaction.query_row(
"SELECT COUNT(id) FROM contacts WHERE addr=?",
[&grpid],
|row| {
let count: isize = row.get(0)?;
Ok(count)
},
)?;
if count == 0 {
transaction.execute("INSERT INTO contacts (addr) VALUES (?)", [&grpid])?;
}
// Always do an update in case the blocking is reset or name is changed.
transaction.execute(
"UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?",
params![&name, Origin::MailinglistAddress, &grpid],
)?;
}
Ok(())
})
.query_map(
"SELECT name, grpid FROM chats WHERE type=? AND blocked=?;",
paramsv![Chattype::Mailinglist, Blocked::Yes],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
|rows| {
rows.collect::<std::result::Result<Vec<_>, _>>()
.map_err(Into::into)
},
)
.await?;
for (name, grpid) in blocked_mailinglists {
if !context
.sql
.exists(
"SELECT COUNT(id) FROM contacts WHERE addr=?;",
paramsv![grpid],
)
.await?
{
context
.sql
.execute("INSERT INTO contacts (addr) VALUES (?);", paramsv![grpid])
.await?;
}
// always do an update in case the blocking is reset or name is changed
context
.sql
.execute(
"UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?;",
paramsv![name, Origin::MailinglistAddress, grpid],
)
.await?;
}
Ok(())
}

View File

@@ -4,11 +4,10 @@ use std::collections::{BTreeMap, HashMap};
use std::ffi::OsString;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use anyhow::{bail, ensure, Context as _, Result};
use anyhow::{bail, ensure, Result};
use async_channel::{self as channel, Receiver, Sender};
use ratelimit::Ratelimit;
use tokio::sync::{Mutex, RwLock};
@@ -27,7 +26,6 @@ use crate::quota::QuotaInfo;
use crate::scheduler::Scheduler;
use crate::sql::Sql;
use crate::stock_str::StockStrings;
use crate::timesmearing::SmearedTimestamp;
use crate::tools::{duration_to_str, time};
/// Builder for the [`Context`].
@@ -190,7 +188,7 @@ pub struct InnerContext {
/// Blob directory path
pub(crate) blobdir: PathBuf,
pub(crate) sql: Sql,
pub(crate) smeared_timestamp: SmearedTimestamp,
pub(crate) last_smeared_timestamp: RwLock<i64>,
running_state: RwLock<RunningState>,
/// Mutex to avoid generating the key for the user more than once.
pub(crate) generating_key_mutex: Mutex<()>,
@@ -208,9 +206,6 @@ pub struct InnerContext {
/// Set to `None` if quota was never tried to load.
pub(crate) quota: RwLock<Option<QuotaInfo>>,
/// Set to true if quota update is requested.
pub(crate) quota_update_request: AtomicBool,
/// Server ID response if ID capability is supported
/// and the server returned non-NIL on the inbox connection.
/// <https://datatracker.ietf.org/doc/html/rfc2971>
@@ -361,7 +356,7 @@ impl Context {
blobdir,
running_state: RwLock::new(Default::default()),
sql: Sql::new(dbfile),
smeared_timestamp: SmearedTimestamp::new(),
last_smeared_timestamp: RwLock::new(0),
generating_key_mutex: Mutex::new(()),
oauth2_mutex: Mutex::new(()),
wrong_pw_warning_mutex: Mutex::new(()),
@@ -370,7 +365,6 @@ impl Context {
scheduler: RwLock::new(None),
ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow to send 6 messages immediately, no more than once every 10 seconds.
quota: RwLock::new(None),
quota_update_request: AtomicBool::new(false),
server_id: RwLock::new(None),
creation_time: std::time::SystemTime::now(),
last_full_folder_scan: Mutex::new(None),
@@ -629,10 +623,6 @@ impl Context {
.get_config(Config::ConfiguredMvboxFolder)
.await?
.unwrap_or_else(|| "<unset>".to_string());
let configured_trash_folder = self
.get_config(Config::ConfiguredTrashFolder)
.await?
.unwrap_or_else(|| "<unset>".to_string());
let mut res = get_info();
@@ -699,7 +689,6 @@ impl Context {
res.insert("configured_inbox_folder", configured_inbox_folder);
res.insert("configured_sentbox_folder", configured_sentbox_folder);
res.insert("configured_mvbox_folder", configured_mvbox_folder);
res.insert("configured_trash_folder", configured_trash_folder);
res.insert("mdns_enabled", mdns_enabled.to_string());
res.insert("e2ee_enabled", e2ee_enabled.to_string());
res.insert(
@@ -733,12 +722,6 @@ impl Context {
.await?
.to_string(),
);
res.insert(
"delete_to_trash",
self.get_config(Config::DeleteToTrash)
.await?
.unwrap_or_else(|| "<unset>".to_string()),
);
res.insert(
"last_housekeeping",
self.get_config_int(Config::LastHousekeeping)
@@ -763,12 +746,6 @@ impl Context {
.await?
.unwrap_or_default(),
);
res.insert(
"sign_unencrypted",
self.get_config_int(Config::SignUnencrypted)
.await?
.to_string(),
);
res.insert(
"debug_logging",
@@ -910,33 +887,6 @@ impl Context {
Ok(mvbox.as_deref() == Some(folder_name))
}
/// Returns true if given folder name is the name of the trash folder.
pub async fn is_trash(&self, folder_name: &str) -> Result<bool> {
let trash = self.get_config(Config::ConfiguredTrashFolder).await?;
Ok(trash.as_deref() == Some(folder_name))
}
pub(crate) async fn should_delete_to_trash(&self) -> Result<bool> {
if let Some(v) = self.get_config_bool_opt(Config::DeleteToTrash).await? {
return Ok(v);
}
if let Some(provider) = self.get_configured_provider().await? {
return Ok(provider.opt.delete_to_trash);
}
Ok(false)
}
/// Returns `target` for deleted messages as per `imap` table. Empty string means "delete w/o
/// moving to trash".
pub(crate) async fn get_delete_msgs_target(&self) -> Result<String> {
if !self.should_delete_to_trash().await? {
return Ok("".into());
}
self.get_config(Config::ConfiguredTrashFolder)
.await?
.context("No configured trash folder")
}
pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf {
let mut blob_fname = OsString::new();
blob_fname.push(dbfile.file_name().unwrap_or_default());

View File

@@ -99,8 +99,7 @@ pub(crate) async fn prepare_decryption(
from,
autocrypt_header.as_ref(),
message_time,
// Disallowing keychanges is disabled for now:
true, // dkim_results.allow_keychange,
dkim_results.allow_keychange,
)
.await?;

View File

@@ -13,6 +13,7 @@ use crate::imap::{Imap, ImapActionResult};
use crate::job::{self, Action, Job, Status};
use crate::message::{Message, MsgId, Viewtype};
use crate::mimeparser::{MimeMessage, Part};
use crate::param::Params;
use crate::tools::time;
use crate::{job_try, stock_str, EventType};
@@ -85,7 +86,11 @@ impl MsgId {
DownloadState::Available | DownloadState::Failure => {
self.update_download_state(context, DownloadState::InProgress)
.await?;
job::add(context, Job::new(Action::DownloadMsg, self.to_u32())).await?;
job::add(
context,
Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0),
)
.await?;
}
}
Ok(())
@@ -133,7 +138,7 @@ impl Job {
context
.sql
.query_row_optional(
"SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target=folder",
"SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
paramsv![msg.rfc724_mid],
|row| {
let server_uid: u32 = row.get(0)?;

View File

@@ -124,19 +124,6 @@ impl EncryptHelper {
Ok(ctext)
}
/// Signs the passed-in `mail` using the private key from `context`.
/// Returns the payload and the signature.
pub async fn sign(
self,
context: &Context,
mail: lettre_email::PartBuilder,
) -> Result<(lettre_email::MimeMessage, String)> {
let sign_key = SignedSecretKey::load_self(context).await?;
let mime_message = mail.build();
let signature = pgp::pk_calc_signature(mime_message.as_string().as_bytes(), &sign_key)?;
Ok((mime_message, signature))
}
}
/// Ensures a private key exists for the configured user.

View File

@@ -588,25 +588,19 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()
now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
),
};
let target = context.get_delete_msgs_target().await?;
context
.sql
.execute(
"UPDATE imap
SET target=?
SET target=''
WHERE rfc724_mid IN (
SELECT rfc724_mid FROM msgs
WHERE ((download_state = 0 AND timestamp < ?) OR
(download_state != 0 AND timestamp < ?) OR
(ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
)",
paramsv![
target,
threshold_timestamp,
threshold_timestamp_extended,
now,
],
paramsv![threshold_timestamp, threshold_timestamp_extended, now],
)
.await?;
@@ -650,7 +644,7 @@ mod tests {
use crate::download::DownloadState;
use crate::receive_imf::receive_imf;
use crate::test_utils::TestContext;
use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE;
use crate::tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
use crate::{
chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
tools::IsNoneOrEmpty,

View File

@@ -113,17 +113,13 @@ impl async_imap::Authenticator for OAuth2 {
}
}
#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)]
pub enum FolderMeaning {
#[derive(Debug, PartialEq, Clone, Copy)]
enum FolderMeaning {
Unknown,
/// Spam folder.
Spam,
Inbox,
Mvbox,
Sent,
Trash,
Drafts,
Other,
/// Virtual folders.
///
@@ -135,15 +131,13 @@ pub enum FolderMeaning {
}
impl FolderMeaning {
pub fn to_config(self) -> Option<Config> {
fn to_config(self) -> Option<Config> {
match self {
FolderMeaning::Unknown => None,
FolderMeaning::Spam => None,
FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder),
FolderMeaning::Drafts => None,
FolderMeaning::Other => None,
FolderMeaning::Virtual => None,
}
}
@@ -151,11 +145,8 @@ impl FolderMeaning {
#[derive(Debug)]
struct ImapConfig {
/// Email address.
pub addr: String,
pub lp: ServerLoginParam,
/// SOCKS 5 configuration.
pub socks5_config: Option<Socks5Config>,
pub strict_tls: bool,
}
@@ -279,7 +270,7 @@ impl Imap {
param
.provider
.map_or(param.socks5_config.is_some(), |provider| {
provider.opt.strict_tls
provider.strict_tls
}),
idle_interrupt_receiver,
)?;
@@ -458,7 +449,7 @@ impl Imap {
&mut self,
context: &Context,
watch_folder: &str,
folder_meaning: FolderMeaning,
is_spam_folder: bool,
) -> Result<()> {
if !context.sql.is_open().await {
// probably shutdown
@@ -467,7 +458,7 @@ impl Imap {
self.prepare(context).await?;
let msgs_fetched = self
.fetch_new_messages(context, watch_folder, folder_meaning, false)
.fetch_new_messages(context, watch_folder, is_spam_folder, false)
.await
.context("fetch_new_messages")?;
if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
@@ -499,60 +490,49 @@ impl Imap {
pub(crate) async fn resync_folder_uids(
&mut self,
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
folder: String,
) -> Result<()> {
// Collect pairs of UID and Message-ID.
let mut msgs = BTreeMap::new();
let mut msg_ids = BTreeMap::new();
let session = self
.session
.as_mut()
.context("IMAP No connection established")?;
session.select_folder(context, Some(folder)).await?;
session.select_folder(context, Some(&folder)).await?;
let mut list = session
.uid_fetch("1:*", RFC724MID_UID)
.await
.with_context(|| format!("can't resync folder {folder}"))?;
while let Some(fetch) = list.next().await {
let fetch = fetch?;
let headers = match get_fetch_headers(&fetch) {
Ok(headers) => headers,
Err(err) => {
warn!(context, "Failed to parse FETCH headers: {}", err);
continue;
}
};
let message_id = prefetch_get_message_id(&headers);
let msg = fetch?;
if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) {
msgs.insert(
uid,
(
rfc724_mid,
target_folder(context, folder, folder_meaning, &headers).await?,
),
);
// Get Message-ID
let message_id =
get_fetch_headers(&msg).map_or(None, |headers| prefetch_get_message_id(&headers));
if let (Some(uid), Some(rfc724_mid)) = (msg.uid, message_id) {
msg_ids.insert(uid, rfc724_mid);
}
}
info!(
context,
"Resync: collected {} message IDs in folder {}",
msgs.len(),
folder,
msg_ids.len(),
&folder
);
let uid_validity = get_uidvalidity(context, folder).await?;
let uid_validity = get_uidvalidity(context, &folder).await?;
// Write collected UIDs to SQLite database.
context
.sql
.transaction(move |transaction| {
transaction.execute("DELETE FROM imap WHERE folder=?", params![folder])?;
for (uid, (rfc724_mid, target)) in &msgs {
for (uid, rfc724_mid) in &msg_ids {
// This may detect previously undetected moved
// messages, so we update server_folder too.
transaction.execute(
@@ -561,7 +541,7 @@ impl Imap {
ON CONFLICT(folder, uid, uidvalidity)
DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
target=excluded.target",
params![rfc724_mid, folder, uid, uid_validity, target],
params![rfc724_mid, folder, uid, uid_validity, folder],
)?;
}
Ok(())
@@ -703,10 +683,10 @@ impl Imap {
&mut self,
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
is_spam_folder: bool,
fetch_existing_msgs: bool,
) -> Result<bool> {
if should_ignore_folder(context, folder, folder_meaning).await? {
if should_ignore_folder(context, folder, is_spam_folder).await? {
info!(context, "Not fetching from {}", folder);
return Ok(false);
}
@@ -733,6 +713,8 @@ impl Imap {
};
let read_cnt = msgs.len();
let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
.unwrap_or_default();
let download_limit = context.download_limit().await?;
let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1);
let mut uid_message_ids = BTreeMap::new();
@@ -750,7 +732,14 @@ impl Imap {
// Get the Message-ID or generate a fake one to identify the message in the database.
let message_id = prefetch_get_or_create_message_id(&headers);
let target = target_folder(context, folder, folder_meaning, &headers).await?;
let target = match target_folder(context, folder, is_spam_folder, &headers).await? {
Some(config) => match context.get_config(config).await? {
Some(target) => target,
None => folder.to_string(),
},
None => folder.to_string(),
};
context
.sql
@@ -774,13 +763,14 @@ impl Imap {
// Never download messages directly from the spam folder.
// If the sender is known, the message will be moved to the Inbox or Mvbox
// and then we download the message from there.
// Also see `spam_target_folder_cfg()`.
&& folder_meaning != FolderMeaning::Spam
// Also see `spam_target_folder()`.
&& !is_spam_folder
&& prefetch_should_download(
context,
&headers,
&message_id,
fetch_response.flags(),
show_emails,
)
.await.context("prefetch_should_download")?
{
@@ -880,21 +870,17 @@ impl Imap {
.context("failed to get recipients from the inbox")?;
if context.get_config_bool(Config::FetchExistingMsgs).await? {
for meaning in [
FolderMeaning::Mvbox,
FolderMeaning::Inbox,
FolderMeaning::Sent,
for config in &[
Config::ConfiguredMvboxFolder,
Config::ConfiguredInboxFolder,
Config::ConfiguredSentboxFolder,
] {
let config = match meaning.to_config() {
Some(c) => c,
None => continue,
};
if let Some(folder) = context.get_config(config).await? {
if let Some(folder) = context.get_config(*config).await? {
info!(
context,
"Fetching existing messages from folder \"{}\"", folder
);
self.fetch_new_messages(context, &folder, meaning, true)
self.fetch_new_messages(context, &folder, false, true)
.await
.context("could not fetch existing messages")?;
}
@@ -966,60 +952,44 @@ impl Session {
return Ok(());
}
Err(err) => {
if context.should_delete_to_trash().await? {
error!(
context,
"Cannot move messages {} to {}, no fallback to COPY/DELETE because \
delete_to_trash is set. Error: {:#}",
set,
target,
err,
);
return Err(err.into());
}
warn!(
context,
"Cannot move messages, fallback to COPY/DELETE {} to {}: {}",
"Cannot move message, fallback to COPY/DELETE {} to {}: {}",
set,
target,
err
);
}
}
}
// Server does not support MOVE or MOVE failed.
// Copy messages to the destination folder if needed and mark records for deletion.
let copy = !context.is_trash(target).await?;
if copy {
} else {
info!(
context,
"Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target
);
self.uid_copy(&set, &target).await?;
} else {
error!(
context,
"Server does not support MOVE, fallback to DELETE {} to {}", set, target,
);
}
context
.sql
.execute(
&format!(
"UPDATE imap SET target='' WHERE id IN ({})",
sql::repeat_vars(row_ids.len())
),
rusqlite::params_from_iter(row_ids),
)
.await
.context("cannot plan deletion of messages")?;
if copy {
context.emit_event(EventType::ImapMessageMoved(format!(
"IMAP messages {set} copied to {target}"
)));
// Server does not support MOVE or MOVE failed.
// Copy the message to the destination folder and mark the record for deletion.
match self.uid_copy(&set, &target).await {
Ok(()) => {
context
.sql
.execute(
&format!(
"UPDATE imap SET target='' WHERE id IN ({})",
sql::repeat_vars(row_ids.len())
),
rusqlite::params_from_iter(row_ids),
)
.await
.context("cannot plan deletion of copied messages")?;
context.emit_event(EventType::ImapMessageMoved(format!(
"IMAP messages {set} copied to {target}"
)));
Ok(())
}
Err(err) => Err(err.into()),
}
Ok(())
}
/// Moves and deletes messages as planned in the `imap` table.
@@ -1674,7 +1644,7 @@ impl Imap {
}
}
let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
let folder_meaning = get_folder_meaning(&folder);
let folder_name_meaning = get_folder_meaning_by_name(folder.name());
if let Some(config) = folder_meaning.to_config() {
// Always takes precedence
@@ -1806,7 +1776,7 @@ async fn should_move_out_of_spam(
/// If this returns None, the message will not be moved out of the
/// Spam folder, and as `fetch_new_messages()` doesn't download
/// messages from the Spam folder, the message will be ignored.
async fn spam_target_folder_cfg(
async fn spam_target_folder(
context: &Context,
headers: &[mailparse::MailHeader<'_>],
) -> Result<Option<Config>> {
@@ -1827,18 +1797,18 @@ async fn spam_target_folder_cfg(
/// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if
/// the message needs to be moved from `folder`. Otherwise returns `None`.
pub async fn target_folder_cfg(
pub async fn target_folder(
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
is_spam_folder: bool,
headers: &[mailparse::MailHeader<'_>],
) -> Result<Option<Config>> {
if context.is_mvbox(folder).await? {
return Ok(None);
}
if folder_meaning == FolderMeaning::Spam {
spam_target_folder_cfg(context, headers).await
if is_spam_folder {
spam_target_folder(context, headers).await
} else if needs_move_to_mvbox(context, headers).await? {
Ok(Some(Config::ConfiguredMvboxFolder))
} else {
@@ -1846,21 +1816,6 @@ pub async fn target_folder_cfg(
}
}
pub async fn target_folder(
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
headers: &[mailparse::MailHeader<'_>],
) -> Result<String> {
match target_folder_cfg(context, folder, folder_meaning, headers).await? {
Some(config) => match context.get_config(config).await? {
Some(target) => Ok(target),
None => Ok(folder.to_string()),
},
None => Ok(folder.to_string()),
}
}
async fn needs_move_to_mvbox(
context: &Context,
headers: &[mailparse::MailHeader<'_>],
@@ -1985,10 +1940,10 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
}
}
fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning {
for attr in folder_attrs {
fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
for attr in folder_name.attributes() {
match attr {
NameAttribute::Trash => return FolderMeaning::Trash,
NameAttribute::Trash => return FolderMeaning::Other,
NameAttribute::Sent => return FolderMeaning::Sent,
NameAttribute::Junk => return FolderMeaning::Spam,
NameAttribute::Drafts => return FolderMeaning::Drafts,
@@ -2006,13 +1961,6 @@ fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning
FolderMeaning::Unknown
}
pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning {
match get_folder_meaning_by_attrs(folder.attributes()) {
FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()),
meaning => meaning,
}
}
/// Parses the headers from the FETCH result.
fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
match prefetch_msg.header() {
@@ -2057,6 +2005,7 @@ pub(crate) async fn prefetch_should_download(
headers: &[mailparse::MailHeader<'_>],
message_id: &str,
mut flags: impl Iterator<Item = Flag<'_>>,
show_emails: ShowEmails,
) -> Result<bool> {
if message::rfc724_mid_exists(context, message_id)
.await?
@@ -2116,9 +2065,6 @@ pub(crate) async fn prefetch_should_download(
})
.unwrap_or_default();
let show_emails =
ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default();
let show = is_autocrypt_setup_message
|| match show_emails {
ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
@@ -2326,7 +2272,7 @@ pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result
async fn should_ignore_folder(
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
is_spam_folder: bool,
) -> Result<bool> {
if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
return Ok(false);
@@ -2335,7 +2281,7 @@ async fn should_ignore_folder(
// Still respect the SentboxWatch setting.
return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
}
Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
Ok(!(context.is_mvbox(folder).await? || is_spam_folder))
}
/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
@@ -2618,13 +2564,14 @@ mod tests {
};
let (headers, _) = mailparse::parse_headers(bytes)?;
let actual = if let Some(config) =
target_folder_cfg(&t, folder, get_folder_meaning_by_name(folder), &headers).await?
{
t.get_config(config).await?
} else {
None
};
let is_spam_folder = folder == "Spam";
let actual =
if let Some(config) = target_folder(&t, folder, is_spam_folder, &headers).await? {
t.get_config(config).await?
} else {
None
};
let expected = if expected_destination == folder {
None

View File

@@ -11,9 +11,9 @@ use tokio::io::BufWriter;
use super::capabilities::Capabilities;
use super::session::Session;
use crate::context::Context;
use crate::login_param::build_tls;
use crate::net::connect_tcp;
use crate::net::session::SessionStream;
use crate::net::tls::wrap_tls;
use crate::socks::Socks5Config;
/// IMAP write and read timeout.
@@ -95,7 +95,8 @@ impl Client {
strict_tls: bool,
) -> Result<Self> {
let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?;
let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?;
let tls = build_tls(strict_tls);
let tls_stream = tls.connect(hostname, tcp_stream).await?;
let buffered_stream = BufWriter::new(tls_stream);
let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
let mut client = ImapClient::new(session_stream);
@@ -141,7 +142,9 @@ impl Client {
.context("STARTTLS command failed")?;
let tcp_stream = client.into_inner();
let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
let tls = build_tls(strict_tls);
let tls_stream = tls
.connect(hostname, tcp_stream)
.await
.context("STARTTLS upgrade failed")?;
@@ -162,7 +165,8 @@ impl Client {
let socks5_stream = socks5_config
.connect(context, domain, port, IMAP_TIMEOUT, strict_tls)
.await?;
let tls_stream = wrap_tls(strict_tls, domain, socks5_stream).await?;
let tls = build_tls(strict_tls);
let tls_stream = tls.connect(domain, socks5_stream).await?;
let buffered_stream = BufWriter::new(tls_stream);
let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
let mut client = ImapClient::new(session_stream);
@@ -217,7 +221,9 @@ impl Client {
.context("STARTTLS command failed")?;
let socks5_stream = client.into_inner();
let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream)
let tls = build_tls(strict_tls);
let tls_stream = tls
.connect(hostname, socks5_stream)
.await
.context("STARTTLS upgrade failed")?;
let buffered_stream = BufWriter::new(tls_stream);

View File

@@ -7,7 +7,7 @@ use futures_lite::FutureExt;
use super::session::Session;
use super::Imap;
use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
use crate::imap::client::IMAP_TIMEOUT;
use crate::{context::Context, scheduler::InterruptInfo};
const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
@@ -113,7 +113,6 @@ impl Imap {
&mut self,
context: &Context,
watch_folder: Option<String>,
folder_meaning: FolderMeaning,
) -> InterruptInfo {
// Idle using polling. This is also needed if we're not yet configured -
// in this case, we're waiting for a configure job (and an interrupt).
@@ -174,7 +173,7 @@ impl Imap {
// will have already fetched the messages so perform_*_fetch
// will not find any new.
match self
.fetch_new_messages(context, &watch_folder, folder_meaning, false)
.fetch_new_messages(context, &watch_folder, false, false)
.await
{
Ok(res) => {

View File

@@ -3,7 +3,7 @@ use std::{collections::BTreeMap, time::Instant};
use anyhow::{Context as _, Result};
use futures::stream::StreamExt;
use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
use super::{get_folder_meaning, get_folder_meaning_by_name};
use crate::config::Config;
use crate::imap::Imap;
use crate::log::LogExt;
@@ -33,7 +33,7 @@ impl Imap {
let mut folder_configs = BTreeMap::new();
for folder in folders {
let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
let folder_meaning = get_folder_meaning(&folder);
if folder_meaning == FolderMeaning::Virtual {
// Gmail has virtual folders that should be skipped. For example,
// emails appear in the inbox and under "All Mail" as soon as it is
@@ -53,22 +53,21 @@ impl Imap {
.or_insert_with(|| folder.name().to_string());
}
let folder_meaning = match folder_meaning {
FolderMeaning::Unknown => folder_name_meaning,
_ => folder_meaning,
};
let is_drafts = folder_meaning == FolderMeaning::Drafts
|| (folder_meaning == FolderMeaning::Unknown
&& folder_name_meaning == FolderMeaning::Drafts);
let is_spam_folder = folder_meaning == FolderMeaning::Spam
|| (folder_meaning == FolderMeaning::Unknown
&& folder_name_meaning == FolderMeaning::Spam);
// Don't scan folders that are watched anyway
if !watched_folders.contains(&folder.name().to_string())
&& folder_meaning != FolderMeaning::Drafts
&& folder_meaning != FolderMeaning::Trash
{
if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
let session = self.session.as_mut().context("no session")?;
// Drain leftover unsolicited EXISTS messages
session.server_sent_unsolicited_exists(context)?;
loop {
self.fetch_move_delete(context, folder.name(), folder_meaning)
self.fetch_move_delete(context, folder.name(), is_spam_folder)
.await
.ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
@@ -81,15 +80,15 @@ impl Imap {
}
}
// Set configs for necessary folders. Or reset if the folder was deleted.
for conf in [
Config::ConfiguredSentboxFolder,
Config::ConfiguredTrashFolder,
] {
context
.set_config(conf, folder_configs.get(&conf).map(|s| s.as_str()))
.await?;
}
// Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
context
.set_config(
Config::ConfiguredSentboxFolder,
folder_configs
.get(&Config::ConfiguredSentboxFolder)
.map(|s| s.as_str()),
)
.await?;
last_scan.replace(Instant::now());
Ok(true)

View File

@@ -1,5 +1,3 @@
//! # IMAP folder selection module.
use anyhow::Context as _;
use super::session::Session as ImapSession;

View File

@@ -5,7 +5,7 @@ use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use ::pgp::types::KeyTrait;
use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
use anyhow::{bail, ensure, format_err, Context as _, Result};
use futures::StreamExt;
use futures_lite::FutureExt;
use rand::{thread_rng, Rng};
@@ -427,6 +427,8 @@ async fn import_backup(
context.get_dbfile().display()
);
context.sql.config_cache.write().await.clear();
let mut archive = Archive::new(backup_file);
let mut entries = archive.entries()?;
@@ -506,9 +508,6 @@ fn get_next_backup_path(folder: &Path, backup_time: i64) -> Result<(PathBuf, Pat
bail!("could not create backup file, disk full?");
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
// get a fine backup file name (the name includes the date so that multiple backup instances are possible)
let now = time();
@@ -522,6 +521,13 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
.await?;
sql::housekeeping(context).await.ok_or_log(context);
context
.sql
.execute("VACUUM;", paramsv![])
.await
.map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
.ok();
ensure!(
context.scheduler.read().await.is_none(),
"cannot export backup, IO is running"
@@ -534,31 +540,11 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
dest_path.display(),
);
let path_str = temp_db_path
.to_str()
.with_context(|| format!("path {temp_db_path:?} is not valid unicode"))?;
context
.sql
.call(|conn| {
if let Err(err) = conn.execute("VACUUM", params![]) {
info!(context, "Vacuum failed, exporting anyway: {:#}.", err);
}
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok::<_, Error>(())
})
.await?;
.export(&temp_db_path, passphrase)
.await
.with_context(|| format!("failed to backup plaintext database to {temp_db_path:?}"))?;
let res = export_backup_inner(context, &temp_db_path, &temp_path).await;
@@ -783,10 +769,7 @@ where
#[cfg(test)]
mod tests {
use std::time::Duration;
use ::pgp::armor::BlockType;
use tokio::task;
use super::*;
use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
@@ -934,46 +917,6 @@ mod tests {
Ok(())
}
/// This is a regression test for
/// https://github.com/deltachat/deltachat-android/issues/2263
/// where the config cache wasn't reset properly after a backup.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_import_backup_reset_config_cache() -> Result<()> {
let backup_dir = tempfile::tempdir()?;
let context1 = TestContext::new_alice().await;
let context2 = TestContext::new().await;
assert!(!context2.is_configured().await?);
// export from context1
imex(&context1, ImexMode::ExportBackup, backup_dir.path(), None).await?;
// import to context2
let backup = has_backup(&context2, backup_dir.path()).await?;
let context2_cloned = context2.clone();
let handle = task::spawn(async move {
imex(
&context2_cloned,
ImexMode::ImportBackup,
backup.as_ref(),
None,
)
.await
.unwrap();
});
while !handle.is_finished() {
// The database is still unconfigured;
// fill the config cache with the old value.
context2.is_configured().await.ok();
tokio::time::sleep(Duration::from_micros(1)).await;
}
// Assert that the config cache has the new value now.
assert!(context2.is_configured().await?);
Ok(())
}
#[test]
fn test_normalize_setup_code() {
let norm = normalize_setup_code("123422343234423452346234723482349234");

View File

@@ -12,7 +12,8 @@ use deltachat_derive::{FromSql, ToSql};
use rand::{thread_rng, Rng};
use crate::context::Context;
use crate::imap::{get_folder_meaning, FolderMeaning, Imap};
use crate::imap::Imap;
use crate::param::Params;
use crate::scheduler::InterruptInfo;
use crate::tools::time;
@@ -57,6 +58,9 @@ macro_rules! job_try {
)]
#[repr(u32)]
pub enum Action {
// this is user initiated so it should have a fairly high priority
UpdateRecentQuota = 140,
// This job will download partially downloaded messages completely
// and is added when download_full() is called.
// Most messages are downloaded automatically on fetch
@@ -76,6 +80,7 @@ pub struct Job {
pub desired_timestamp: i64,
pub added_timestamp: i64,
pub tries: u32,
pub param: Params,
}
impl fmt::Display for Job {
@@ -85,19 +90,24 @@ impl fmt::Display for Job {
}
impl Job {
pub fn new(action: Action, foreign_id: u32) -> Self {
pub fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self {
let timestamp = time();
Self {
job_id: 0,
action,
foreign_id,
desired_timestamp: timestamp,
desired_timestamp: timestamp + delay_seconds,
added_timestamp: timestamp,
tries: 0,
param,
}
}
pub fn delay_seconds(&self) -> i64 {
self.desired_timestamp - self.added_timestamp
}
/// Deletes the job from the database.
async fn delete(self, context: &Context) -> Result<()> {
if self.job_id != 0 {
@@ -120,21 +130,23 @@ impl Job {
context
.sql
.execute(
"UPDATE jobs SET desired_timestamp=?, tries=? WHERE id=?;",
"UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
paramsv![
self.desired_timestamp,
i64::from(self.tries),
self.param.to_string(),
self.job_id as i32,
],
)
.await?;
} else {
context.sql.execute(
"INSERT INTO jobs (added_timestamp, action, foreign_id, desired_timestamp) VALUES (?,?,?,?);",
"INSERT INTO jobs (added_timestamp, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?);",
paramsv![
self.added_timestamp,
self.action,
self.foreign_id,
self.param.to_string(),
self.desired_timestamp
]
).await?;
@@ -160,12 +172,8 @@ impl Job {
let mut any_failed = false;
for folder in all_folders {
let folder_meaning = get_folder_meaning(&folder);
if folder_meaning == FolderMeaning::Virtual {
continue;
}
if let Err(e) = imap
.resync_folder_uids(context, folder.name(), folder_meaning)
.resync_folder_uids(context, folder.name().to_string())
.await
{
warn!(context, "{:#}", e);
@@ -190,6 +198,17 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
Ok(())
}
pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
let exists = context
.sql
.exists(
"SELECT COUNT(*) FROM jobs WHERE action=?;",
paramsv![action],
)
.await?;
Ok(exists)
}
pub(crate) enum Connection<'a> {
Inbox(&'a mut Imap),
}
@@ -217,7 +236,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
if tries < JOB_RETRIES {
info!(context, "increase job {} tries to {}", job, tries);
job.tries = tries;
let time_offset = get_backoff_time_offset(tries);
let time_offset = get_backoff_time_offset(tries, job.action);
job.desired_timestamp = time() + time_offset;
info!(
context,
@@ -266,6 +285,10 @@ async fn perform_job_action(
let try_res = match job.action {
Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
Ok(status) => status,
Err(err) => Status::Finished(Err(err)),
},
Action::DownloadMsg => job.download_msg(context, connection.inbox()).await,
};
@@ -274,30 +297,50 @@ async fn perform_job_action(
try_res
}
fn get_backoff_time_offset(tries: u32) -> i64 {
// Exponential backoff
let n = 2_i32.pow(tries - 1) * 60;
let mut rng = thread_rng();
let r: i32 = rng.gen();
let mut seconds = r % (n + 1);
if seconds < 1 {
seconds = 1;
fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
match action {
// Just try every 10s to update the quota
// If all retries are exhausted, a new job will be created when the quota information is needed
Action::UpdateRecentQuota => 10,
_ => {
// Exponential backoff
let n = 2_i32.pow(tries - 1) * 60;
let mut rng = thread_rng();
let r: i32 = rng.gen();
let mut seconds = r % (n + 1);
if seconds < 1 {
seconds = 1;
}
i64::from(seconds)
}
}
i64::from(seconds)
}
pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
kill_action(context, Action::ResyncFolders).await?;
add(context, Job::new(Action::ResyncFolders, 0)).await?;
add(
context,
Job::new(Action::ResyncFolders, 0, Params::new(), 0),
)
.await?;
Ok(())
}
/// Adds a job to the database, scheduling it.
pub async fn add(context: &Context, job: Job) -> Result<()> {
let action = job.action;
let delay_seconds = job.delay_seconds();
job.save(context).await.context("failed to save job")?;
info!(context, "interrupt: imap");
context.interrupt_inbox(InterruptInfo::new(false)).await;
if delay_seconds == 0 {
match action {
Action::ResyncFolders | Action::UpdateRecentQuota | Action::DownloadMsg => {
info!(context, "interrupt: imap");
context.interrupt_inbox(InterruptInfo::new(false)).await;
}
}
}
Ok(())
}
@@ -349,6 +392,7 @@ LIMIT 1;
desired_timestamp: row.get("desired_timestamp")?,
added_timestamp: row.get("added_timestamp")?,
tries: row.get("tries")?,
param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
};
Ok(job)
@@ -388,8 +432,8 @@ mod tests {
.sql
.execute(
"INSERT INTO jobs
(added_timestamp, action, foreign_id, desired_timestamp)
VALUES (?, ?, ?, ?);",
(added_timestamp, action, foreign_id, param, desired_timestamp)
VALUES (?, ?, ?, ?, ?);",
paramsv![
now,
if valid {
@@ -398,6 +442,7 @@ mod tests {
-1
},
foreign_id,
Params::new().to_string(),
now
],
)

View File

@@ -289,41 +289,39 @@ pub async fn store_self_keypair(
keypair: &KeyPair,
default: KeyPairUse,
) -> Result<()> {
context
.sql
.transaction(|transaction| {
let public_key = DcKey::to_bytes(&keypair.public);
let secret_key = DcKey::to_bytes(&keypair.secret);
transaction
.execute(
"DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
paramsv![public_key, secret_key],
)
.context("failed to remove old use of key")?;
if default == KeyPairUse::Default {
transaction
.execute("UPDATE keypairs SET is_default=0;", paramsv![])
.context("failed to clear default")?;
}
let is_default = match default {
KeyPairUse::Default => i32::from(true),
KeyPairUse::ReadOnly => i32::from(false),
};
let mut conn = context.sql.get_conn().await?;
let transaction = conn.transaction()?;
let addr = keypair.addr.to_string();
let t = time();
let public_key = DcKey::to_bytes(&keypair.public);
let secret_key = DcKey::to_bytes(&keypair.secret);
transaction
.execute(
"DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
paramsv![public_key, secret_key],
)
.context("failed to remove old use of key")?;
if default == KeyPairUse::Default {
transaction
.execute("UPDATE keypairs SET is_default=0;", paramsv![])
.context("failed to clear default")?;
}
let is_default = match default {
KeyPairUse::Default => i32::from(true),
KeyPairUse::ReadOnly => i32::from(false),
};
transaction
.execute(
"INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
let addr = keypair.addr.to_string();
let t = time();
transaction
.execute(
"INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
VALUES (?,?,?,?,?);",
paramsv![addr, is_default, public_key, secret_key, t],
)
.context("failed to insert keypair")?;
paramsv![addr, is_default, public_key, secret_key, t],
)
.context("failed to insert keypair")?;
Ok(())
})
.await?;
transaction.commit()?;
Ok(())
}

View File

@@ -92,7 +92,6 @@ mod smtp;
mod socks;
pub mod stock_str;
mod sync;
mod timesmearing;
mod token;
mod update_helper;
pub mod webxdc;

View File

@@ -601,38 +601,32 @@ pub(crate) async fn save(
..
} = location;
context
.sql
.call(|conn| {
let mut stmt_test = conn
.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
let conn = context.sql.get_conn().await?;
let mut stmt_test =
conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
let exists = stmt_test.exists(paramsv![timestamp, contact_id])?;
let exists = stmt_test.exists(paramsv![timestamp, contact_id])?;
if independent || !exists {
stmt_insert.execute(paramsv![
timestamp,
contact_id,
chat_id,
latitude,
longitude,
accuracy,
independent,
])?;
if independent || !exists {
stmt_insert.execute(paramsv![
timestamp,
contact_id,
chat_id,
latitude,
longitude,
accuracy,
independent,
])?;
if timestamp > newest_timestamp {
// okay to drop, as we use cached prepared statements
drop(stmt_test);
drop(stmt_insert);
newest_timestamp = timestamp;
newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?);
}
}
Ok(())
})
.await?;
if timestamp > newest_timestamp {
// okay to drop, as we use cached prepared statements
drop(stmt_test);
drop(stmt_insert);
newest_timestamp = timestamp;
newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?);
}
}
}
Ok(newest_location_id)

View File

@@ -3,6 +3,8 @@
use std::fmt;
use anyhow::{ensure, Result};
use async_native_tls::Certificate;
use once_cell::sync::Lazy;
use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
use crate::provider::{get_provider_by_id, Provider};
@@ -304,6 +306,28 @@ fn unset_empty(s: &str) -> &str {
}
}
// this certificate is missing on older android devices (eg. lg with android6 from 2017)
// certificate downloaded from https://letsencrypt.org/certificates/
static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
Certificate::from_der(include_bytes!(
"../assets/root-certificates/letsencrypt/isrgrootx1.der"
))
.unwrap()
});
pub fn build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
let tls_builder =
async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone());
if strict_tls {
tls_builder
} else {
tls_builder
.danger_accept_invalid_hostnames(true)
.danger_accept_invalid_certs(true)
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -354,4 +378,13 @@ mod tests {
assert_eq!(param, loaded);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_build_tls() -> Result<()> {
// we are using some additional root certificates.
// make sure, they do not break construction of TlsConnector
let _ = build_tls(true);
let _ = build_tls(false);
Ok(())
}
}

View File

@@ -265,8 +265,6 @@ pub struct Message {
pub(crate) text: Option<String>,
/// Message subject.
///
/// If empty, a default subject will be generated when sending.
pub(crate) subject: String,
/// `Message-ID` header value.
@@ -797,12 +795,6 @@ impl Message {
self.text = text;
}
/// Sets the email's subject. If it's empty, a default subject
/// will be used (e.g. `Message from Alice` or `Re: <last subject>`).
pub fn set_subject(&mut self, subject: String) {
self.subject = subject;
}
/// Sets the file associated with a message.
///
/// This function does not use the file or check if it exists,
@@ -1386,12 +1378,11 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
context.emit_event(EventType::WebxdcInstanceDeleted { msg_id: *msg_id });
}
let target = context.get_delete_msgs_target().await?;
context
.sql
.execute(
"UPDATE imap SET target=? WHERE rfc724_mid=?",
paramsv![target, msg.rfc724_mid],
"UPDATE imap SET target='' WHERE rfc724_mid=?",
paramsv![msg.rfc724_mid],
)
.await?;
@@ -1771,8 +1762,13 @@ async fn ndn_maybe_add_info_msg(
// Tell the user which of the recipients failed if we know that (because in
// a group, this might otherwise be unclear)
let text = stock_str::failed_sending_to(context, contact.get_display_name()).await;
chat::add_info_msg(context, chat_id, &text, create_smeared_timestamp(context))
.await?;
chat::add_info_msg(
context,
chat_id,
&text,
create_smeared_timestamp(context).await,
)
.await?;
context.emit_event(EventType::ChatModified(chat_id));
}
}

View File

@@ -250,7 +250,7 @@ impl<'a> MimeFactory<'a> {
.get_config(Config::Selfstatus)
.await?
.unwrap_or_default();
let timestamp = create_smeared_timestamp(context);
let timestamp = create_smeared_timestamp(context).await;
let res = MimeFactory::<'a> {
from_addr,
@@ -779,36 +779,10 @@ impl<'a> MimeFactory<'a> {
};
// Store protected headers in the outer message.
let message = headers
headers
.protected
.into_iter()
.fold(message, |message, header| message.header(header));
if self.should_skip_autocrypt()
|| !context.get_config_bool(Config::SignUnencrypted).await?
{
message
} else {
let (payload, signature) = encrypt_helper.sign(context, message).await?;
PartBuilder::new()
.header((
"Content-Type".to_string(),
"multipart/signed; protocol=\"application/pgp-signature\"".to_string(),
))
.child(payload)
.child(
PartBuilder::new()
.content_type(
&"application/pgp-signature; name=\"signature.asc\""
.parse::<mime::Mime>()
.unwrap(),
)
.header(("Content-Description", "OpenPGP digital signature"))
.header(("Content-Disposition", "attachment; filename=\"signature\";"))
.body(signature)
.build(),
)
}
.fold(message, |message, header| message.header(header))
};
// Store the unprotected headers on the outer message.
@@ -1638,22 +1612,6 @@ mod tests {
assert_eq!(maybe_encode_words("äöü"), "=?utf-8?b?w6TDtsO8?=");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_manually_set_subject() -> Result<()> {
let t = TestContext::new_alice().await;
let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
let mut msg = Message::new(Viewtype::Text);
msg.set_subject("Subjeeeeect".to_string());
let sent_msg = t.send_msg(chat.id, &mut msg).await;
let payload = sent_msg.payload();
assert_eq!(payload.match_indices("Subject: Subjeeeeect").count(), 1);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_subject_from_mua() {
// 1.: Receive a mail from an MUA
@@ -2166,96 +2124,6 @@ mod tests {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_selfavatar_unencrypted_signed() {
// create chat with bob, set selfavatar
let t = TestContext::new_alice().await;
t.set_config(Config::SignUnencrypted, Some("1"))
.await
.unwrap();
let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
let file = t.dir.path().join("avatar.png");
let bytes = include_bytes!("../test-data/image/avatar64x64.png");
tokio::fs::write(&file, bytes).await.unwrap();
t.set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
.await
.unwrap();
// send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
// make sure, `Subject:` stays in the outer header (imf header)
let mut msg = Message::new(Viewtype::Text);
msg.set_text(Some("this is the text!".to_string()));
let sent_msg = t.send_msg(chat.id, &mut msg).await;
let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n");
let part = payload.next().unwrap();
assert_eq!(part.match_indices("multipart/signed").count(), 1);
assert_eq!(part.match_indices("Subject:").count(), 0);
assert_eq!(part.match_indices("Autocrypt:").count(), 1);
assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
let part = payload.next().unwrap();
assert_eq!(part.match_indices("multipart/mixed").count(), 1);
assert_eq!(part.match_indices("Subject:").count(), 1);
assert_eq!(part.match_indices("Autocrypt:").count(), 0);
assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
let part = payload.next().unwrap();
assert_eq!(part.match_indices("text/plain").count(), 1);
assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1);
assert_eq!(part.match_indices("Subject:").count(), 0);
let body = payload.next().unwrap();
assert_eq!(body.match_indices("this is the text!").count(), 1);
let bob = TestContext::new_bob().await;
bob.recv_msg(&sent_msg).await;
let alice_id = Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown)
.await
.unwrap()
.unwrap();
let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap();
assert!(alice_contact
.get_profile_image(&bob.ctx)
.await
.unwrap()
.is_some());
// if another message is sent, that one must not contain the avatar
// and no artificial multipart/mixed nesting
let sent_msg = t.send_msg(chat.id, &mut msg).await;
let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
let part = payload.next().unwrap();
assert_eq!(part.match_indices("multipart/signed").count(), 1);
assert_eq!(part.match_indices("Subject:").count(), 0);
assert_eq!(part.match_indices("Autocrypt:").count(), 1);
assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
let part = payload.next().unwrap();
assert_eq!(part.match_indices("text/plain").count(), 1);
assert_eq!(part.match_indices("Subject:").count(), 1);
assert_eq!(part.match_indices("Autocrypt:").count(), 0);
assert_eq!(part.match_indices("multipart/mixed").count(), 0);
assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
let body = payload.next().unwrap();
assert_eq!(body.match_indices("this is the text!").count(), 1);
assert_eq!(body.match_indices("text/plain").count(), 0);
assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
assert_eq!(body.match_indices("Subject:").count(), 0);
bob.recv_msg(&sent_msg).await;
let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap();
assert!(alice_contact
.get_profile_image(&bob.ctx)
.await
.unwrap()
.is_some());
}
/// Test that removed member address does not go into the `To:` field.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_remove_member_bcc() -> Result<()> {

View File

@@ -224,32 +224,8 @@ impl MimeMessage {
// Parse hidden headers.
let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
let (part, mimetype) =
if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "signed" {
if let Some(part) = mail.subparts.first() {
// We don't remove "subject" from `headers` because currently just signed
// messages are shown as unencrypted anyway.
MimeMessage::merge_headers(
context,
&mut headers,
&mut recipients,
&mut from,
&mut list_post,
&mut chat_disposition_notification_to,
&part.headers,
);
(part, part.ctype.mimetype.parse::<Mime>()?)
} else {
// If it's a partially fetched message, there are no subparts.
(&mail, mimetype)
}
} else {
// Currently we do not sign unencrypted messages by default.
(&mail, mimetype)
};
if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" {
if let Some(part) = part.subparts.first() {
if let Some(part) = mail.subparts.first() {
for field in &part.headers {
let key = field.get_key().to_lowercase();
@@ -280,27 +256,26 @@ impl MimeMessage {
hop_info += &decryption_info.dkim_results.to_string();
let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref());
let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| {
try_decrypt(context, &mail, &private_keyring, &public_keyring)
}) {
Ok(Some((raw, signatures))) => {
mail_raw = raw;
let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(
context,
"decrypted message mime-body:\n{}",
String::from_utf8_lossy(&mail_raw),
);
let (mail, mut signatures, encrypted) =
match try_decrypt(context, &mail, &private_keyring, &public_keyring) {
Ok(Some((raw, signatures))) => {
mail_raw = raw;
let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(
context,
"decrypted message mime-body:\n{}",
String::from_utf8_lossy(&mail_raw),
);
}
(Ok(decrypted_mail), signatures, true)
}
(Ok(decrypted_mail), signatures, true)
}
Ok(None) => (Ok(mail), HashSet::new(), false),
Err(err) => {
warn!(context, "decryption failed: {:#}", err);
(Err(err), HashSet::new(), false)
}
};
Ok(None) => (Ok(mail), HashSet::new(), false),
Err(err) => {
warn!(context, "decryption failed: {:#}", err);
(Err(err), HashSet::new(), false)
}
};
let mail = mail.as_ref().map(|mail| {
let (content, signatures_detached) = validate_detached_signature(mail, &public_keyring)
.unwrap_or((mail, Default::default()));
@@ -367,8 +342,7 @@ impl MimeMessage {
if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
if message_time > peerstate.last_seen_autocrypt
&& mail.ctype.mimetype != "multipart/report"
// Disallowing keychanges is disabled for now:
// && decryption_info.dkim_results.allow_keychange
&& decryption_info.dkim_results.allow_keychange
{
peerstate.degrade_encryption(message_time);
}
@@ -439,12 +413,11 @@ impl MimeMessage {
parser.heuristically_parse_ndn(context).await;
parser.parse_headers(context).await?;
// Disallowing keychanges is disabled for now
// if !decryption_info.dkim_results.allow_keychange {
// for part in parser.parts.iter_mut() {
// part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string());
// }
// }
if !parser.decryption_info.dkim_results.allow_keychange {
for part in parser.parts.iter_mut() {
part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string());
}
}
if parser.is_mime_modified {
parser.decoded_data = mail_raw;

View File

@@ -13,7 +13,6 @@ use crate::context::Context;
use crate::tools::time;
pub(crate) mod session;
pub(crate) mod tls;
async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result<TcpStream> {
let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))

View File

@@ -1,52 +0,0 @@
//! TLS support.
use anyhow::Result;
use async_native_tls::{Certificate, Protocol, TlsConnector, TlsStream};
use once_cell::sync::Lazy;
use tokio::io::{AsyncRead, AsyncWrite};
// this certificate is missing on older android devices (eg. lg with android6 from 2017)
// certificate downloaded from https://letsencrypt.org/certificates/
static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
Certificate::from_der(include_bytes!(
"../../assets/root-certificates/letsencrypt/isrgrootx1.der"
))
.unwrap()
});
pub fn build_tls(strict_tls: bool) -> TlsConnector {
let tls_builder = TlsConnector::new()
.min_protocol_version(Some(Protocol::Tlsv12))
.add_root_certificate(LETSENCRYPT_ROOT.clone());
if strict_tls {
tls_builder
} else {
tls_builder
.danger_accept_invalid_hostnames(true)
.danger_accept_invalid_certs(true)
}
}
pub async fn wrap_tls<T: AsyncRead + AsyncWrite + Unpin>(
strict_tls: bool,
hostname: &str,
stream: T,
) -> Result<TlsStream<T>> {
let tls = build_tls(strict_tls);
let tls_stream = tls.connect(hostname, stream).await?;
Ok(tls_stream)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_build_tls() {
// we are using some additional root certificates.
// make sure, they do not break construction of TlsConnector
let _ = build_tls(true);
let _ = build_tls(false);
}
}

View File

@@ -186,7 +186,7 @@ impl Peerstate {
async fn from_stmt(
context: &Context,
query: &str,
params: impl rusqlite::Params + Send,
params: impl rusqlite::Params,
) -> Result<Option<Peerstate>> {
let peerstate = context
.sql
@@ -688,10 +688,6 @@ pub(crate) async fn maybe_do_aeap_transition(
Ok(())
}
/// Type of the peerstate change.
///
/// Changes to the peerstate are notified to the user via a message
/// explaining the happened change.
enum PeerstateChange {
/// The contact's public key fingerprint changed, likely because
/// the contact uses a new device and didn't transfer their key.

View File

@@ -262,20 +262,6 @@ pub async fn pk_encrypt(
.await?
}
/// Signs `plain` text using `private_key_for_signing`.
pub fn pk_calc_signature(
plain: &[u8],
private_key_for_signing: &SignedSecretKey,
) -> Result<String> {
let msg = Message::new_literal_bytes("", plain).sign(
private_key_for_signing,
|| "".into(),
Default::default(),
)?;
let signature = msg.into_signature().to_armored_string(None)?;
Ok(signature)
}
/// Decrypts the message with keys from the private key keyring.
///
/// Receiver private keys are provided in

View File

@@ -129,16 +129,6 @@ pub struct Provider {
/// Default configuration values to set when provider is configured.
pub config_defaults: Option<Vec<ConfigDefault>>,
/// Type of OAuth 2 authorization if provider supports it.
pub oauth2_authorizer: Option<Oauth2Authorizer>,
/// Options with good defaults.
pub opt: ProviderOptions,
}
/// Provider options with good defaults.
#[derive(Debug, PartialEq, Eq)]
pub struct ProviderOptions {
/// True if provider is known to use use proper,
/// not self-signed certificates.
pub strict_tls: bool,
@@ -146,18 +136,8 @@ pub struct ProviderOptions {
/// Maximum number of recipients the provider allows to send a single email to.
pub max_smtp_rcpt_to: Option<u16>,
/// Move messages to the Trash folder instead of marking them "\Deleted".
pub delete_to_trash: bool,
}
impl Default for ProviderOptions {
fn default() -> Self {
Self {
strict_tls: true,
max_smtp_rcpt_to: None,
delete_to_trash: false,
}
}
/// Type of OAuth 2 authorization if provider supports it.
pub oauth2_authorizer: Option<Oauth2Authorizer>,
}
/// Get resolver to query MX records.

View File

@@ -1,14 +1,13 @@
// file generated by src/provider/update.py
use std::collections::HashMap;
use once_cell::sync::Lazy;
use crate::provider::Protocol::*;
use crate::provider::Socket::*;
use crate::provider::UsernamePattern::*;
use crate::provider::{
Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,
};
use std::collections::HashMap;
use once_cell::sync::Lazy;
use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};
// 163.md: 163.com
static P_163: Lazy<Provider> = Lazy::new(|| Provider {
@@ -33,8 +32,9 @@ static P_163: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -61,8 +61,9 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -78,8 +79,9 @@ static P_AOL: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.aol.com", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "smtp.aol.com", port: 465, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -107,8 +109,9 @@ static P_ARCOR_DE: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -135,8 +138,9 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -163,8 +167,9 @@ static P_BLINDZELN_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -191,8 +196,9 @@ static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -219,8 +225,9 @@ static P_BUZON_UY: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -247,8 +254,9 @@ static P_CHELLO_AT: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -260,8 +268,9 @@ static P_COMCAST: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/comcast",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -273,8 +282,9 @@ static P_DISMAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/dismail-de",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -301,8 +311,9 @@ static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -329,8 +340,9 @@ static P_E_EMAIL: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -342,8 +354,9 @@ static P_ESPIV_NET: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/espiv-net",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -359,8 +372,9 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.example.com", port: 1337, username_pattern: Email },
Server { protocol: Smtp, socket: Starttls, hostname: "smtp.example.com", port: 1337, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -389,8 +403,9 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -404,8 +419,9 @@ static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
overview_page: "https://providers.delta.chat/firemail-de",
server: vec![
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -418,7 +434,6 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/five-chat",
server: vec![],
opt: Default::default(),
config_defaults: Some(vec![
ConfigDefault {
key: Config::BccSelf,
@@ -433,6 +448,8 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
value: "0",
},
]),
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -448,8 +465,9 @@ static P_FREENET_DE: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "mx.freenet.de", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Starttls, hostname: "mx.freenet.de", port: 587, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -466,11 +484,9 @@ static P_GMAIL: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.gmail.com", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "smtp.gmail.com", port: 465, username_pattern: Email },
],
opt: ProviderOptions {
delete_to_trash: true,
..Default::default()
},
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
}
});
@@ -505,8 +521,9 @@ static P_GMX_NET: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -518,10 +535,6 @@ static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/hermes-radio",
server: vec![],
opt: ProviderOptions {
strict_tls: false,
..Default::default()
},
config_defaults: Some(vec![
ConfigDefault {
key: Config::MdnsEnabled,
@@ -536,6 +549,8 @@ static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
value: "2",
},
]),
strict_tls: false,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -549,8 +564,9 @@ static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
overview_page: "https://providers.delta.chat/hey-com",
server: vec![
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -563,8 +579,9 @@ static P_I_UA: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/i-ua",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -576,8 +593,9 @@ static P_I3_NET: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/i3-net",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -604,8 +622,9 @@ static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -632,11 +651,9 @@ static P_INFOMANIAK_COM: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: ProviderOptions {
max_smtp_rcpt_to: Some(10),
..Default::default()
},
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: Some(10),
oauth2_authorizer: None,
});
@@ -648,8 +665,9 @@ static P_KOLST_COM: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/kolst-com",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -661,8 +679,9 @@ static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/kontent-com",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -689,8 +708,9 @@ static P_MAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -706,8 +726,9 @@ static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.ru", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -735,8 +756,9 @@ static P_MAIL2TOR: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -763,8 +785,9 @@ static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -791,8 +814,9 @@ static P_MAILO_COM: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -819,11 +843,6 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: ProviderOptions {
max_smtp_rcpt_to: Some(20),
strict_tls: false,
..Default::default()
},
config_defaults: Some(vec![
ConfigDefault {
key: Config::DeleteServerAfter,
@@ -850,6 +869,8 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
value: "0",
},
]),
strict_tls: false,
max_smtp_rcpt_to: Some(20),
oauth2_authorizer: None,
});
@@ -876,8 +897,9 @@ static P_NAVER: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -904,8 +926,9 @@ static P_NUBO_COOP: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -932,8 +955,9 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -960,8 +984,9 @@ static P_OUVATON_COOP: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -973,13 +998,6 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/posteo",
server: vec![
Server {
protocol: Imap,
socket: Ssl,
hostname: "posteo.de",
port: 993,
username_pattern: Email,
},
Server {
protocol: Imap,
socket: Starttls,
@@ -987,13 +1005,6 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
port: 143,
username_pattern: Email,
},
Server {
protocol: Smtp,
socket: Ssl,
hostname: "posteo.de",
port: 465,
username_pattern: Email,
},
Server {
protocol: Smtp,
socket: Starttls,
@@ -1002,8 +1013,9 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1017,8 +1029,9 @@ static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
overview_page: "https://providers.delta.chat/protonmail",
server: vec![
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1035,8 +1048,9 @@ static P_QQ: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart },
Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1064,8 +1078,9 @@ static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1077,21 +1092,9 @@ static P_ROGERS_COM: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/rogers-com",
server: vec![],
opt: Default::default(),
config_defaults: None,
oauth2_authorizer: None,
});
// sonic.md: sonic.net
static P_SONIC: Lazy<Provider> = Lazy::new(|| Provider {
id: "sonic",
status: Status::Ok,
before_login_hint: "",
after_login_hint: "",
overview_page: "https://providers.delta.chat/sonic",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1118,8 +1121,9 @@ static P_SYSTEMAUSFALL_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1146,8 +1150,9 @@ static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1163,8 +1168,9 @@ static P_T_ONLINE: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "secureimap.t-online.de", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "securesmtp.t-online.de", port: 465, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1199,7 +1205,6 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: Some(vec![
ConfigDefault {
key: Config::BccSelf,
@@ -1214,6 +1219,8 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
value: "0",
},
]),
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1240,8 +1247,9 @@ static P_TISCALI_IT: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1255,8 +1263,9 @@ static P_TUTANOTA: Lazy<Provider> = Lazy::new(|| {
overview_page: "https://providers.delta.chat/tutanota",
server: vec![
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1269,8 +1278,9 @@ static P_UKR_NET: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/ukr-net",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1297,8 +1307,9 @@ static P_UNDERNET_UY: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1310,8 +1321,9 @@ static P_VFEMAIL: Lazy<Provider> = Lazy::new(|| Provider {
after_login_hint: "",
overview_page: "https://providers.delta.chat/vfemail",
server: vec![],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1338,8 +1350,9 @@ static P_VIVALDI: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1366,8 +1379,9 @@ static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1384,8 +1398,9 @@ static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Starttls, hostname: "imap.web.de", port: 143, username_pattern: Emaillocalpart },
Server { protocol: Smtp, socket: Starttls, hostname: "smtp.web.de", port: 587, username_pattern: Emaillocalpart },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1402,8 +1417,9 @@ static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: Email },
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1431,8 +1447,9 @@ static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
});
@@ -1448,10 +1465,11 @@ static P_YGGMAIL: Lazy<Provider> = Lazy::new(|| {
Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email },
Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email },
],
opt: Default::default(),
config_defaults: Some(vec![
ConfigDefault { key: Config::MvboxMove, value: "0" },
]),
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
}
});
@@ -1479,8 +1497,9 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1507,8 +1526,9 @@ static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
username_pattern: Email,
},
],
opt: Default::default(),
config_defaults: None,
strict_tls: true,
max_smtp_rcpt_to: None,
oauth2_authorizer: None,
});
@@ -1802,7 +1822,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
("foxmail.com", &*P_QQ),
("riseup.net", &*P_RISEUP_NET),
("rogers.com", &*P_ROGERS_COM),
("sonic.net", &*P_SONIC),
("systemausfall.org", &*P_SYSTEMAUSFALL_ORG),
("solidaris.me", &*P_SYSTEMAUSFALL_ORG),
("systemli.org", &*P_SYSTEMLI_ORG),
@@ -1927,7 +1946,6 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
("qq", &*P_QQ),
("riseup.net", &*P_RISEUP_NET),
("rogers.com", &*P_ROGERS_COM),
("sonic", &*P_SONIC),
("systemausfall.org", &*P_SYSTEMAUSFALL_ORG),
("systemli.org", &*P_SYSTEMLI_ORG),
("t-online", &*P_T_ONLINE),
@@ -1952,4 +1970,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
});
pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 21).unwrap());
Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 1, 6).unwrap());

View File

@@ -40,23 +40,6 @@ def file2url(f):
return "https://providers.delta.chat/" + f
def process_opt(data):
if not "opt" in data:
return "Default::default()"
opt = "ProviderOptions {\n"
opt_data = data.get("opt", "")
for key in opt_data:
value = str(opt_data[key])
if key == "max_smtp_rcpt_to":
value = "Some(" + value + ")"
if value in {"True", "False"}:
value = value.lower()
opt += " " + key + ": " + value + ",\n"
opt += " ..Default::default()\n"
opt += " }"
return opt
def process_config_defaults(data):
if not "config_defaults" in data:
return "None"
@@ -123,9 +106,14 @@ def process_data(data, file):
server += (" Server { protocol: " + protocol.capitalize() + ", socket: " + socket.capitalize() + ", hostname: \""
+ hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern.capitalize() + " },\n")
opt = process_opt(data)
config_defaults = process_config_defaults(data)
strict_tls = data.get("strict_tls", True)
strict_tls = "true" if strict_tls else "false"
max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0)
max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None"
oauth2 = data.get("oauth2", "")
oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
@@ -140,8 +128,9 @@ def process_data(data, file):
provider += " after_login_hint: \"" + after_login_hint + "\",\n"
provider += " overview_page: \"" + file2url(file) + "\",\n"
provider += " server: vec![\n" + server + " ],\n"
provider += " opt: " + opt + ",\n"
provider += " config_defaults: " + config_defaults + ",\n"
provider += " strict_tls: " + strict_tls + ",\n"
provider += " max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n"
provider += " oauth2_authorizer: " + oauth2 + ",\n"
provider += "});\n\n"
else:
@@ -185,9 +174,7 @@ if __name__ == "__main__":
"use crate::provider::Protocol::*;\n"
"use crate::provider::Socket::*;\n"
"use crate::provider::UsernamePattern::*;\n"
"use crate::provider::{\n"
" Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,\n"
"};\n"
"use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};\n"
"use std::collections::HashMap;\n\n"
"use once_cell::sync::Lazy;\n\n")

View File

@@ -473,15 +473,11 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result<Qr> {
#[derive(Debug, Deserialize)]
struct CreateAccountSuccessResponse {
/// Email address.
email: String,
/// Password.
password: String,
}
#[derive(Debug, Deserialize)]
struct CreateAccountErrorResponse {
/// Reason for the failure to create account returned by the server.
reason: String,
}

View File

@@ -1,7 +1,6 @@
//! # Support for IMAP QUOTA extension.
use std::collections::BTreeMap;
use std::sync::atomic::Ordering;
use anyhow::{anyhow, Context as _, Result};
use async_imap::types::{Quota, QuotaResource};
@@ -12,10 +11,11 @@ use crate::context::Context;
use crate::imap::scan_folders::get_watched_folders;
use crate::imap::session::Session as ImapSession;
use crate::imap::Imap;
use crate::job::{Action, Status};
use crate::message::{Message, Viewtype};
use crate::scheduler::InterruptInfo;
use crate::param::Params;
use crate::tools::time;
use crate::{stock_str, EventType};
use crate::{job, stock_str, EventType};
/// warn about a nearly full mailbox after this usage percentage is reached.
/// quota icon is "yellow".
@@ -112,10 +112,12 @@ pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> b
impl Context {
// Adds a job to update `quota.recent`
pub(crate) async fn schedule_quota_update(&self) -> Result<()> {
let requested = self.quota_update_request.swap(true, Ordering::Relaxed);
if !requested {
// Quota update was not requested before.
self.interrupt_inbox(InterruptInfo::new(false)).await;
if !job::action_exists(self, Action::UpdateRecentQuota).await? {
job::add(
self,
job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
)
.await?;
}
Ok(())
}
@@ -130,10 +132,10 @@ impl Context {
/// and new space is allocated as needed.
///
/// Called in response to `Action::UpdateRecentQuota`.
pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> {
pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<Status> {
if let Err(err) = imap.prepare(self).await {
warn!(self, "could not connect: {:#}", err);
return Ok(());
return Ok(Status::RetryNow);
}
let session = imap.session.as_mut().context("no session")?;
@@ -164,16 +166,13 @@ impl Context {
}
}
// Clear the request to update quota.
self.quota_update_request.store(false, Ordering::Relaxed);
*self.quota.write().await = Some(QuotaInfo {
recent: quota,
modified: time(),
});
self.emit_event(EventType::ConnectivityChanged);
Ok(())
Ok(Status::Finished(Ok(())))
}
}

View File

@@ -203,7 +203,7 @@ pub(crate) async fn receive_imf_inner(
)
.await?;
let rcvd_timestamp = smeared_time(context);
let rcvd_timestamp = smeared_time(context).await;
// Sender timestamp is allowed to be a bit in the future due to
// unsynchronized clocks, but not too much.
@@ -342,12 +342,11 @@ pub(crate) async fn receive_imf_inner(
if received_msg.needs_delete_job
|| (delete_server_after == Some(0) && is_partial_download.is_none())
{
let target = context.get_delete_msgs_target().await?;
context
.sql
.execute(
"UPDATE imap SET target=? WHERE rfc724_mid=?",
paramsv![target, rfc724_mid],
"UPDATE imap SET target='' WHERE rfc724_mid=?",
paramsv![rfc724_mid],
)
.await?;
} else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
@@ -1085,6 +1084,8 @@ async fn add_parts(
let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
let conn = context.sql.get_conn().await?;
for part in &mime_parser.parts {
if part.is_reaction {
set_msg_reaction(
@@ -1116,6 +1117,39 @@ async fn add_parts(
}
let mut txt_raw = "".to_string();
let mut stmt = conn.prepare_cached(
r#"
INSERT INTO msgs
(
id,
rfc724_mid, chat_id,
from_id, to_id, timestamp, timestamp_sent,
timestamp_rcvd, type, state, msgrmsg,
txt, subject, txt_raw, param,
bytes, mime_headers, mime_in_reply_to,
mime_references, mime_modified, error, ephemeral_timer,
ephemeral_timestamp, download_state, hop_info
)
VALUES (
?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?
)
ON CONFLICT (id) DO UPDATE
SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent,
timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg,
txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to,
mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info
"#,
)?;
let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg {
(better_msg, Viewtype::Text)
} else {
@@ -1149,91 +1183,56 @@ async fn add_parts(
// also change `MsgId::trash()` and `delete_expired_messages()`
let trash = chat_id.is_trash() || (is_location_kml && msg.is_empty());
let row_id = context
.sql
.call(|conn| {
let mut stmt = conn.prepare_cached(
r#"
INSERT INTO msgs
(
id,
rfc724_mid, chat_id,
from_id, to_id, timestamp, timestamp_sent,
timestamp_rcvd, type, state, msgrmsg,
txt, subject, txt_raw, param,
bytes, mime_headers, mime_in_reply_to,
mime_references, mime_modified, error, ephemeral_timer,
ephemeral_timestamp, download_state, hop_info
)
VALUES (
?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?
)
ON CONFLICT (id) DO UPDATE
SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent,
timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg,
txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to,
mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info
"#)?;
stmt.execute(params![
replace_msg_id,
rfc724_mid,
if trash { DC_CHAT_ID_TRASH } else { chat_id },
if trash { ContactId::UNDEFINED } else { from_id },
if trash { ContactId::UNDEFINED } else { to_id },
sort_timestamp,
sent_timestamp,
rcvd_timestamp,
typ,
state,
is_dc_message,
if trash { "" } else { msg },
if trash { "" } else { &subject },
// txt_raw might contain invalid utf8
if trash { "" } else { &txt_raw },
if trash {
"".to_string()
} else {
param.to_string()
},
part.bytes as isize,
if (save_mime_headers || mime_modified) && !trash {
mime_headers.clone()
} else {
Vec::new()
},
mime_in_reply_to,
mime_references,
mime_modified,
part.error.as_deref().unwrap_or_default(),
ephemeral_timer,
ephemeral_timestamp,
if is_partial_download.is_some() {
DownloadState::Available
} else {
DownloadState::Done
},
mime_parser.hop_info
])?;
let row_id = conn.last_insert_rowid();
Ok(row_id)
})
.await?;
stmt.execute(paramsv![
replace_msg_id,
rfc724_mid,
if trash { DC_CHAT_ID_TRASH } else { chat_id },
if trash { ContactId::UNDEFINED } else { from_id },
if trash { ContactId::UNDEFINED } else { to_id },
sort_timestamp,
sent_timestamp,
rcvd_timestamp,
typ,
state,
is_dc_message,
if trash { "" } else { msg },
if trash { "" } else { &subject },
// txt_raw might contain invalid utf8
if trash { "" } else { &txt_raw },
if trash {
"".to_string()
} else {
param.to_string()
},
part.bytes as isize,
if (save_mime_headers || mime_modified) && !trash {
mime_headers.clone()
} else {
Vec::new()
},
mime_in_reply_to,
mime_references,
mime_modified,
part.error.as_deref().unwrap_or_default(),
ephemeral_timer,
ephemeral_timestamp,
if is_partial_download.is_some() {
DownloadState::Available
} else {
DownloadState::Done
},
mime_parser.hop_info
])?;
// We only replace placeholder with a first part,
// afterwards insert additional parts.
replace_msg_id = None;
let row_id = conn.last_insert_rowid();
drop(stmt);
created_db_entries.push(MsgId::new(u32::try_from(row_id)?));
}
drop(conn);
// check all parts whether they contain a new logging webxdc
for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) {
@@ -1380,7 +1379,7 @@ async fn calc_sort_timestamp(
}
}
Ok(min(sort_timestamp, smeared_time(context)))
Ok(min(sort_timestamp, smeared_time(context).await))
}
async fn lookup_chat_by_reply(

View File

@@ -5,7 +5,7 @@ use crate::aheader::EncryptPreference;
use crate::chat::get_chat_contacts;
use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility};
use crate::chatlist::Chatlist;
use crate::constants::DC_GCL_NO_SPECIALS;
use crate::constants::{ShowEmails, DC_GCL_NO_SPECIALS};
use crate::imap::prefetch_should_download;
use crate::message::Message;
use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
@@ -647,11 +647,15 @@ async fn test_parse_ndn(
// Check that the ndn would be downloaded:
let headers = mailparse::parse_mail(raw_ndn).unwrap().headers;
assert!(
prefetch_should_download(&t, &headers, "some-other-message-id", std::iter::empty(),)
.await
.unwrap()
);
assert!(prefetch_should_download(
&t,
&headers,
"some-other-message-id",
std::iter::empty(),
ShowEmails::Off,
)
.await
.unwrap());
receive_imf(&t, raw_ndn, false).await.unwrap();
let msg = Message::load_from_db(&t, msg_id).await.unwrap();
@@ -2133,76 +2137,6 @@ Original signature updated",
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_ignore_old_status_updates() -> Result<()> {
let t = TestContext::new_alice().await;
let bob_id = Contact::add_or_lookup(
&t,
"",
ContactAddress::new("bob@example.net")?,
Origin::AddressBook,
)
.await?
.0;
receive_imf(
&t,
b"From: Bob <bob@example.net>
To: Alice <alice@example.org>
Message-ID: <2@example.org>
Date: Wed, 22 Feb 2023 20:00:00 +0000
body
--
sig wednesday",
false,
)
.await?;
let chat_id = t.get_last_msg().await.chat_id;
let bob = Contact::load_from_db(&t, bob_id).await?;
assert_eq!(bob.get_status(), "sig wednesday");
assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 1);
receive_imf(
&t,
b"From: Bob <bob@example.net>
To: Alice <alice@example.org>
Message-ID: <1@example.org>
Date: Tue, 21 Feb 2023 20:00:00 +0000
body
--
sig tuesday",
false,
)
.await?;
let bob = Contact::load_from_db(&t, bob_id).await?;
assert_eq!(bob.get_status(), "sig wednesday");
assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 2);
receive_imf(
&t,
b"From: Bob <bob@example.net>
To: Alice <alice@example.org>
Message-ID: <3@example.org>
Date: Thu, 23 Feb 2023 20:00:00 +0000
body
--
sig thursday",
false,
)
.await?;
let bob = Contact::load_from_db(&t, bob_id).await?;
assert_eq!(bob.get_status(), "sig thursday");
assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 3);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_chat_assignment_private_classical_reply() {
for outgoing_is_classical in &[true, false] {

View File

@@ -1,9 +1,6 @@
use std::iter::{self, once};
use std::sync::atomic::Ordering;
use anyhow::{bail, Context as _, Result};
use async_channel::{self as channel, Receiver, Sender};
use futures::future::try_join_all;
use futures::try_join;
use futures_lite::FutureExt;
use tokio::task;
@@ -12,7 +9,7 @@ use crate::config::Config;
use crate::contact::{ContactId, RecentlySeenLoop};
use crate::context::Context;
use crate::ephemeral::{self, delete_expired_imap_messages};
use crate::imap::{FolderMeaning, Imap};
use crate::imap::Imap;
use crate::job;
use crate::location;
use crate::log::LogExt;
@@ -23,19 +20,15 @@ use crate::tools::{duration_to_str, maybe_add_time_based_warnings};
pub(crate) mod connectivity;
#[derive(Debug)]
struct SchedBox {
meaning: FolderMeaning,
conn_state: ImapConnectionState,
handle: task::JoinHandle<()>,
}
/// Job and connection scheduler.
#[derive(Debug)]
pub(crate) struct Scheduler {
inbox: SchedBox,
/// Optional boxes -- mvbox, sentbox.
oboxes: Vec<SchedBox>,
inbox: ImapConnectionState,
inbox_handle: task::JoinHandle<()>,
mvbox: ImapConnectionState,
mvbox_handle: Option<task::JoinHandle<()>>,
sentbox: ImapConnectionState,
sentbox_handle: Option<task::JoinHandle<()>>,
smtp: SmtpConnectionState,
smtp_handle: task::JoinHandle<()>,
ephemeral_handle: task::JoinHandle<()>,
@@ -129,13 +122,6 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
info = Default::default();
}
None => {
let requested = ctx.quota_update_request.swap(false, Ordering::Relaxed);
if requested {
if let Err(err) = ctx.update_recent_quota(&mut connection).await {
warn!(ctx, "Failed to update quota: {:#}.", err);
}
}
maybe_add_time_based_warnings(&ctx).await;
match ctx.get_config_i64(Config::LastHousekeeping).await {
@@ -175,7 +161,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
}
}
info = fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await;
info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
}
}
}
@@ -196,20 +182,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
/// handling all the errors. In case of an error, it is logged, but not propagated upwards. If
/// critical operation fails such as fetching new messages fails, connection is reset via
/// `trigger_reconnect`, so a fresh one can be opened.
async fn fetch_idle(
ctx: &Context,
connection: &mut Imap,
folder_meaning: FolderMeaning,
) -> InterruptInfo {
let folder_config = match folder_meaning.to_config() {
Some(c) => c,
None => {
error!(ctx, "Bad folder meaning: {}", folder_meaning);
return connection
.fake_idle(ctx, None, FolderMeaning::Unknown)
.await;
}
};
async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) -> InterruptInfo {
let folder = match ctx.get_config(folder_config).await {
Ok(folder) => folder,
Err(err) => {
@@ -217,9 +190,7 @@ async fn fetch_idle(
ctx,
"Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err
);
return connection
.fake_idle(ctx, None, FolderMeaning::Unknown)
.await;
return connection.fake_idle(ctx, None).await;
}
};
@@ -228,9 +199,7 @@ async fn fetch_idle(
} else {
connection.connectivity.set_not_configured(ctx).await;
info!(ctx, "Can not watch {} folder, not set", folder_config);
return connection
.fake_idle(ctx, None, FolderMeaning::Unknown)
.await;
return connection.fake_idle(ctx, None).await;
};
// connect and fake idle if unable to connect
@@ -241,9 +210,7 @@ async fn fetch_idle(
{
warn!(ctx, "{:#}", err);
connection.trigger_reconnect(ctx);
return connection
.fake_idle(ctx, Some(watch_folder), folder_meaning)
.await;
return connection.fake_idle(ctx, Some(watch_folder)).await;
}
if folder_config == Config::ConfiguredInboxFolder {
@@ -260,7 +227,7 @@ async fn fetch_idle(
// Fetch the watched folder.
if let Err(err) = connection
.fetch_move_delete(ctx, &watch_folder, folder_meaning)
.fetch_move_delete(ctx, &watch_folder, false)
.await
.context("fetch_move_delete")
{
@@ -298,7 +265,7 @@ async fn fetch_idle(
// no new messages. We want to select the watched folder anyway before going IDLE
// there, so this does not take additional protocol round-trip.
if let Err(err) = connection
.fetch_move_delete(ctx, &watch_folder, folder_meaning)
.fetch_move_delete(ctx, &watch_folder, false)
.await
.context("fetch_move_delete after scan_folders")
{
@@ -326,9 +293,7 @@ async fn fetch_idle(
ctx,
"IMAP session does not support IDLE, going to fake idle."
);
return connection
.fake_idle(ctx, Some(watch_folder), folder_meaning)
.await;
return connection.fake_idle(ctx, Some(watch_folder)).await;
}
info!(ctx, "IMAP session supports IDLE, using it.");
@@ -353,9 +318,7 @@ async fn fetch_idle(
}
} else {
warn!(ctx, "No IMAP session, going to fake idle.");
connection
.fake_idle(ctx, Some(watch_folder), folder_meaning)
.await
connection.fake_idle(ctx, Some(watch_folder)).await
}
}
@@ -363,11 +326,11 @@ async fn simple_imap_loop(
ctx: Context,
started: Sender<()>,
inbox_handlers: ImapConnectionHandlers,
folder_meaning: FolderMeaning,
folder_config: Config,
) {
use futures::future::FutureExt;
info!(ctx, "starting simple loop for {}", folder_meaning);
info!(ctx, "starting simple loop for {}", folder_config);
let ImapConnectionHandlers {
mut connection,
stop_receiver,
@@ -383,7 +346,7 @@ async fn simple_imap_loop(
}
loop {
fetch_idle(&ctx, &mut connection, folder_meaning).await;
fetch_idle(&ctx, &mut connection, folder_config).await;
}
};
@@ -480,56 +443,75 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
impl Scheduler {
/// Start the scheduler.
pub async fn start(ctx: Context) -> Result<Self> {
let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?;
let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?;
let (smtp, smtp_handlers) = SmtpConnectionState::new();
let (inbox, inbox_handlers) = ImapConnectionState::new(&ctx).await?;
let (inbox_start_send, inbox_start_recv) = channel::bounded(1);
let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1);
let mut mvbox_handle = None;
let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
let mut sentbox_handle = None;
let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1);
let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1);
let mut oboxes = Vec::new();
let mut start_recvs = Vec::new();
let (conn_state, inbox_handlers) = ImapConnectionState::new(&ctx).await?;
let (inbox_start_send, inbox_start_recv) = channel::bounded(1);
let handle = {
let inbox_handle = {
let ctx = ctx.clone();
task::spawn(async move { inbox_loop(ctx, inbox_start_send, inbox_handlers).await })
};
let inbox = SchedBox {
meaning: FolderMeaning::Inbox,
conn_state,
handle,
};
start_recvs.push(inbox_start_recv);
for (meaning, should_watch) in [
(FolderMeaning::Mvbox, ctx.should_watch_mvbox().await),
(
FolderMeaning::Sent,
ctx.get_config_bool(Config::SentboxWatch).await,
),
] {
if should_watch? {
let (conn_state, handlers) = ImapConnectionState::new(&ctx).await?;
let (start_send, start_recv) = channel::bounded(1);
let ctx = ctx.clone();
let handle = task::spawn(async move {
simple_imap_loop(ctx, start_send, handlers, meaning).await
});
oboxes.push(SchedBox {
meaning,
conn_state,
handle,
});
start_recvs.push(start_recv);
}
if ctx.should_watch_mvbox().await? {
let ctx = ctx.clone();
mvbox_handle = Some(task::spawn(async move {
simple_imap_loop(
ctx,
mvbox_start_send,
mvbox_handlers,
Config::ConfiguredMvboxFolder,
)
.await
}));
} else {
mvbox_start_send
.send(())
.await
.context("mvbox start send, missing receiver")?;
mvbox_handlers
.connection
.connectivity
.set_not_configured(&ctx)
.await
}
if ctx.get_config_bool(Config::SentboxWatch).await? {
let ctx = ctx.clone();
sentbox_handle = Some(task::spawn(async move {
simple_imap_loop(
ctx,
sentbox_start_send,
sentbox_handlers,
Config::ConfiguredSentboxFolder,
)
.await
}));
} else {
sentbox_start_send
.send(())
.await
.context("sentbox start send, missing receiver")?;
sentbox_handlers
.connection
.connectivity
.set_not_configured(&ctx)
.await
}
let smtp_handle = {
let ctx = ctx.clone();
task::spawn(async move { smtp_loop(ctx, smtp_start_send, smtp_handlers).await })
};
start_recvs.push(smtp_start_recv);
let ephemeral_handle = {
let ctx = ctx.clone();
@@ -549,8 +531,12 @@ impl Scheduler {
let res = Self {
inbox,
oboxes,
mvbox,
sentbox,
smtp,
inbox_handle,
mvbox_handle,
sentbox_handle,
smtp_handle,
ephemeral_handle,
ephemeral_interrupt_send,
@@ -560,7 +546,12 @@ impl Scheduler {
};
// wait for all loops to be started
if let Err(err) = try_join_all(start_recvs.iter().map(|r| r.recv())).await {
if let Err(err) = try_join!(
inbox_start_recv.recv(),
mvbox_start_recv.recv(),
sentbox_start_recv.recv(),
smtp_start_recv.recv()
) {
bail!("failed to start scheduler: {}", err);
}
@@ -568,26 +559,30 @@ impl Scheduler {
Ok(res)
}
fn boxes(&self) -> iter::Chain<iter::Once<&SchedBox>, std::slice::Iter<'_, SchedBox>> {
once(&self.inbox).chain(self.oboxes.iter())
}
fn maybe_network(&self) {
for b in self.boxes() {
b.conn_state.interrupt(InterruptInfo::new(true));
}
self.interrupt_inbox(InterruptInfo::new(true));
self.interrupt_mvbox(InterruptInfo::new(true));
self.interrupt_sentbox(InterruptInfo::new(true));
self.interrupt_smtp(InterruptInfo::new(true));
}
fn maybe_network_lost(&self) {
for b in self.boxes() {
b.conn_state.interrupt(InterruptInfo::new(false));
}
self.interrupt_inbox(InterruptInfo::new(false));
self.interrupt_mvbox(InterruptInfo::new(false));
self.interrupt_sentbox(InterruptInfo::new(false));
self.interrupt_smtp(InterruptInfo::new(false));
}
fn interrupt_inbox(&self, info: InterruptInfo) {
self.inbox.conn_state.interrupt(info);
self.inbox.interrupt(info);
}
fn interrupt_mvbox(&self, info: InterruptInfo) {
self.mvbox.interrupt(info);
}
fn interrupt_sentbox(&self, info: InterruptInfo) {
self.sentbox.interrupt(info);
}
fn interrupt_smtp(&self, info: InterruptInfo) {
@@ -610,17 +605,29 @@ impl Scheduler {
///
/// It consumes the scheduler and never fails to stop it. In the worst case, long-running tasks
/// are forcefully terminated if they cannot shutdown within the timeout.
pub(crate) async fn stop(self, context: &Context) {
pub(crate) async fn stop(mut self, context: &Context) {
// Send stop signals to tasks so they can shutdown cleanly.
for b in self.boxes() {
b.conn_state.stop().await.ok_or_log(context);
self.inbox.stop().await.ok_or_log(context);
if self.mvbox_handle.is_some() {
self.mvbox.stop().await.ok_or_log(context);
}
if self.sentbox_handle.is_some() {
self.sentbox.stop().await.ok_or_log(context);
}
self.smtp.stop().await.ok_or_log(context);
// Actually shutdown tasks.
let timeout_duration = std::time::Duration::from_secs(30);
for b in once(self.inbox).chain(self.oboxes.into_iter()) {
tokio::time::timeout(timeout_duration, b.handle)
tokio::time::timeout(timeout_duration, self.inbox_handle)
.await
.ok_or_log(context);
if let Some(mvbox_handle) = self.mvbox_handle.take() {
tokio::time::timeout(timeout_duration, mvbox_handle)
.await
.ok_or_log(context);
}
if let Some(sentbox_handle) = self.sentbox_handle.take() {
tokio::time::timeout(timeout_duration, sentbox_handle)
.await
.ok_or_log(context);
}

View File

@@ -1,18 +1,18 @@
use core::fmt;
use std::{iter::once, ops::Deref, sync::Arc};
use std::{ops::Deref, sync::Arc};
use anyhow::{anyhow, Result};
use humansize::{format_size, BINARY};
use tokio::sync::{Mutex, RwLockReadGuard};
use crate::events::EventType;
use crate::imap::{scan_folders::get_watched_folder_configs, FolderMeaning};
use crate::imap::scan_folders::get_watched_folder_configs;
use crate::quota::{
QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
};
use crate::tools::time;
use crate::{config::Config, scheduler::Scheduler, stock_str, tools};
use crate::{context::Context, log::LogExt};
use crate::{scheduler::Scheduler, stock_str, tools};
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)]
pub enum Connectivity {
@@ -157,14 +157,17 @@ impl ConnectivityStore {
/// Called during `dc_maybe_network()` to make sure that `dc_accounts_all_work_done()`
/// returns false immediately after `dc_maybe_network()`.
pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option<Scheduler>>) {
let (inbox, oboxes) = match &*scheduler {
Some(Scheduler { inbox, oboxes, .. }) => (
inbox.conn_state.state.connectivity.clone(),
oboxes
.iter()
.map(|b| b.conn_state.state.connectivity.clone())
.collect::<Vec<_>>(),
),
let [inbox, mvbox, sentbox] = match &*scheduler {
Some(Scheduler {
inbox,
mvbox,
sentbox,
..
}) => [
inbox.state.connectivity.clone(),
mvbox.state.connectivity.clone(),
sentbox.state.connectivity.clone(),
],
None => return,
};
drop(scheduler);
@@ -182,7 +185,7 @@ pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option<Sched
}
drop(connectivity_lock);
for state in oboxes {
for state in &[&mvbox, &sentbox] {
let mut connectivity_lock = state.0.lock().await;
if *connectivity_lock == DetailedConnectivity::Connected {
*connectivity_lock = DetailedConnectivity::InterruptingIdle;
@@ -199,11 +202,17 @@ pub(crate) async fn maybe_network_lost(
context: &Context,
scheduler: RwLockReadGuard<'_, Option<Scheduler>>,
) {
let stores: Vec<_> = match &*scheduler {
Some(sched) => sched
.boxes()
.map(|b| b.conn_state.state.connectivity.clone())
.collect(),
let stores = match &*scheduler {
Some(Scheduler {
inbox,
mvbox,
sentbox,
..
}) => [
inbox.state.connectivity.clone(),
mvbox.state.connectivity.clone(),
sentbox.state.connectivity.clone(),
],
None => return,
};
drop(scheduler);
@@ -251,9 +260,14 @@ impl Context {
pub async fn get_connectivity(&self) -> Connectivity {
let lock = self.scheduler.read().await;
let stores: Vec<_> = match &*lock {
Some(sched) => sched
.boxes()
.map(|b| b.conn_state.state.connectivity.clone())
Some(Scheduler {
inbox,
mvbox,
sentbox,
..
}) => [&inbox.state, &mvbox.state, &sentbox.state]
.iter()
.map(|state| state.connectivity.clone())
.collect(),
None => return Connectivity::NotConnected,
};
@@ -334,12 +348,28 @@ impl Context {
let lock = self.scheduler.read().await;
let (folders_states, smtp) = match &*lock {
Some(sched) => (
sched
.boxes()
.map(|b| (b.meaning, b.conn_state.state.connectivity.clone()))
.collect::<Vec<_>>(),
sched.smtp.state.connectivity.clone(),
Some(Scheduler {
inbox,
mvbox,
sentbox,
smtp,
..
}) => (
[
(
Config::ConfiguredInboxFolder,
inbox.state.connectivity.clone(),
),
(
Config::ConfiguredMvboxFolder,
mvbox.state.connectivity.clone(),
),
(
Config::ConfiguredSentboxFolder,
sentbox.state.connectivity.clone(),
),
],
smtp.state.connectivity.clone(),
),
None => {
return Err(anyhow!("Not started"));
@@ -360,8 +390,8 @@ impl Context {
for (folder, state) in &folders_states {
let mut folder_added = false;
if let Some(config) = folder.to_config().filter(|c| watched_folders.contains(c)) {
let f = self.get_config(config).await.ok_or_log(self).flatten();
if watched_folders.contains(folder) {
let f = self.get_config(*folder).await.ok_or_log(self).flatten();
if let Some(foldername) = f {
let detailed = &state.get_detailed().await;
@@ -377,7 +407,7 @@ impl Context {
}
}
if !folder_added && folder == &FolderMeaning::Inbox {
if !folder_added && folder == &Config::ConfiguredInboxFolder {
let detailed = &state.get_detailed().await;
if let DetailedConnectivity::Error(_) = detailed {
// On the inbox thread, we also do some other things like scan_folders and run jobs
@@ -505,10 +535,14 @@ impl Context {
pub async fn all_work_done(&self) -> bool {
let lock = self.scheduler.read().await;
let stores: Vec<_> = match &*lock {
Some(sched) => sched
.boxes()
.map(|b| &b.conn_state.state)
.chain(once(&sched.smtp.state))
Some(Scheduler {
inbox,
mvbox,
sentbox,
smtp,
..
}) => [&inbox.state, &mvbox.state, &sentbox.state, &smtp.state]
.iter()
.map(|state| state.connectivity.clone())
.collect(),
None => return false,

View File

@@ -13,13 +13,12 @@ use tokio::task;
use crate::config::Config;
use crate::contact::{Contact, ContactId};
use crate::events::EventType;
use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
use crate::login_param::{build_tls, CertificateChecks, LoginParam, ServerLoginParam};
use crate::message::Message;
use crate::message::{self, MsgId};
use crate::mimefactory::MimeFactory;
use crate::net::connect_tcp;
use crate::net::session::SessionStream;
use crate::net::tls::wrap_tls;
use crate::oauth2::get_oauth2_access_token;
use crate::provider::Socket;
use crate::socks::Socks5Config;
@@ -102,9 +101,8 @@ impl Smtp {
&lp.smtp,
&lp.socks5_config,
&lp.addr,
lp.provider.map_or(lp.socks5_config.is_some(), |provider| {
provider.opt.strict_tls
}),
lp.provider
.map_or(lp.socks5_config.is_some(), |provider| provider.strict_tls),
)
.await
}
@@ -120,7 +118,8 @@ impl Smtp {
let socks5_stream = socks5_config
.connect(context, hostname, port, SMTP_TIMEOUT, strict_tls)
.await?;
let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream).await?;
let tls = build_tls(strict_tls);
let tls_stream = tls.connect(hostname, socks5_stream).await?;
let buffered_stream = BufWriter::new(tls_stream);
let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
let client = smtp::SmtpClient::new().smtp_utf8(true);
@@ -144,7 +143,9 @@ impl Smtp {
let client = smtp::SmtpClient::new().smtp_utf8(true);
let transport = SmtpTransport::new(client, socks5_stream).await?;
let tcp_stream = transport.starttls().await?;
let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
let tls = build_tls(strict_tls);
let tls_stream = tls
.connect(hostname, tcp_stream)
.await
.context("STARTTLS upgrade failed")?;
let buffered_stream = BufWriter::new(tls_stream);
@@ -179,7 +180,8 @@ impl Smtp {
strict_tls: bool,
) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?;
let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?;
let tls = build_tls(strict_tls);
let tls_stream = tls.connect(hostname, tcp_stream).await?;
let buffered_stream = BufWriter::new(tls_stream);
let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
let client = smtp::SmtpClient::new().smtp_utf8(true);
@@ -200,7 +202,9 @@ impl Smtp {
let client = smtp::SmtpClient::new().smtp_utf8(true);
let transport = SmtpTransport::new(client, tcp_stream).await?;
let tcp_stream = transport.starttls().await?;
let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
let tls = build_tls(strict_tls);
let tls_stream = tls
.connect(hostname, tcp_stream)
.await
.context("STARTTLS upgrade failed")?;
let buffered_stream = BufWriter::new(tls_stream);

View File

@@ -47,7 +47,7 @@ impl Smtp {
let chunk_size = context
.get_configured_provider()
.await?
.and_then(|provider| provider.opt.max_smtp_rcpt_to)
.and_then(|provider| provider.max_smtp_rcpt_to)
.map_or(DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
for recipients_chunk in recipients.chunks(chunk_size) {

View File

@@ -2,10 +2,11 @@
use std::collections::{HashMap, HashSet};
use std::convert::TryFrom;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::path::PathBuf;
use anyhow::{bail, Context as _, Result};
use rusqlite::{self, config::DbConfig, Connection, OpenFlags, TransactionBehavior};
use rusqlite::{self, config::DbConfig, Connection, OpenFlags};
use tokio::sync::RwLock;
use crate::blob::BlobObject;
@@ -48,7 +49,7 @@ pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator<Item = &d
mod migrations;
mod pool;
use pool::Pool;
use pool::{Pool, PooledConnection};
/// A wrapper around the underlying Sqlite3 object.
#[derive(Debug)]
@@ -123,55 +124,74 @@ impl Sql {
// drop closes the connection
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
pub(crate) async fn export(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
.to_str()
.with_context(|| format!("path {path:?} is not valid unicode"))?;
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
}
/// Imports the database from a separate file with the given passphrase.
pub(crate) async fn import(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
.to_str()
.with_context(|| format!("path {path:?} is not valid unicode"))?
.to_string();
let res = self
.call(move |conn| {
// Check that backup passphrase is correct before resetting our database.
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
if let Err(err) = conn
.query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(()))
.context("backup passphrase is not correct")
{
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
return Err(err);
}
.with_context(|| format!("path {path:?} is not valid unicode"))?;
let conn = self.get_conn().await?;
// Reset the database without reopening it. We don't want to reopen the database because we
// don't have main database passphrase at this point.
// See <https://sqlite.org/c3ref/c_dbconfig_enable_fkey.html> for documentation.
// Without resetting import may fail due to existing tables.
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true)
.context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?;
conn.execute("VACUUM", [])
.context("failed to vacuum the database")?;
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false)
.context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?;
let res = conn
.query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| {
Ok(())
})
.context("failed to import from attached backup database");
tokio::task::block_in_place(move || {
// Check that backup passphrase is correct before resetting our database.
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
if let Err(err) = conn
.query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(()))
.context("backup passphrase is not correct")
{
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
.await;
return Err(err);
}
// The config cache is wrong now that we have a different database
self.config_cache.write().await.clear();
res
// Reset the database without reopening it. We don't want to reopen the database because we
// don't have main database passphrase at this point.
// See <https://sqlite.org/c3ref/c_dbconfig_enable_fkey.html> for documentation.
// Without resetting import may fail due to existing tables.
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true)
.context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?;
conn.execute("VACUUM", [])
.context("failed to vacuum the database")?;
conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false)
.context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?;
let res = conn
.query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| {
Ok(())
})
.context("failed to import from attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
}
/// Creates a new connection pool.
@@ -299,41 +319,22 @@ impl Sql {
}
}
/// Allocates a connection and calls given function with the connection.
///
/// Returns the result of the function.
pub async fn call<'a, F, R>(&'a self, function: F) -> Result<R>
where
F: 'a + FnOnce(&mut Connection) -> Result<R> + Send,
R: Send + 'static,
{
let lock = self.pool.read().await;
let pool = lock.as_ref().context("no SQL connection")?;
let mut conn = pool.get().await?;
let res = tokio::task::block_in_place(move || function(&mut conn))?;
Ok(res)
}
/// Execute the given query, returning the number of affected rows.
pub async fn execute(
&self,
query: &str,
params: impl rusqlite::Params + Send,
) -> Result<usize> {
self.call(move |conn| {
pub async fn execute(&self, query: &str, params: impl rusqlite::Params) -> Result<usize> {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let res = conn.execute(query, params)?;
Ok(res)
})
.await
}
/// Executes the given query, returning the last inserted row ID.
pub async fn insert(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<i64> {
self.call(move |conn| {
pub async fn insert(&self, query: &str, params: impl rusqlite::Params) -> Result<i64> {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
conn.execute(query, params)?;
Ok(conn.last_insert_rowid())
})
.await
}
/// Prepares and executes the statement and maps a function over the resulting rows.
@@ -342,32 +343,40 @@ impl Sql {
pub async fn query_map<T, F, G, H>(
&self,
sql: &str,
params: impl rusqlite::Params + Send,
params: impl rusqlite::Params,
f: F,
mut g: G,
) -> Result<H>
where
F: Send + FnMut(&rusqlite::Row) -> rusqlite::Result<T>,
G: Send + FnMut(rusqlite::MappedRows<F>) -> Result<H>,
H: Send + 'static,
F: FnMut(&rusqlite::Row) -> rusqlite::Result<T>,
G: FnMut(rusqlite::MappedRows<F>) -> Result<H>,
{
self.call(move |conn| {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let mut stmt = conn.prepare(sql)?;
let res = stmt.query_map(params, f)?;
g(res)
})
.await
}
/// Allocates a connection from the connection pool and returns it.
pub(crate) async fn get_conn(&self) -> Result<PooledConnection> {
let lock = self.pool.read().await;
let pool = lock.as_ref().context("no SQL connection")?;
let conn = pool.get();
Ok(conn)
}
/// Used for executing `SELECT COUNT` statements only. Returns the resulting count.
pub async fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<usize> {
pub async fn count(&self, query: &str, params: impl rusqlite::Params) -> Result<usize> {
let count: isize = self.query_row(query, params, |row| row.get(0)).await?;
Ok(usize::try_from(count)?)
}
/// Used for executing `SELECT COUNT` statements only. Returns `true`, if the count is at least
/// one, `false` otherwise.
pub async fn exists(&self, sql: &str, params: impl rusqlite::Params + Send) -> Result<bool> {
pub async fn exists(&self, sql: &str, params: impl rusqlite::Params) -> Result<bool> {
let count = self.count(sql, params).await?;
Ok(count > 0)
}
@@ -376,37 +385,31 @@ impl Sql {
pub async fn query_row<T, F>(
&self,
query: &str,
params: impl rusqlite::Params + Send,
params: impl rusqlite::Params,
f: F,
) -> Result<T>
where
F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T> + Send,
T: Send + 'static,
F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
{
self.call(move |conn| {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let res = conn.query_row(query, params, f)?;
Ok(res)
})
.await
}
/// Execute the function inside a transaction.
///
/// If the function returns an error, the transaction will be rolled back. If it does not return an
/// error, the transaction will be committed.
///
/// Transactions started use IMMEDIATE behavior
/// rather than default DEFERRED behavior
/// to avoid "database is busy" errors
/// which may happen when DEFERRED transaction
/// is attempted to be promoted to a write transaction.
pub async fn transaction<G, H>(&self, callback: G) -> Result<H>
where
H: Send + 'static,
G: Send + FnOnce(&mut rusqlite::Transaction<'_>) -> Result<H>,
{
self.call(move |conn| {
let mut transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let mut transaction = conn.transaction()?;
let ret = callback(&mut transaction);
match ret {
@@ -420,12 +423,12 @@ impl Sql {
}
}
})
.await
}
/// Query the database if the requested table already exists.
pub async fn table_exists(&self, name: &str) -> Result<bool> {
self.call(move |conn| {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let mut exists = false;
conn.pragma(None, "table_info", name.to_string(), |_row| {
// will only be executed if the info was found
@@ -435,12 +438,12 @@ impl Sql {
Ok(exists)
})
.await
}
/// Check if a column exists in a given table.
pub async fn col_exists(&self, table_name: &str, col_name: &str) -> Result<bool> {
self.call(move |conn| {
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
let mut exists = false;
// `PRAGMA table_info` returns one row per column,
// each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value
@@ -454,27 +457,29 @@ impl Sql {
Ok(exists)
})
.await
}
/// Execute a query which is expected to return zero or one row.
pub async fn query_row_optional<T, F>(
&self,
sql: &str,
params: impl rusqlite::Params + Send,
params: impl rusqlite::Params,
f: F,
) -> Result<Option<T>>
where
F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
T: Send + 'static,
F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
{
self.call(move |conn| match conn.query_row(sql.as_ref(), params, f) {
Ok(res) => Ok(Some(res)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => Ok(None),
Err(err) => Err(err.into()),
})
.await
let conn = self.get_conn().await?;
let res =
tokio::task::block_in_place(move || match conn.query_row(sql.as_ref(), params, f) {
Ok(res) => Ok(Some(res)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => {
Ok(None)
}
Err(err) => Err(err),
})?;
Ok(res)
}
/// Executes a query which is expected to return one row and one
@@ -483,10 +488,10 @@ impl Sql {
pub async fn query_get_value<T>(
&self,
query: &str,
params: impl rusqlite::Params + Send,
params: impl rusqlite::Params,
) -> Result<Option<T>>
where
T: rusqlite::types::FromSql + Send + 'static,
T: rusqlite::types::FromSql,
{
self.query_row_optional(query, params, |row| row.get::<_, T>(0))
.await
@@ -949,16 +954,11 @@ mod tests {
async fn test_auto_vacuum() -> Result<()> {
let t = TestContext::new().await;
let auto_vacuum = t
.sql
.call(|conn| {
let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| {
let auto_vacuum: i32 = row.get(0)?;
Ok(auto_vacuum)
})?;
Ok(auto_vacuum)
})
.await?;
let conn = t.sql.get_conn().await?;
let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| {
let auto_vacuum: i32 = row.get(0)?;
Ok(auto_vacuum)
})?;
// auto_vacuum=2 is the same as auto_vacuum=INCREMENTAL
assert_eq!(auto_vacuum, 2);
@@ -983,7 +983,7 @@ mod tests {
assert_eq!(avatar_bytes, &tokio::fs::read(&a).await.unwrap()[..]);
t.sql.close().await;
housekeeping(&t).await.unwrap(); // housekeeping should emit warnings but not fail
housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed
t.sql.open(&t, "".to_string()).await.unwrap();
let a = t.get_config(Config::Selfavatar).await.unwrap().unwrap();

View File

@@ -1,29 +1,21 @@
//! # SQLite connection pool.
//!
//! The connection pool holds a number of SQLite connections and allows to allocate them.
//! When allocated connection is dropped, underlying connection is returned back to the pool.
//!
//! The pool is organized as a stack. It always allocates the most recently used connection.
//! Each SQLite connection has its own page cache, so allocating recently used connections
//! improves the performance compared to, for example, organizing the pool as a queue
//! and returning the least recently used connection each time.
//! Connection pool.
use std::fmt;
use std::ops::{Deref, DerefMut};
use std::sync::{Arc, Weak};
use anyhow::{Context, Result};
use parking_lot::Mutex;
use parking_lot::{Condvar, Mutex};
use rusqlite::Connection;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
/// Inner connection pool.
#[derive(Debug)]
struct InnerPool {
/// Available connections.
connections: Mutex<Vec<Connection>>,
/// Counts the number of available connections.
semaphore: Arc<Semaphore>,
/// Conditional variable to notify about added connections.
///
/// Used to wait for available connection when the pool is empty.
cond: Condvar,
}
impl InnerPool {
@@ -34,6 +26,7 @@ impl InnerPool {
let mut connections = self.connections.lock();
connections.push(connection);
drop(connections);
self.cond.notify_one();
}
}
@@ -44,9 +37,6 @@ pub struct PooledConnection {
/// Only `None` right after moving the connection back to the pool.
conn: Option<Connection>,
/// Semaphore permit, dropped after returning the connection to the pool.
_permit: OwnedSemaphorePermit,
}
impl Drop for PooledConnection {
@@ -54,6 +44,7 @@ impl Drop for PooledConnection {
// Put the connection back unless the pool is already dropped.
if let Some(pool) = self.pool.upgrade() {
if let Some(conn) = self.conn.take() {
conn.release_memory().ok();
pool.put(conn);
}
}
@@ -75,35 +66,41 @@ impl DerefMut for PooledConnection {
}
/// Connection pool.
#[derive(Clone, Debug)]
#[derive(Clone)]
pub struct Pool {
/// Reference to the actual connection pool.
inner: Arc<InnerPool>,
}
impl fmt::Debug for Pool {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "Pool")
}
}
impl Pool {
/// Creates a new connection pool.
pub fn new(connections: Vec<Connection>) -> Self {
let semaphore = Arc::new(Semaphore::new(connections.len()));
let inner = Arc::new(InnerPool {
connections: Mutex::new(connections),
semaphore,
cond: Condvar::new(),
});
Pool { inner }
}
/// Retrieves a connection from the pool.
pub async fn get(&self) -> Result<PooledConnection> {
let permit = self.inner.semaphore.clone().acquire_owned().await?;
pub fn get(&self) -> PooledConnection {
let mut connections = self.inner.connections.lock();
let conn = connections
.pop()
.context("got a permit when there are no connections in the pool")?;
let conn = PooledConnection {
pool: Arc::downgrade(&self.inner),
conn: Some(conn),
_permit: permit,
};
Ok(conn)
loop {
if let Some(conn) = connections.pop() {
return PooledConnection {
pool: Arc::downgrade(&self.inner),
conn: Some(conn),
};
}
self.inner.cond.wait(&mut connections);
}
}
}

View File

@@ -1,193 +0,0 @@
//! # Time smearing.
//!
//! As e-mails typically only use a second-based-resolution for timestamps,
//! the order of two mails sent withing one second is unclear.
//! This is bad e.g. when forwarding some messages from a chat -
//! these messages will appear at the recipient easily out of order.
//!
//! We work around this issue by not sending out two mails with the same timestamp.
//! For this purpose, in short, we track the last timestamp used in `last_smeared_timestamp`
//! when another timestamp is needed in the same second, we use `last_smeared_timestamp+1`
//! after some moments without messages sent out,
//! `last_smeared_timestamp` is again in sync with the normal time.
//!
//! However, we do not do all this for the far future,
//! but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
use std::cmp::{max, min};
use std::sync::atomic::{AtomicI64, Ordering};
pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
/// Smeared timestamp generator.
#[derive(Debug)]
pub struct SmearedTimestamp {
/// Next timestamp available for allocation.
smeared_timestamp: AtomicI64,
}
impl SmearedTimestamp {
/// Creates a new smeared timestamp generator.
pub fn new() -> Self {
Self {
smeared_timestamp: AtomicI64::new(0),
}
}
/// Allocates `count` unique timestamps.
///
/// Returns the first allocated timestamp.
pub fn create_n(&self, now: i64, count: i64) -> i64 {
let mut prev = self.smeared_timestamp.load(Ordering::Relaxed);
loop {
// Advance the timestamp if it is in the past,
// but keep `count - 1` timestamps from the past if possible.
let t = max(prev, now - count + 1);
// Rewind the time back if there is no room
// to allocate `count` timestamps without going too far into the future.
// Not going too far into the future
// is more important than generating unique timestamps.
let first = min(t, now + MAX_SECONDS_TO_LEND_FROM_FUTURE - count + 1);
// Allocate `count` timestamps by advancing the current timestamp.
let next = first + count;
if let Err(x) = self.smeared_timestamp.compare_exchange_weak(
prev,
next,
Ordering::Relaxed,
Ordering::Relaxed,
) {
prev = x;
} else {
return first;
}
}
}
/// Creates a single timestamp.
pub fn create(&self, now: i64) -> i64 {
self.create_n(now, 1)
}
/// Returns the current smeared timestamp.
pub fn current(&self) -> i64 {
self.smeared_timestamp.load(Ordering::Relaxed)
}
}
#[cfg(test)]
mod tests {
use std::time::SystemTime;
use super::*;
use crate::test_utils::TestContext;
use crate::tools::{create_smeared_timestamp, create_smeared_timestamps, smeared_time, time};
#[test]
fn test_smeared_timestamp() {
let smeared_timestamp = SmearedTimestamp::new();
let now = time();
assert_eq!(smeared_timestamp.current(), 0);
for i in 0..MAX_SECONDS_TO_LEND_FROM_FUTURE {
assert_eq!(smeared_timestamp.create(now), now + i);
}
assert_eq!(
smeared_timestamp.create(now),
now + MAX_SECONDS_TO_LEND_FROM_FUTURE
);
assert_eq!(
smeared_timestamp.create(now),
now + MAX_SECONDS_TO_LEND_FROM_FUTURE
);
// System time rewinds back by 1000 seconds.
let now = now - 1000;
assert_eq!(
smeared_timestamp.create(now),
now + MAX_SECONDS_TO_LEND_FROM_FUTURE
);
assert_eq!(
smeared_timestamp.create(now),
now + MAX_SECONDS_TO_LEND_FROM_FUTURE
);
assert_eq!(
smeared_timestamp.create(now + 1),
now + MAX_SECONDS_TO_LEND_FROM_FUTURE + 1
);
assert_eq!(smeared_timestamp.create(now + 100), now + 100);
assert_eq!(smeared_timestamp.create(now + 100), now + 101);
assert_eq!(smeared_timestamp.create(now + 100), now + 102);
}
#[test]
fn test_create_n_smeared_timestamps() {
let smeared_timestamp = SmearedTimestamp::new();
let now = time();
// Create a single timestamp to initialize the generator.
assert_eq!(smeared_timestamp.create(now), now);
// Wait a minute.
let now = now + 60;
// Simulate forwarding 7 messages.
let forwarded_messages = 7;
// We have not sent anything for a minute,
// so we can take the current timestamp and take 6 timestamps from the past.
assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 6);
assert_eq!(smeared_timestamp.current(), now + 1);
// Wait 4 seconds.
// Now we have 3 free timestamps in the past.
let now = now + 4;
assert_eq!(smeared_timestamp.current(), now - 3);
// Forward another 7 messages.
// We can only lend 3 timestamps from the past.
assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 3);
// We had to borrow 3 timestamps from the future
// because there were not enough timestamps in the past.
assert_eq!(smeared_timestamp.current(), now + 4);
// Forward another 7 messages.
// We cannot use more than 5 timestamps from the future,
// so we use 5 timestamps from the future,
// the current timestamp and one timestamp from the past.
assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_smeared_timestamp() {
let t = TestContext::new().await;
assert_ne!(create_smeared_timestamp(&t), create_smeared_timestamp(&t));
assert!(
create_smeared_timestamp(&t)
>= SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs() as i64
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_smeared_timestamps() {
let t = TestContext::new().await;
let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
let start = create_smeared_timestamps(&t, count as usize);
let next = smeared_time(&t);
assert!((start + count - 1) < next);
let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
let start = create_smeared_timestamps(&t, count as usize);
let next = smeared_time(&t);
assert!((start + count - 1) < next);
}
}

View File

@@ -3,6 +3,7 @@
#![allow(missing_docs)]
use core::cmp::{max, min};
use std::borrow::Cow;
use std::fmt;
use std::io::Cursor;
@@ -139,27 +140,63 @@ pub(crate) fn gm2local_offset() -> i64 {
i64::from(lt.offset().local_minus_utc())
}
// timesmearing
// - as e-mails typically only use a second-based-resolution for timestamps,
// the order of two mails sent withing one second is unclear.
// this is bad eg. when forwarding some messages from a chat -
// these messages will appear at the recipient easily out of order.
// - we work around this issue by not sending out two mails with the same timestamp.
// - for this purpose, in short, we track the last timestamp used in `last_smeared_timestamp`
// when another timestamp is needed in the same second, we use `last_smeared_timestamp+1`
// - after some moments without messages sent out,
// `last_smeared_timestamp` is again in sync with the normal time.
// - however, we do not do all this for the far future,
// but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
/// Returns the current smeared timestamp,
///
/// The returned timestamp MUST NOT be sent out.
pub(crate) fn smeared_time(context: &Context) -> i64 {
let now = time();
let ts = context.smeared_timestamp.current();
std::cmp::max(ts, now)
pub(crate) async fn smeared_time(context: &Context) -> i64 {
let mut now = time();
let ts = *context.last_smeared_timestamp.read().await;
if ts >= now {
now = ts + 1;
}
now
}
/// Returns a timestamp that is guaranteed to be unique.
pub(crate) fn create_smeared_timestamp(context: &Context) -> i64 {
pub(crate) async fn create_smeared_timestamp(context: &Context) -> i64 {
let now = time();
context.smeared_timestamp.create(now)
let mut ret = now;
let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
if ret <= *last_smeared_timestamp {
ret = *last_smeared_timestamp + 1;
if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
ret = now + MAX_SECONDS_TO_LEND_FROM_FUTURE
}
}
*last_smeared_timestamp = ret;
ret
}
// creates `count` timestamps that are guaranteed to be unique.
// the first created timestamps is returned directly,
// the frist created timestamps is returned directly,
// get the other timestamps just by adding 1..count-1
pub(crate) fn create_smeared_timestamps(context: &Context, count: usize) -> i64 {
pub(crate) async fn create_smeared_timestamps(context: &Context, count: usize) -> i64 {
let now = time();
context.smeared_timestamp.create_n(now, count as i64)
let count = count as i64;
let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
start = max(*last_smeared_timestamp + 1, start);
*last_smeared_timestamp = start + count - 1;
start
}
// if the system time is not plausible, once a day, add a device message.
@@ -555,8 +592,6 @@ pub(crate) fn improve_single_line_input(input: &str) -> String {
}
pub(crate) trait IsNoneOrEmpty<T> {
/// Returns true if an Option does not contain a string
/// or contains an empty string.
fn is_none_or_empty(&self) -> bool;
}
impl<T> IsNoneOrEmpty<T> for Option<T>
@@ -1034,6 +1069,36 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
assert!(!file_exist!(context, &fn0));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_smeared_timestamp() {
let t = TestContext::new().await;
assert_ne!(
create_smeared_timestamp(&t).await,
create_smeared_timestamp(&t).await
);
assert!(
create_smeared_timestamp(&t).await
>= SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs() as i64
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_smeared_timestamps() {
let t = TestContext::new().await;
let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
let start = create_smeared_timestamps(&t, count as usize).await;
let next = smeared_time(&t).await;
assert!((start + count - 1) < next);
let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
let start = create_smeared_timestamps(&t, count as usize).await;
let next = smeared_time(&t).await;
assert!((start + count - 1) < next);
}
#[test]
fn test_duration_to_str() {
assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");

View File

@@ -2,8 +2,8 @@
use anyhow::Result;
use crate::chat::ChatId;
use crate::contact::ContactId;
use crate::chat::{Chat, ChatId};
use crate::contact::{Contact, ContactId};
use crate::context::Context;
use crate::param::{Param, Params};
@@ -17,26 +17,12 @@ impl Context {
scope: Param,
new_timestamp: i64,
) -> Result<bool> {
self.sql
.transaction(|transaction| {
let mut param: Params = transaction.query_row(
"SELECT param FROM contacts WHERE id=?",
[contact_id],
|row| {
let param: String = row.get(0)?;
Ok(param.parse().unwrap_or_default())
},
)?;
let update = param.update_timestamp(scope, new_timestamp)?;
if update {
transaction.execute(
"UPDATE contacts SET param=? WHERE id=?",
params![param.to_string(), contact_id],
)?;
}
Ok(update)
})
.await
let mut contact = Contact::load_from_db(self, contact_id).await?;
if contact.param.update_timestamp(scope, new_timestamp)? {
contact.update_param(self).await?;
return Ok(true);
}
Ok(false)
}
}
@@ -49,24 +35,12 @@ impl ChatId {
scope: Param,
new_timestamp: i64,
) -> Result<bool> {
context
.sql
.transaction(|transaction| {
let mut param: Params =
transaction.query_row("SELECT param FROM chats WHERE id=?", [self], |row| {
let param: String = row.get(0)?;
Ok(param.parse().unwrap_or_default())
})?;
let update = param.update_timestamp(scope, new_timestamp)?;
if update {
transaction.execute(
"UPDATE chats SET param=? WHERE id=?",
params![param.to_string(), self],
)?;
}
Ok(update)
})
.await
let mut chat = Chat::load_from_db(context, *self).await?;
if chat.param.update_timestamp(scope, new_timestamp)? {
chat.update_param(context).await?;
return Ok(true);
}
Ok(false)
}
}
@@ -86,7 +60,6 @@ impl Params {
#[cfg(test)]
mod tests {
use super::*;
use crate::chat::Chat;
use crate::receive_imf::receive_imf;
use crate::test_utils::TestContext;
use crate::tools::time;

View File

@@ -408,7 +408,7 @@ impl Context {
.create_status_update_record(
&mut instance,
update_str,
create_smeared_timestamp(self),
create_smeared_timestamp(self).await,
send_now,
ContactId::SELF,
)

View File

@@ -3,58 +3,26 @@
Some of the standards Delta Chat is based on:
Tasks | Standards
-------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Transport | IMAP v4 ([RFC 3501][]), SMTP ([RFC 5321][]) and Internet Message Format (IMF, [RFC 5322][])
Proxy | SOCKS5 ([RFC 1928][])
Embedded media | MIME Document Series ([RFC 2045][], [RFC 2046][]), Content-Disposition Header ([RFC 2183][]), Multipart/Related ([RFC 2387][])
Text and Quote encoding | Fixed, Flowed ([RFC 3676][])
Reactions | Reaction: Indicating Summary Reaction to a Message ([RFC 9078][])
Filename encoding | Encoded Words ([RFC 2047][]), Encoded Word Extensions ([RFC 2231][])
Identify server folders | IMAP LIST Extension ([RFC 6154][])
Push | IMAP IDLE ([RFC 2177][])
Quota | IMAP QUOTA extension ([RFC 2087][])
Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162][])
Client/server identification | IMAP ID extension ([RFC 2971][])
Authorization | OAuth2 ([RFC 6749][])
End-to-end encryption | [Autocrypt Level 1][], OpenPGP ([RFC 4880][]), Security Multiparts for MIME ([RFC 1847][]) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html)
Detect/prevent active attacks | [countermitm][] protocols
Compare public keys | [openpgp4fpr][] URI Scheme
---------------------------------|---------------------------------------------
Transport | IMAP v4 ([RFC 3501](https://tools.ietf.org/html/rfc3501)), SMTP ([RFC 5321](https://tools.ietf.org/html/rfc5321)) and Internet Message Format (IMF, [RFC 5322](https://tools.ietf.org/html/rfc5322))
Proxy | SOCKS5 ([RFC 1928](https://tools.ietf.org/html/rfc1928))
Embedded media | MIME Document Series ([RFC 2045](https://tools.ietf.org/html/rfc2045), [RFC 2046](https://tools.ietf.org/html/rfc2046)), Content-Disposition Header ([RFC 2183](https://tools.ietf.org/html/rfc2183)), Multipart/Related ([RFC 2387](https://tools.ietf.org/html/rfc2387))
Text and Quote encoding | Fixed, Flowed ([RFC 3676](https://tools.ietf.org/html/rfc3676))
Reactions | Reaction: Indicating Summary Reaction to a Message [RFC 9078](https://datatracker.ietf.org/doc/rfc9078/)
Filename encoding | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231))
Identify server folders | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154))
Push | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177))
Quota | IMAP QUOTA extension ([RFC 2087](https://tools.ietf.org/html/rfc2087))
Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162](https://tools.ietf.org/html/rfc7162))
Client/server identification | IMAP ID extension ([RFC 2971](https://datatracker.ietf.org/doc/html/rfc2971))
Authorization | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749))
End-to-end encryption | [Autocrypt Level 1](https://autocrypt.org/level1.html), OpenPGP ([RFC 4880](https://tools.ietf.org/html/rfc4880)), Security Multiparts for MIME ([RFC 1847](https://tools.ietf.org/html/rfc1847)) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html)
Header encryption | [Protected Headers for Cryptographic E-mail](https://datatracker.ietf.org/doc/draft-autocrypt-lamps-protected-headers/)
Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover][]
Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)
Messenger functions | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-mail-specification)
Detect mailing list | List-Id ([RFC 2919][]) and Precedence ([RFC 3834][])
User and chat colors | [XEP-0392][]: Consistent Color Generation
Send and receive system messages | Multipart/Report Media Type ([RFC 6522][])
Return receipts | Message Disposition Notification (MDN, [RFC 8098][], [RFC 3503][]) using the Chat-Disposition-Notification-To header
Detect mailing list | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834))
User and chat colors | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation
Send and receive system messages | Multipart/Report Media Type ([RFC 6522](https://tools.ietf.org/html/rfc6522))
Return receipts | Message Disposition Notification (MDN, [RFC 8098](https://tools.ietf.org/html/rfc8098), [RFC 3503](https://tools.ietf.org/html/rfc3503)) using the Chat-Disposition-Notification-To header
Locations | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/))
[Autocrypt Level 1]: https://autocrypt.org/level1.html
[countermitm]: https://countermitm.readthedocs.io/en/latest/
[openpgp4fpr]: https://metacode.biz/openpgp/openpgp4fpr
[Autodiscover]: https://learn.microsoft.com/en-us/exchange/autodiscover-service-for-exchange-2013
[XEP-0392]: https://xmpp.org/extensions/xep-0392.html
[RFC 1847]: https://tools.ietf.org/html/rfc1847
[RFC 1928]: https://tools.ietf.org/html/rfc1928
[RFC 2045]: https://tools.ietf.org/html/rfc2045
[RFC 2046]: https://tools.ietf.org/html/rfc2046
[RFC 2047]: https://tools.ietf.org/html/rfc2047
[RFC 2087]: https://tools.ietf.org/html/rfc2087
[RFC 2177]: https://tools.ietf.org/html/rfc2177
[RFC 2183]: https://tools.ietf.org/html/rfc2183
[RFC 2231]: https://tools.ietf.org/html/rfc2231
[RFC 2387]: https://tools.ietf.org/html/rfc2387
[RFC 2919]: https://tools.ietf.org/html/rfc2919
[RFC 2971]: https://tools.ietf.org/html/rfc2971
[RFC 3501]: https://tools.ietf.org/html/rfc3501
[RFC 3503]: https://tools.ietf.org/html/rfc3503
[RFC 3676]: https://tools.ietf.org/html/rfc3676
[RFC 3834]: https://tools.ietf.org/html/rfc3834
[RFC 4880]: https://tools.ietf.org/html/rfc4880
[RFC 5321]: https://tools.ietf.org/html/rfc5321
[RFC 5322]: https://tools.ietf.org/html/rfc5322
[RFC 6154]: https://tools.ietf.org/html/rfc6154
[RFC 6522]: https://tools.ietf.org/html/rfc6522
[RFC 6749]: https://tools.ietf.org/html/rfc6749
[RFC 7162]: https://tools.ietf.org/html/rfc7162
[RFC 8098]: https://tools.ietf.org/html/rfc8098
[RFC 9078]: https://tools.ietf.org/html/rfc9078