diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf8185b22..449161d7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install rustfmt and clippy - run: rustup toolchain install $RUSTUP_TOOLCHAIN --component rustfmt --component clippy + run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy - name: Cache rust cargo artifacts uses: swatinem/rust-cache@v2 - name: Run rustfmt @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@master - name: Install Rust ${{ matrix.rust }} - run: rustup toolchain install ${{ matrix.rust }} + run: rustup toolchain install --profile minimal ${{ matrix.rust }} - run: rustup override set ${{ matrix.rust }} - name: Cache rust cargo artifacts diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml new file mode 100644 index 000000000..9f8f83982 --- /dev/null +++ b/.github/workflows/deltachat-rpc-server.yml @@ -0,0 +1,45 @@ +# Manually triggered action to build deltachat-rpc-server binaries. + +name: Build deltachat-rpc-server binaries + +on: + workflow_dispatch: + +jobs: + build_server: + name: Build deltachat-rpc-server + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + artifact: gnu-linux-x86_64 + path: deltachat-rpc-server + target: x86_64-unknown-linux-gnu + + - os: windows-latest + artifact: win32.exe + path: deltachat-rpc-server.exe + target: i686-pc-windows-msvc + + - os: windows-latest + artifact: win64.exe + path: deltachat-rpc-server.exe + target: x86_64-pc-windows-msvc + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - name: Setup rust target + run: rustup target add ${{ matrix.target }} + + - name: Build + run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-${{ matrix.artifact }} + path: target/${{ matrix.target}}/release/${{ matrix.path }} + if-no-files-found: error diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 7d7f0145e..b7f2f34d9 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -9,7 +9,7 @@ on: jobs: pack-module: name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat" - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Install tree run: sudo apt install tree diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index b737ca00c..4ae546397 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@v3 @@ -65,7 +65,7 @@ jobs: pack-module: needs: prebuild name: Package deltachat-node and upload to download.delta.chat - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Install tree run: sudo apt install tree @@ -99,7 +99,7 @@ jobs: - name: Download Ubuntu prebuild uses: actions/download-artifact@v1 with: - name: ubuntu-18.04 + name: ubuntu-20.04 - name: Download macOS prebuild uses: actions/download-artifact@v1 with: @@ -111,11 +111,11 @@ jobs: - shell: bash run: | mkdir node/prebuilds - tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C node/prebuilds + tar -xvzf ubuntu-20.04/ubuntu-20.04.tar.gz -C node/prebuilds tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds tree node/prebuilds - rm -rf ubuntu-18.04 macos-latest windows-latest + rm -rf ubuntu-20.04 macos-latest windows-latest - name: Install dependencies without running scripts run: | npm install --ignore-scripts diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 73e8d0cc4..7836f9b33 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 72eb1eff8..a67da796a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,19 @@ ### Changes - use transaction in `Contact::add_or_lookup()` #4059 +- Organize the connection pool as a stack rather than a queue to ensure that + connection page cache is reused more often. + This speeds up tests by 28%, real usage will have lower speedup. #4065 +- Use transaction in `update_blocked_mailinglist_contacts`. #4058 +- Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 +- Updated provider database. - ability to send backup over network and QR code to setup second device #4007 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 +- Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted. + Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972 +- Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 ### API-Changes diff --git a/Cargo.lock b/Cargo.lock index 285f69661..44a858626 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1010,7 +1010,6 @@ dependencies = [ "bitflags", "chrono", "criterion", - "crossbeam-queue", "deltachat_derive", "email", "encoded-words", @@ -1027,11 +1026,11 @@ dependencies = [ "libc", "log", "mailparse", - "native-tls", "num-derive", "num-traits", "num_cpus", "once_cell", + "parking_lot", "percent-encoding", "pgp", "pretty_env_logger", diff --git a/Cargo.toml b/Cargo.toml index 26ebe0507..9e5729574 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,6 @@ backtrace = "0.3" base64 = "0.21" bitflags = "1.3" chrono = { version = "0.4", default-features=false, features = ["clock", "std"] } -crossbeam-queue = "0.3" email = { git = "https://github.com/deltachat/rust-email", branch = "master" } encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" } escaper = "0.1" @@ -52,12 +51,12 @@ kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" mailparse = "0.14" -native-tls = "0.2" num_cpus = "1.15" num-derive = "0.3" num-traits = "0.2" once_cell = "1.17.0" percent-encoding = "2.2" +parking_lot = "0.12" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } qrcodegen = "1.7.0" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 2ad9f2b83..e2958a4b2 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -36,5 +36,6 @@ tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] } [features] -default = [] +default = ["vendored"] webserver = ["env_logger", "axum", "tokio/full", "yerpc/support-axum"] +vendored = ["deltachat/vendored"] diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 039057017..aded96fcc 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -14,7 +14,7 @@ "c8": "^7.10.0", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", - "esbuild": "^0.14.11", + "esbuild": "^0.17.9", "http-server": "^14.1.1", "mocha": "^9.1.1", "node-fetch": "^2.6.1", @@ -24,13 +24,20 @@ "typescript": "^4.5.5", "ws": "^8.5.0" }, + "exports": { + ".": { + "require": "./dist/deltachat.cjs", + "import": "./dist/deltachat.js" + } + }, "license": "MPL-2.0", "main": "dist/deltachat.js", "name": "@deltachat/jsonrpc-client", "scripts": { - "build": "run-s generate-bindings extract-constants build:tsc build:bundle", + "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs", "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js", "build:tsc": "tsc", + "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs", "docs": "typedoc --out docs deltachat.ts", "example": "run-s build example:build example:start", "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js", diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index ef76fa82b..f7d7fc905 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -25,7 +25,35 @@ deltachat_rpc_client = [ line-length = 120 [tool.ruff] -select = ["E", "F", "W", "N", "YTT", "B", "C4", "ISC", "ICN", "PT", "RET", "SIM", "TID", "ARG", "DTZ", "ERA", "PLC", "PLE", "PLW", "PIE", "COM"] +select = [ + "E", "W", # pycodestyle + "F", # Pyflakes + "N", # pep8-naming + "I", # isort + + "ARG", # flake8-unused-arguments + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "COM", # flake8-commas + "DTZ", # flake8-datetimez + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PIE", # flake8-pie + "PT", # flake8-pytest-style + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "TID", # flake8-tidy-imports + "YTT", # flake8-2020 + + "ERA", # eradicate + + "PLC", # Pylint Convention + "PLE", # Pylint Error + "PLW", # Pylint Warning + + "RUF006" # asyncio-dangling-task +] line-length = 120 [tool.isort] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/account.py b/deltachat-rpc-client/src/deltachat_rpc_client/account.py index 63a9b169c..b1e9fea63 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py @@ -1,15 +1,15 @@ -from typing import TYPE_CHECKING, List, Optional, Tuple, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from ._utils import AttrDict from .chat import Chat from .const import ChatlistFlag, ContactFlag, SpecialContactId from .contact import Contact from .message import Message -from .rpc import Rpc if TYPE_CHECKING: from .deltachat import DeltaChat + from .rpc import Rpc @dataclass @@ -20,7 +20,7 @@ class Account: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.manager.rpc async def wait_for_event(self) -> AttrDict: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index c2e5ca364..7dac7d2f4 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -1,16 +1,17 @@ import calendar -from datetime import datetime -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from ._utils import AttrDict from .const import ChatVisibility from .contact import Contact from .message import Message -from .rpc import Rpc if TYPE_CHECKING: + from datetime import datetime + from .account import Account + from .rpc import Rpc @dataclass @@ -21,7 +22,7 @@ class Chat: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def delete(self) -> None: @@ -217,8 +218,8 @@ class Chat: async def get_locations( self, contact: Optional[Contact] = None, - timestamp_from: Optional[datetime] = None, - timestamp_to: Optional[datetime] = None, + timestamp_from: Optional["datetime"] = None, + timestamp_to: Optional["datetime"] = None, ) -> List[AttrDict]: """Get list of location snapshots for the given contact in the given timespan.""" time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/client.py b/deltachat-rpc-client/src/deltachat_rpc_client/client.py index 393ac3cc7..6f816e5de 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/client.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/client.py @@ -2,6 +2,7 @@ import inspect import logging from typing import ( + TYPE_CHECKING, Callable, Coroutine, Dict, @@ -13,8 +14,6 @@ from typing import ( Union, ) -from deltachat_rpc_client.account import Account - from ._utils import ( AttrDict, parse_system_add_remove, @@ -31,13 +30,16 @@ from .events import ( RawEvent, ) +if TYPE_CHECKING: + from deltachat_rpc_client.account import Account + class Client: """Simple Delta Chat client that listen to events of a single account.""" def __init__( self, - account: Account, + account: "Account", hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None, logger: Optional[logging.Logger] = None, ) -> None: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py index 7999d59ed..efb3e9297 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py @@ -1,12 +1,12 @@ -from typing import TYPE_CHECKING from dataclasses import dataclass +from typing import TYPE_CHECKING from ._utils import AttrDict -from .rpc import Rpc if TYPE_CHECKING: from .account import Account from .chat import Chat + from .rpc import Rpc @dataclass @@ -21,7 +21,7 @@ class Contact: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def block(self) -> None: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py index 16afe458b..c2cecd60d 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py @@ -1,8 +1,10 @@ -from typing import Dict, List +from typing import TYPE_CHECKING, Dict, List from ._utils import AttrDict from .account import Account -from .rpc import Rpc + +if TYPE_CHECKING: + from .rpc import Rpc class DeltaChat: @@ -11,7 +13,7 @@ class DeltaChat: This is the root of the object oriented API. """ - def __init__(self, rpc: Rpc) -> None: + def __init__(self, rpc: "Rpc") -> None: self.rpc = rpc async def add_account(self) -> Account: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 0e160445c..4896527b9 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -2,11 +2,13 @@ import inspect import re from abc import ABC, abstractmethod -from typing import Callable, Iterable, Iterator, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union -from ._utils import AttrDict from .const import EventType +if TYPE_CHECKING: + from ._utils import AttrDict + def _tuple_of(obj, type_: type) -> tuple: if not obj: @@ -80,7 +82,7 @@ class RawEvent(EventFilter): return (self.types, self.func) == (other.types, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.types and event.type not in self.types: return False return await self._call_func(event) @@ -118,7 +120,7 @@ class NewMessage(EventFilter): command: Optional[str] = None, is_bot: Optional[bool] = False, is_info: Optional[bool] = None, - func: Optional[Callable[[AttrDict], bool]] = None, + func: Optional[Callable[["AttrDict"], bool]] = None, ) -> None: super().__init__(func=func) self.is_bot = is_bot @@ -157,7 +159,7 @@ class NewMessage(EventFilter): ) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot: return False if self.is_info is not None and self.is_info != event.message_snapshot.is_info: @@ -199,7 +201,7 @@ class MemberListChanged(EventFilter): return (self.added, self.func) == (other.added, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.added is not None and self.added != event.member_added: return False return await self._call_func(event) @@ -231,7 +233,7 @@ class GroupImageChanged(EventFilter): return (self.deleted, self.func) == (other.deleted, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.deleted is not None and self.deleted != event.image_deleted: return False return await self._call_func(event) @@ -256,7 +258,7 @@ class GroupNameChanged(EventFilter): return self.func == other.func return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: return await self._call_func(event) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/message.py b/deltachat-rpc-client/src/deltachat_rpc_client/message.py index 7784d8dac..5ec30961a 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/message.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/message.py @@ -1,13 +1,13 @@ import json -from typing import TYPE_CHECKING, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, Union from ._utils import AttrDict from .contact import Contact -from .rpc import Rpc if TYPE_CHECKING: from .account import Account + from .rpc import Rpc @dataclass @@ -18,7 +18,7 @@ class Message: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def send_reaction(self, *reaction: str): diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 87e97f0e0..065f7744e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -1,8 +1,8 @@ +import asyncio import json import os from typing import AsyncGenerator, List, Optional -import asyncio import aiohttp import pytest_asyncio diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 47b4cf8e3..fc05854d6 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -1,8 +1,7 @@ +import asyncio from unittest.mock import MagicMock import pytest -import asyncio - from deltachat_rpc_client import EventType, events from deltachat_rpc_client.rpc import JsonRpcError diff --git a/deltachat-rpc-client/tests/test_webxdc.py b/deltachat-rpc-client/tests/test_webxdc.py index 22d9db0b4..8a0584d03 100644 --- a/deltachat-rpc-client/tests/test_webxdc.py +++ b/deltachat-rpc-client/tests/test_webxdc.py @@ -1,5 +1,4 @@ import pytest - from deltachat_rpc_client import EventType diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 619841fc9..a332e21b5 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -13,7 +13,7 @@ categories = ["cryptography", "std", "email"] name = "deltachat-rpc-server" [dependencies] -deltachat-jsonrpc = { path = "../deltachat-jsonrpc" } +deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false } anyhow = "1" env_logger = { version = "0.10.0" } @@ -23,3 +23,7 @@ serde_json = "1.0.91" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.25.0", features = ["io-std"] } yerpc = { version = "0.4.0", features = ["anyhow_expose"] } + +[features] +default = ["vendored"] +vendored = ["deltachat-jsonrpc/vendored"] diff --git a/python/pyproject.toml b/python/pyproject.toml index 4aa6666db..227fb73d2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ dependencies = [ "cffi>=1.0.0", "imap-tools", + "importlib_metadata;python_version<'3.8'", "pluggy", "requests", ] diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 048d8e3a7..3d2734cca 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,6 +1,9 @@ import sys -from pkg_resources import DistributionNotFound, get_distribution +if sys.version_info >= (3, 8): + from importlib.metadata import PackageNotFoundError, version +else: + from importlib_metadata import PackageNotFoundError, version from . import capi, events, hookspec # noqa from .account import Account, get_core_info # noqa @@ -11,8 +14,8 @@ from .hookspec import account_hookimpl, global_hookimpl # noqa from .message import Message # noqa try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: + __version__ = version(__name__) +except PackageNotFoundError: # package is not installed __version__ = "0.0.0.dev0-unknown" diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index b38afe88c..c25fcb85b 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1954,6 +1954,7 @@ def test_immediate_autodelete(acfactory, lp): assert msg.text == "hello" lp.sec("ac2: wait for close/expunge on autodelete") + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") ac2._evtracker.get_info_contains("close/expunge succeeded") lp.sec("ac2: check that message was autodeleted on server") @@ -1995,6 +1996,34 @@ def test_delete_multiple_messages(acfactory, lp): assert len(ac2.direct_imap.get_all_messages()) == 1 +def test_trash_multiple_messages(acfactory, lp): + ac1, ac2 = acfactory.get_online_accounts(2) + ac2.set_config("delete_to_trash", "1") + chat12 = acfactory.get_accepted_chat(ac1, ac2) + + lp.sec("ac1: sending 3 messages") + texts = ["first", "second", "third"] + for text in texts: + chat12.send_text(text) + + lp.sec("ac2: waiting for all messages on the other side") + to_delete = [] + for text in texts: + msg = ac2._evtracker.wait_next_incoming_message() + assert msg.text in texts + if text != "second": + to_delete.append(msg) + + lp.sec("ac2: deleting all messages except second") + assert len(to_delete) == len(texts) - 1 + ac2.delete_messages(to_delete) + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + + lp.sec("ac2: test that only one message is left") + ac2.direct_imap.select_config_folder("inbox") + assert len(ac2.direct_imap.get_all_messages()) == 1 + + def test_configure_error_msgs_wrong_pw(acfactory): configdict = acfactory.get_next_liveconfig() ac1 = acfactory.get_unconfigured_account() diff --git a/src/chat.rs b/src/chat.rs index dd88cdd30..522f55aa7 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -907,7 +907,8 @@ impl ChatId { async fn parent_query(self, context: &Context, fields: &str, f: F) -> Result> where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result, + T: Send + 'static, { let sql = &context.sql; let query = format!( diff --git a/src/config.rs b/src/config.rs index 67d77811f..8c1357a8e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,7 @@ //! # Key-value configuration management. +use std::str::FromStr; + use anyhow::{ensure, Context as _, Result}; use strum::{EnumProperty, IntoEnumIterator}; use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString}; @@ -173,6 +175,10 @@ pub enum Config { #[strum(props(default = "0"))] DeleteDeviceAfter, + /// Move messages to the Trash folder instead of marking them "\Deleted". Overrides + /// `ProviderOptions::delete_to_trash`. + DeleteToTrash, + /// Save raw MIME messages with headers in the database if true. SaveMimeHeaders, @@ -227,6 +233,9 @@ pub enum Config { /// Configured "Sent" folder. ConfiguredSentboxFolder, + /// Configured "Trash" folder. + ConfiguredTrashFolder, + /// Unix timestamp of the last successful configuration. ConfiguredTimestamp, @@ -327,30 +336,37 @@ impl Context { } } - /// Returns 32-bit signed integer configuration value for the given key. - pub async fn get_config_int(&self, key: Config) -> Result { + /// Returns Some(T) if a value for the given key exists and was successfully parsed. + /// Returns None if could not parse. + pub async fn get_config_parsed(&self, key: Config) -> Result> { self.get_config(key) .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + .map(|s: Option| s.and_then(|s| s.parse().ok())) + } + + /// Returns 32-bit signed integer configuration value for the given key. + pub async fn get_config_int(&self, key: Config) -> Result { + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) } /// Returns 64-bit signed integer configuration value for the given key. pub async fn get_config_i64(&self, key: Config) -> Result { - self.get_config(key) - .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) } /// Returns 64-bit unsigned integer configuration value for the given key. pub async fn get_config_u64(&self, key: Config) -> Result { - self.get_config(key) - .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) + } + + /// Returns boolean configuration value (if any) for the given key. + pub async fn get_config_bool_opt(&self, key: Config) -> Result> { + Ok(self.get_config_parsed::(key).await?.map(|x| x != 0)) } /// Returns boolean configuration value for the given key. pub async fn get_config_bool(&self, key: Config) -> Result { - Ok(self.get_config_int(key).await? != 0) + Ok(self.get_config_bool_opt(key).await?.unwrap_or_default()) } /// Returns true if movebox ("DeltaChat" folder) should be watched. @@ -550,7 +566,6 @@ fn get_config_keys_string() -> String { #[cfg(test)] mod tests { - use std::str::FromStr; use std::string::ToString; use num_traits::FromPrimitive; diff --git a/src/configure.rs b/src/configure.rs index 4350f08f6..565169470 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -249,7 +249,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> { } } }, - strict_tls: Some(provider.strict_tls), + strict_tls: Some(provider.opt.strict_tls), }) .collect(); @@ -338,7 +338,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> { .collect(); let provider_strict_tls = param .provider - .map_or(socks5_config.is_some(), |provider| provider.strict_tls); + .map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls); let smtp_config_task = task::spawn(async move { let mut smtp_configured = false; diff --git a/src/constants.rs b/src/constants.rs index 2775f97ef..4c19abba6 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -201,7 +201,7 @@ pub const BALANCED_IMAGE_SIZE: u32 = 1280; pub const WORSE_IMAGE_SIZE: u32 = 640; // this value can be increased if the folder configuration is changed and must be redone on next program start -pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; +pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4; #[cfg(test)] mod tests { diff --git a/src/contact.rs b/src/contact.rs index d27ccc361..2ae958603 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -869,47 +869,45 @@ impl Contact { Ok(ret) } - // add blocked mailinglists as contacts - // to allow unblocking them as if they are contacts - // (this way, only one unblock-ffi is needed and only one set of ui-functions, - // from the users perspective, - // there is not much difference in an email- and a mailinglist-address) + /// Adds blocked mailinglists as contacts + /// to allow unblocking them as if they are contacts + /// (this way, only one unblock-ffi is needed and only one set of ui-functions, + /// from the users perspective, + /// there is not much difference in an email- and a mailinglist-address) async fn update_blocked_mailinglist_contacts(context: &Context) -> Result<()> { - let blocked_mailinglists = context + context .sql - .query_map( - "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;", - paramsv![Chattype::Mailinglist, Blocked::Yes], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - |rows| { - rows.collect::, _>>() - .map_err(Into::into) - }, - ) + .transaction(move |transaction| { + let mut stmt = transaction + .prepare("SELECT name, grpid FROM chats WHERE type=? AND blocked=?")?; + let rows = stmt.query_map(params![Chattype::Mailinglist, Blocked::Yes], |row| { + let name: String = row.get(0)?; + let grpid: String = row.get(1)?; + Ok((name, grpid)) + })?; + let blocked_mailinglists = rows.collect::, _>>()?; + for (name, grpid) in blocked_mailinglists { + let count = transaction.query_row( + "SELECT COUNT(id) FROM contacts WHERE addr=?", + [&grpid], + |row| { + let count: isize = row.get(0)?; + Ok(count) + }, + )?; + if count == 0 { + transaction.execute("INSERT INTO contacts (addr) VALUES (?)", [&grpid])?; + } + + // Always do an update in case the blocking is reset or name is changed. + transaction.execute( + "UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?", + params![&name, Origin::MailinglistAddress, &grpid], + )?; + } + Ok(()) + }) .await?; - for (name, grpid) in blocked_mailinglists { - if !context - .sql - .exists( - "SELECT COUNT(id) FROM contacts WHERE addr=?;", - paramsv![grpid], - ) - .await? - { - context - .sql - .execute("INSERT INTO contacts (addr) VALUES (?);", paramsv![grpid]) - .await?; - } - // always do an update in case the blocking is reset or name is changed - context - .sql - .execute( - "UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?;", - paramsv![name, Origin::MailinglistAddress, grpid], - ) - .await?; - } Ok(()) } diff --git a/src/context.rs b/src/context.rs index 28ddae26e..919af0ed0 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; -use anyhow::{bail, ensure, Result}; +use anyhow::{bail, ensure, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; use ratelimit::Ratelimit; use tokio::sync::{Mutex, RwLock}; @@ -655,6 +655,10 @@ impl Context { .get_config(Config::ConfiguredMvboxFolder) .await? .unwrap_or_else(|| "".to_string()); + let configured_trash_folder = self + .get_config(Config::ConfiguredTrashFolder) + .await? + .unwrap_or_else(|| "".to_string()); let mut res = get_info(); @@ -721,6 +725,7 @@ impl Context { res.insert("configured_inbox_folder", configured_inbox_folder); res.insert("configured_sentbox_folder", configured_sentbox_folder); res.insert("configured_mvbox_folder", configured_mvbox_folder); + res.insert("configured_trash_folder", configured_trash_folder); res.insert("mdns_enabled", mdns_enabled.to_string()); res.insert("e2ee_enabled", e2ee_enabled.to_string()); res.insert( @@ -754,6 +759,12 @@ impl Context { .await? .to_string(), ); + res.insert( + "delete_to_trash", + self.get_config(Config::DeleteToTrash) + .await? + .unwrap_or_else(|| "".to_string()), + ); res.insert( "last_housekeeping", self.get_config_int(Config::LastHousekeeping) @@ -919,6 +930,33 @@ impl Context { Ok(mvbox.as_deref() == Some(folder_name)) } + /// Returns true if given folder name is the name of the trash folder. + pub async fn is_trash(&self, folder_name: &str) -> Result { + let trash = self.get_config(Config::ConfiguredTrashFolder).await?; + Ok(trash.as_deref() == Some(folder_name)) + } + + pub(crate) async fn should_delete_to_trash(&self) -> Result { + if let Some(v) = self.get_config_bool_opt(Config::DeleteToTrash).await? { + return Ok(v); + } + if let Some(provider) = self.get_configured_provider().await? { + return Ok(provider.opt.delete_to_trash); + } + Ok(false) + } + + /// Returns `target` for deleted messages as per `imap` table. Empty string means "delete w/o + /// moving to trash". + pub(crate) async fn get_delete_msgs_target(&self) -> Result { + if !self.should_delete_to_trash().await? { + return Ok("".into()); + } + self.get_config(Config::ConfiguredTrashFolder) + .await? + .context("No configured trash folder") + } + pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf { let mut blob_fname = OsString::new(); blob_fname.push(dbfile.file_name().unwrap_or_default()); diff --git a/src/download.rs b/src/download.rs index fb56f729c..5b544fe31 100644 --- a/src/download.rs +++ b/src/download.rs @@ -138,7 +138,7 @@ impl Job { context .sql .query_row_optional( - "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''", + "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target=folder", paramsv![msg.rfc724_mid], |row| { let server_uid: u32 = row.get(0)?; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 2e38386ea..606c7270f 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -588,19 +588,25 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<() now - max(delete_server_after, MIN_DELETE_SERVER_AFTER), ), }; + let target = context.get_delete_msgs_target().await?; context .sql .execute( "UPDATE imap - SET target='' + SET target=? WHERE rfc724_mid IN ( SELECT rfc724_mid FROM msgs WHERE ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?) OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?)) )", - paramsv![threshold_timestamp, threshold_timestamp_extended, now], + paramsv![ + target, + threshold_timestamp, + threshold_timestamp_extended, + now, + ], ) .await?; diff --git a/src/imap.rs b/src/imap.rs index 56a2cc6e6..d7fcf1397 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -113,13 +113,15 @@ impl async_imap::Authenticator for OAuth2 { } } -#[derive(Debug, PartialEq, Clone, Copy)] -enum FolderMeaning { +#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)] +pub enum FolderMeaning { Unknown, Spam, + Inbox, + Mvbox, Sent, + Trash, Drafts, - Other, /// Virtual folders. /// @@ -131,13 +133,15 @@ enum FolderMeaning { } impl FolderMeaning { - fn to_config(self) -> Option { + pub fn to_config(self) -> Option { match self { FolderMeaning::Unknown => None, FolderMeaning::Spam => None, + FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder), + FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder), FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder), + FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder), FolderMeaning::Drafts => None, - FolderMeaning::Other => None, FolderMeaning::Virtual => None, } } @@ -270,7 +274,7 @@ impl Imap { param .provider .map_or(param.socks5_config.is_some(), |provider| { - provider.strict_tls + provider.opt.strict_tls }), idle_interrupt_receiver, )?; @@ -449,7 +453,7 @@ impl Imap { &mut self, context: &Context, watch_folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, ) -> Result<()> { if !context.sql.is_open().await { // probably shutdown @@ -458,7 +462,7 @@ impl Imap { self.prepare(context).await?; let msgs_fetched = self - .fetch_new_messages(context, watch_folder, is_spam_folder, false) + .fetch_new_messages(context, watch_folder, folder_meaning, false) .await .context("fetch_new_messages")?; if msgs_fetched && context.get_config_delete_device_after().await?.is_some() { @@ -490,49 +494,60 @@ impl Imap { pub(crate) async fn resync_folder_uids( &mut self, context: &Context, - folder: String, + folder: &str, + folder_meaning: FolderMeaning, ) -> Result<()> { // Collect pairs of UID and Message-ID. - let mut msg_ids = BTreeMap::new(); + let mut msgs = BTreeMap::new(); let session = self .session .as_mut() .context("IMAP No connection established")?; - session.select_folder(context, Some(&folder)).await?; + session.select_folder(context, Some(folder)).await?; let mut list = session .uid_fetch("1:*", RFC724MID_UID) .await .with_context(|| format!("can't resync folder {folder}"))?; while let Some(fetch) = list.next().await { - let msg = fetch?; + let fetch = fetch?; + let headers = match get_fetch_headers(&fetch) { + Ok(headers) => headers, + Err(err) => { + warn!(context, "Failed to parse FETCH headers: {}", err); + continue; + } + }; + let message_id = prefetch_get_message_id(&headers); - // Get Message-ID - let message_id = - get_fetch_headers(&msg).map_or(None, |headers| prefetch_get_message_id(&headers)); - - if let (Some(uid), Some(rfc724_mid)) = (msg.uid, message_id) { - msg_ids.insert(uid, rfc724_mid); + if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) { + msgs.insert( + uid, + ( + rfc724_mid, + target_folder(context, folder, folder_meaning, &headers).await?, + ), + ); } } info!( context, "Resync: collected {} message IDs in folder {}", - msg_ids.len(), - &folder + msgs.len(), + folder, ); - let uid_validity = get_uidvalidity(context, &folder).await?; + let uid_validity = get_uidvalidity(context, folder).await?; // Write collected UIDs to SQLite database. context .sql .transaction(move |transaction| { transaction.execute("DELETE FROM imap WHERE folder=?", params![folder])?; - for (uid, rfc724_mid) in &msg_ids { + for (uid, (rfc724_mid, target)) in &msgs { // This may detect previously undetected moved // messages, so we update server_folder too. transaction.execute( @@ -541,7 +556,7 @@ impl Imap { ON CONFLICT(folder, uid, uidvalidity) DO UPDATE SET rfc724_mid=excluded.rfc724_mid, target=excluded.target", - params![rfc724_mid, folder, uid, uid_validity, folder], + params![rfc724_mid, folder, uid, uid_validity, target], )?; } Ok(()) @@ -683,10 +698,10 @@ impl Imap { &mut self, context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, fetch_existing_msgs: bool, ) -> Result { - if should_ignore_folder(context, folder, is_spam_folder).await? { + if should_ignore_folder(context, folder, folder_meaning).await? { info!(context, "Not fetching from {}", folder); return Ok(false); } @@ -713,8 +728,6 @@ impl Imap { }; let read_cnt = msgs.len(); - let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?) - .unwrap_or_default(); let download_limit = context.download_limit().await?; let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1); let mut uid_message_ids = BTreeMap::new(); @@ -732,14 +745,7 @@ impl Imap { // Get the Message-ID or generate a fake one to identify the message in the database. let message_id = prefetch_get_or_create_message_id(&headers); - - let target = match target_folder(context, folder, is_spam_folder, &headers).await? { - Some(config) => match context.get_config(config).await? { - Some(target) => target, - None => folder.to_string(), - }, - None => folder.to_string(), - }; + let target = target_folder(context, folder, folder_meaning, &headers).await?; context .sql @@ -763,14 +769,13 @@ impl Imap { // Never download messages directly from the spam folder. // If the sender is known, the message will be moved to the Inbox or Mvbox // and then we download the message from there. - // Also see `spam_target_folder()`. - && !is_spam_folder + // Also see `spam_target_folder_cfg()`. + && folder_meaning != FolderMeaning::Spam && prefetch_should_download( context, &headers, &message_id, fetch_response.flags(), - show_emails, ) .await.context("prefetch_should_download")? { @@ -870,17 +875,21 @@ impl Imap { .context("failed to get recipients from the inbox")?; if context.get_config_bool(Config::FetchExistingMsgs).await? { - for config in &[ - Config::ConfiguredMvboxFolder, - Config::ConfiguredInboxFolder, - Config::ConfiguredSentboxFolder, + for meaning in [ + FolderMeaning::Mvbox, + FolderMeaning::Inbox, + FolderMeaning::Sent, ] { - if let Some(folder) = context.get_config(*config).await? { + let config = match meaning.to_config() { + Some(c) => c, + None => continue, + }; + if let Some(folder) = context.get_config(config).await? { info!( context, "Fetching existing messages from folder \"{}\"", folder ); - self.fetch_new_messages(context, &folder, false, true) + self.fetch_new_messages(context, &folder, meaning, true) .await .context("could not fetch existing messages")?; } @@ -952,44 +961,60 @@ impl Session { return Ok(()); } Err(err) => { + if context.should_delete_to_trash().await? { + error!( + context, + "Cannot move messages {} to {}, no fallback to COPY/DELETE because \ + delete_to_trash is set. Error: {:#}", + set, + target, + err, + ); + return Err(err.into()); + } warn!( context, - "Cannot move message, fallback to COPY/DELETE {} to {}: {}", + "Cannot move messages, fallback to COPY/DELETE {} to {}: {}", set, target, err ); } } - } else { + } + + // Server does not support MOVE or MOVE failed. + // Copy messages to the destination folder if needed and mark records for deletion. + let copy = !context.is_trash(target).await?; + if copy { info!( context, "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target ); + self.uid_copy(&set, &target).await?; + } else { + error!( + context, + "Server does not support MOVE, fallback to DELETE {} to {}", set, target, + ); } - - // Server does not support MOVE or MOVE failed. - // Copy the message to the destination folder and mark the record for deletion. - match self.uid_copy(&set, &target).await { - Ok(()) => { - context - .sql - .execute( - &format!( - "UPDATE imap SET target='' WHERE id IN ({})", - sql::repeat_vars(row_ids.len()) - ), - rusqlite::params_from_iter(row_ids), - ) - .await - .context("cannot plan deletion of copied messages")?; - context.emit_event(EventType::ImapMessageMoved(format!( - "IMAP messages {set} copied to {target}" - ))); - Ok(()) - } - Err(err) => Err(err.into()), + context + .sql + .execute( + &format!( + "UPDATE imap SET target='' WHERE id IN ({})", + sql::repeat_vars(row_ids.len()) + ), + rusqlite::params_from_iter(row_ids), + ) + .await + .context("cannot plan deletion of messages")?; + if copy { + context.emit_event(EventType::ImapMessageMoved(format!( + "IMAP messages {set} copied to {target}" + ))); } + Ok(()) } /// Moves and deletes messages as planned in the `imap` table. @@ -1644,7 +1669,7 @@ impl Imap { } } - let folder_meaning = get_folder_meaning(&folder); + let folder_meaning = get_folder_meaning_by_attrs(folder.attributes()); let folder_name_meaning = get_folder_meaning_by_name(folder.name()); if let Some(config) = folder_meaning.to_config() { // Always takes precedence @@ -1776,7 +1801,7 @@ async fn should_move_out_of_spam( /// If this returns None, the message will not be moved out of the /// Spam folder, and as `fetch_new_messages()` doesn't download /// messages from the Spam folder, the message will be ignored. -async fn spam_target_folder( +async fn spam_target_folder_cfg( context: &Context, headers: &[mailparse::MailHeader<'_>], ) -> Result> { @@ -1797,18 +1822,18 @@ async fn spam_target_folder( /// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if /// the message needs to be moved from `folder`. Otherwise returns `None`. -pub async fn target_folder( +pub async fn target_folder_cfg( context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, headers: &[mailparse::MailHeader<'_>], ) -> Result> { if context.is_mvbox(folder).await? { return Ok(None); } - if is_spam_folder { - spam_target_folder(context, headers).await + if folder_meaning == FolderMeaning::Spam { + spam_target_folder_cfg(context, headers).await } else if needs_move_to_mvbox(context, headers).await? { Ok(Some(Config::ConfiguredMvboxFolder)) } else { @@ -1816,6 +1841,21 @@ pub async fn target_folder( } } +pub async fn target_folder( + context: &Context, + folder: &str, + folder_meaning: FolderMeaning, + headers: &[mailparse::MailHeader<'_>], +) -> Result { + match target_folder_cfg(context, folder, folder_meaning, headers).await? { + Some(config) => match context.get_config(config).await? { + Some(target) => Ok(target), + None => Ok(folder.to_string()), + }, + None => Ok(folder.to_string()), + } +} + async fn needs_move_to_mvbox( context: &Context, headers: &[mailparse::MailHeader<'_>], @@ -1940,10 +1980,10 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning { } } -fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { - for attr in folder_name.attributes() { +fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning { + for attr in folder_attrs { match attr { - NameAttribute::Trash => return FolderMeaning::Other, + NameAttribute::Trash => return FolderMeaning::Trash, NameAttribute::Sent => return FolderMeaning::Sent, NameAttribute::Junk => return FolderMeaning::Spam, NameAttribute::Drafts => return FolderMeaning::Drafts, @@ -1961,6 +2001,13 @@ fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { FolderMeaning::Unknown } +pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning { + match get_folder_meaning_by_attrs(folder.attributes()) { + FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()), + meaning => meaning, + } +} + /// Parses the headers from the FETCH result. fn get_fetch_headers(prefetch_msg: &Fetch) -> Result> { match prefetch_msg.header() { @@ -2005,7 +2052,6 @@ pub(crate) async fn prefetch_should_download( headers: &[mailparse::MailHeader<'_>], message_id: &str, mut flags: impl Iterator>, - show_emails: ShowEmails, ) -> Result { if message::rfc724_mid_exists(context, message_id) .await? @@ -2065,6 +2111,9 @@ pub(crate) async fn prefetch_should_download( }) .unwrap_or_default(); + let show_emails = + ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default(); + let show = is_autocrypt_setup_message || match show_emails { ShowEmails::Off => is_chat_message || is_reply_to_chat_message, @@ -2272,7 +2321,7 @@ pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result async fn should_ignore_folder( context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, ) -> Result { if !context.get_config_bool(Config::OnlyFetchMvbox).await? { return Ok(false); @@ -2281,7 +2330,7 @@ async fn should_ignore_folder( // Still respect the SentboxWatch setting. return Ok(!context.get_config_bool(Config::SentboxWatch).await?); } - Ok(!(context.is_mvbox(folder).await? || is_spam_folder)) + Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam)) } /// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000 @@ -2564,14 +2613,13 @@ mod tests { }; let (headers, _) = mailparse::parse_headers(bytes)?; - - let is_spam_folder = folder == "Spam"; - let actual = - if let Some(config) = target_folder(&t, folder, is_spam_folder, &headers).await? { - t.get_config(config).await? - } else { - None - }; + let actual = if let Some(config) = + target_folder_cfg(&t, folder, get_folder_meaning_by_name(folder), &headers).await? + { + t.get_config(config).await? + } else { + None + }; let expected = if expected_destination == folder { None diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 4c0910fbe..fbce499b7 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -7,7 +7,7 @@ use futures_lite::FutureExt; use super::session::Session; use super::Imap; -use crate::imap::client::IMAP_TIMEOUT; +use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning}; use crate::{context::Context, scheduler::InterruptInfo}; const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60); @@ -113,6 +113,7 @@ impl Imap { &mut self, context: &Context, watch_folder: Option, + folder_meaning: FolderMeaning, ) -> InterruptInfo { // Idle using polling. This is also needed if we're not yet configured - // in this case, we're waiting for a configure job (and an interrupt). @@ -173,7 +174,7 @@ impl Imap { // will have already fetched the messages so perform_*_fetch // will not find any new. match self - .fetch_new_messages(context, &watch_folder, false, false) + .fetch_new_messages(context, &watch_folder, folder_meaning, false) .await { Ok(res) => { diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index 991b6b38f..29051e7b9 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, time::Instant}; use anyhow::{Context as _, Result}; use futures::stream::StreamExt; -use super::{get_folder_meaning, get_folder_meaning_by_name}; +use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name}; use crate::config::Config; use crate::imap::Imap; use crate::log::LogExt; @@ -33,7 +33,7 @@ impl Imap { let mut folder_configs = BTreeMap::new(); for folder in folders { - let folder_meaning = get_folder_meaning(&folder); + let folder_meaning = get_folder_meaning_by_attrs(folder.attributes()); if folder_meaning == FolderMeaning::Virtual { // Gmail has virtual folders that should be skipped. For example, // emails appear in the inbox and under "All Mail" as soon as it is @@ -53,21 +53,22 @@ impl Imap { .or_insert_with(|| folder.name().to_string()); } - let is_drafts = folder_meaning == FolderMeaning::Drafts - || (folder_meaning == FolderMeaning::Unknown - && folder_name_meaning == FolderMeaning::Drafts); - let is_spam_folder = folder_meaning == FolderMeaning::Spam - || (folder_meaning == FolderMeaning::Unknown - && folder_name_meaning == FolderMeaning::Spam); + let folder_meaning = match folder_meaning { + FolderMeaning::Unknown => folder_name_meaning, + _ => folder_meaning, + }; // Don't scan folders that are watched anyway - if !watched_folders.contains(&folder.name().to_string()) && !is_drafts { + if !watched_folders.contains(&folder.name().to_string()) + && folder_meaning != FolderMeaning::Drafts + && folder_meaning != FolderMeaning::Trash + { let session = self.session.as_mut().context("no session")?; // Drain leftover unsolicited EXISTS messages session.server_sent_unsolicited_exists(context)?; loop { - self.fetch_move_delete(context, folder.name(), is_spam_folder) + self.fetch_move_delete(context, folder.name(), folder_meaning) .await .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder"); @@ -80,15 +81,15 @@ impl Imap { } } - // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted. - context - .set_config( - Config::ConfiguredSentboxFolder, - folder_configs - .get(&Config::ConfiguredSentboxFolder) - .map(|s| s.as_str()), - ) - .await?; + // Set configs for necessary folders. Or reset if the folder was deleted. + for conf in [ + Config::ConfiguredSentboxFolder, + Config::ConfiguredTrashFolder, + ] { + context + .set_config(conf, folder_configs.get(&conf).map(|s| s.as_str())) + .await?; + } last_scan.replace(Instant::now()); Ok(true) diff --git a/src/imex.rs b/src/imex.rs index 24786d7e9..09c3fe432 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -431,8 +431,6 @@ async fn import_backup( context.get_dbfile().display() ); - context.sql.config_cache.write().await.clear(); - let mut archive = Archive::new(backup_file); let mut entries = archive.entries()?; @@ -760,29 +758,34 @@ async fn export_database(context: &Context, dest: &Path, passphrase: String) -> context.sql.set_raw_config_int("backup_time", now).await?; sql::housekeeping(context).await.ok_or_log(context); - let conn = context.sql.get_conn().await?; - tokio::task::block_in_place(move || { - conn.execute("VACUUM;", params![]) - .map_err(|err| warn!(context, "Vacuum failed, exporting anyway {err}")) - .ok(); - conn.execute( - "ATTACH DATABASE ? AS backup KEY ?", - paramsv![dest, passphrase], - ) - .context("failed to attach backup database")?; - let res = conn - .query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(())) - .context("failed to export to attached backup database"); - conn.execute("DETACH DATABASE backup", []) - .context("failed to detach backup database")?; - res?; - Ok(()) - }) + context + .sql + .call(|conn| { + conn.execute("VACUUM;", params![]) + .map_err(|err| warn!(context, "Vacuum failed, exporting anyway {err}")) + .ok(); + conn.execute( + "ATTACH DATABASE ? AS backup KEY ?", + paramsv![dest, passphrase], + ) + .context("failed to attach backup database")?; + let res = conn + .query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(())) + .context("failed to export to attached backup database"); + conn.execute("DETACH DATABASE backup", []) + .context("failed to detach backup database")?; + res?; + Ok(()) + }) + .await } #[cfg(test)] mod tests { + use std::time::Duration; + use ::pgp::armor::BlockType; + use tokio::task; use super::*; use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE}; @@ -930,6 +933,46 @@ mod tests { Ok(()) } + /// This is a regression test for + /// https://github.com/deltachat/deltachat-android/issues/2263 + /// where the config cache wasn't reset properly after a backup. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_import_backup_reset_config_cache() -> Result<()> { + let backup_dir = tempfile::tempdir()?; + let context1 = TestContext::new_alice().await; + let context2 = TestContext::new().await; + assert!(!context2.is_configured().await?); + + // export from context1 + imex(&context1, ImexMode::ExportBackup, backup_dir.path(), None).await?; + + // import to context2 + let backup = has_backup(&context2, backup_dir.path()).await?; + let context2_cloned = context2.clone(); + let handle = task::spawn(async move { + imex( + &context2_cloned, + ImexMode::ImportBackup, + backup.as_ref(), + None, + ) + .await + .unwrap(); + }); + + while !handle.is_finished() { + // The database is still unconfigured; + // fill the config cache with the old value. + context2.is_configured().await.ok(); + tokio::time::sleep(Duration::from_micros(1)).await; + } + + // Assert that the config cache has the new value now. + assert!(context2.is_configured().await?); + + Ok(()) + } + #[test] fn test_normalize_setup_code() { let norm = normalize_setup_code("123422343234423452346234723482349234"); diff --git a/src/job.rs b/src/job.rs index 51c06d372..3279843a9 100644 --- a/src/job.rs +++ b/src/job.rs @@ -12,7 +12,7 @@ use deltachat_derive::{FromSql, ToSql}; use rand::{thread_rng, Rng}; use crate::context::Context; -use crate::imap::Imap; +use crate::imap::{get_folder_meaning, FolderMeaning, Imap}; use crate::param::Params; use crate::scheduler::InterruptInfo; use crate::tools::time; @@ -172,8 +172,12 @@ impl Job { let mut any_failed = false; for folder in all_folders { + let folder_meaning = get_folder_meaning(&folder); + if folder_meaning == FolderMeaning::Virtual { + continue; + } if let Err(e) = imap - .resync_folder_uids(context, folder.name().to_string()) + .resync_folder_uids(context, folder.name(), folder_meaning) .await { warn!(context, "{:#}", e); diff --git a/src/key.rs b/src/key.rs index bee59c60b..3b48fe19e 100644 --- a/src/key.rs +++ b/src/key.rs @@ -289,39 +289,41 @@ pub async fn store_self_keypair( keypair: &KeyPair, default: KeyPairUse, ) -> Result<()> { - let mut conn = context.sql.get_conn().await?; - let transaction = conn.transaction()?; + context + .sql + .transaction(|transaction| { + let public_key = DcKey::to_bytes(&keypair.public); + let secret_key = DcKey::to_bytes(&keypair.secret); + transaction + .execute( + "DELETE FROM keypairs WHERE public_key=? OR private_key=?;", + paramsv![public_key, secret_key], + ) + .context("failed to remove old use of key")?; + if default == KeyPairUse::Default { + transaction + .execute("UPDATE keypairs SET is_default=0;", paramsv![]) + .context("failed to clear default")?; + } + let is_default = match default { + KeyPairUse::Default => i32::from(true), + KeyPairUse::ReadOnly => i32::from(false), + }; - let public_key = DcKey::to_bytes(&keypair.public); - let secret_key = DcKey::to_bytes(&keypair.secret); - transaction - .execute( - "DELETE FROM keypairs WHERE public_key=? OR private_key=?;", - paramsv![public_key, secret_key], - ) - .context("failed to remove old use of key")?; - if default == KeyPairUse::Default { - transaction - .execute("UPDATE keypairs SET is_default=0;", paramsv![]) - .context("failed to clear default")?; - } - let is_default = match default { - KeyPairUse::Default => i32::from(true), - KeyPairUse::ReadOnly => i32::from(false), - }; + let addr = keypair.addr.to_string(); + let t = time(); - let addr = keypair.addr.to_string(); - let t = time(); - - transaction - .execute( - "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) + transaction + .execute( + "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) VALUES (?,?,?,?,?);", - paramsv![addr, is_default, public_key, secret_key, t], - ) - .context("failed to insert keypair")?; + paramsv![addr, is_default, public_key, secret_key, t], + ) + .context("failed to insert keypair")?; - transaction.commit()?; + Ok(()) + }) + .await?; Ok(()) } diff --git a/src/location.rs b/src/location.rs index 24a202c17..6698dea3b 100644 --- a/src/location.rs +++ b/src/location.rs @@ -601,32 +601,38 @@ pub(crate) async fn save( .. } = location; - let conn = context.sql.get_conn().await?; - let mut stmt_test = - conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?; - let mut stmt_insert = conn.prepare_cached(stmt_insert)?; + context + .sql + .call(|conn| { + let mut stmt_test = conn + .prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?; + let mut stmt_insert = conn.prepare_cached(stmt_insert)?; - let exists = stmt_test.exists(paramsv![timestamp, contact_id])?; + let exists = stmt_test.exists(paramsv![timestamp, contact_id])?; - if independent || !exists { - stmt_insert.execute(paramsv![ - timestamp, - contact_id, - chat_id, - latitude, - longitude, - accuracy, - independent, - ])?; + if independent || !exists { + stmt_insert.execute(paramsv![ + timestamp, + contact_id, + chat_id, + latitude, + longitude, + accuracy, + independent, + ])?; - if timestamp > newest_timestamp { - // okay to drop, as we use cached prepared statements - drop(stmt_test); - drop(stmt_insert); - newest_timestamp = timestamp; - newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?); - } - } + if timestamp > newest_timestamp { + // okay to drop, as we use cached prepared statements + drop(stmt_test); + drop(stmt_insert); + newest_timestamp = timestamp; + newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?); + } + } + + Ok(()) + }) + .await?; } Ok(newest_location_id) diff --git a/src/message.rs b/src/message.rs index e4bae9a67..a7b2e27e3 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1386,11 +1386,12 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> { context.emit_event(EventType::WebxdcInstanceDeleted { msg_id: *msg_id }); } + let target = context.get_delete_msgs_target().await?; context .sql .execute( - "UPDATE imap SET target='' WHERE rfc724_mid=?", - paramsv![msg.rfc724_mid], + "UPDATE imap SET target=? WHERE rfc724_mid=?", + paramsv![target, msg.rfc724_mid], ) .await?; diff --git a/src/peerstate.rs b/src/peerstate.rs index acac59a65..e7433f0f3 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -186,7 +186,7 @@ impl Peerstate { async fn from_stmt( context: &Context, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, ) -> Result> { let peerstate = context .sql diff --git a/src/provider.rs b/src/provider.rs index 565d54d10..2d37b1347 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -129,6 +129,16 @@ pub struct Provider { /// Default configuration values to set when provider is configured. pub config_defaults: Option>, + /// Type of OAuth 2 authorization if provider supports it. + pub oauth2_authorizer: Option, + + /// Options with good defaults. + pub opt: ProviderOptions, +} + +/// Provider options with good defaults. +#[derive(Debug, PartialEq, Eq)] +pub struct ProviderOptions { /// True if provider is known to use use proper, /// not self-signed certificates. pub strict_tls: bool, @@ -136,8 +146,18 @@ pub struct Provider { /// Maximum number of recipients the provider allows to send a single email to. pub max_smtp_rcpt_to: Option, - /// Type of OAuth 2 authorization if provider supports it. - pub oauth2_authorizer: Option, + /// Move messages to the Trash folder instead of marking them "\Deleted". + pub delete_to_trash: bool, +} + +impl Default for ProviderOptions { + fn default() -> Self { + Self { + strict_tls: true, + max_smtp_rcpt_to: None, + delete_to_trash: false, + } + } } /// Get resolver to query MX records. diff --git a/src/provider/data.rs b/src/provider/data.rs index 21084f821..73eb49634 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1,13 +1,14 @@ // file generated by src/provider/update.py -use std::collections::HashMap; - -use once_cell::sync::Lazy; - use crate::provider::Protocol::*; use crate::provider::Socket::*; use crate::provider::UsernamePattern::*; -use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status}; +use crate::provider::{ + Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status, +}; +use std::collections::HashMap; + +use once_cell::sync::Lazy; // 163.md: 163.com static P_163: Lazy = Lazy::new(|| Provider { @@ -32,9 +33,8 @@ static P_163: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -61,9 +61,8 @@ static P_AKTIVIX_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -79,9 +78,8 @@ static P_AOL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.aol.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.aol.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -109,9 +107,8 @@ static P_ARCOR_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -138,9 +135,8 @@ static P_AUTISTICI_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -167,9 +163,8 @@ static P_BLINDZELN_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -196,9 +191,8 @@ static P_BLUEWIN_CH: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -225,9 +219,8 @@ static P_BUZON_UY: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -254,9 +247,8 @@ static P_CHELLO_AT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -268,9 +260,8 @@ static P_COMCAST: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/comcast", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -282,9 +273,8 @@ static P_DISMAIL_DE: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/dismail-de", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -311,9 +301,8 @@ static P_DISROOT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -340,9 +329,8 @@ static P_E_EMAIL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -354,9 +342,8 @@ static P_ESPIV_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/espiv-net", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -372,9 +359,8 @@ static P_EXAMPLE_COM: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.example.com", port: 1337, username_pattern: Email }, Server { protocol: Smtp, socket: Starttls, hostname: "smtp.example.com", port: 1337, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -403,9 +389,8 @@ static P_FASTMAIL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -419,9 +404,8 @@ static P_FIREMAIL_DE: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/firemail-de", server: vec![ ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -434,6 +418,7 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/five-chat", server: vec![], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::BccSelf, @@ -448,8 +433,6 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -465,9 +448,8 @@ static P_FREENET_DE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "mx.freenet.de", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Starttls, hostname: "mx.freenet.de", port: 587, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -484,9 +466,11 @@ static P_GMAIL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.gmail.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.gmail.com", port: 465, username_pattern: Email }, ], + opt: ProviderOptions { + delete_to_trash: true, + ..Default::default() + }, config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Gmail), } }); @@ -521,9 +505,8 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -535,6 +518,10 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/hermes-radio", server: vec![], + opt: ProviderOptions { + strict_tls: false, + ..Default::default() + }, config_defaults: Some(vec![ ConfigDefault { key: Config::MdnsEnabled, @@ -549,8 +536,6 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { value: "2", }, ]), - strict_tls: false, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -564,9 +549,8 @@ static P_HEY_COM: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/hey-com", server: vec![ ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -579,9 +563,8 @@ static P_I_UA: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/i-ua", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -593,9 +576,8 @@ static P_I3_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/i3-net", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -622,9 +604,8 @@ static P_ICLOUD: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -651,9 +632,11 @@ static P_INFOMANIAK_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: ProviderOptions { + max_smtp_rcpt_to: Some(10), + ..Default::default() + }, config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: Some(10), oauth2_authorizer: None, }); @@ -665,9 +648,8 @@ static P_KOLST_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/kolst-com", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -679,9 +661,8 @@ static P_KONTENT_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/kontent-com", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -708,9 +689,8 @@ static P_MAIL_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -726,9 +706,8 @@ static P_MAIL_RU: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.ru", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -756,9 +735,8 @@ static P_MAIL2TOR: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -785,9 +763,8 @@ static P_MAILBOX_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -814,9 +791,8 @@ static P_MAILO_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -843,6 +819,11 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: ProviderOptions { + max_smtp_rcpt_to: Some(20), + strict_tls: false, + ..Default::default() + }, config_defaults: Some(vec![ ConfigDefault { key: Config::DeleteServerAfter, @@ -869,8 +850,6 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: false, - max_smtp_rcpt_to: Some(20), oauth2_authorizer: None, }); @@ -897,9 +876,8 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -926,9 +904,8 @@ static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -955,9 +932,8 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -984,9 +960,8 @@ static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -998,6 +973,13 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/posteo", server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "posteo.de", + port: 993, + username_pattern: Email, + }, Server { protocol: Imap, socket: Starttls, @@ -1005,6 +987,13 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { port: 143, username_pattern: Email, }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "posteo.de", + port: 465, + username_pattern: Email, + }, Server { protocol: Smtp, socket: Starttls, @@ -1013,9 +1002,8 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1029,9 +1017,8 @@ static P_PROTONMAIL: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/protonmail", server: vec![ ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1048,9 +1035,8 @@ static P_QQ: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1078,9 +1064,8 @@ static P_RISEUP_NET: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1092,9 +1077,21 @@ static P_ROGERS_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/rogers-com", server: vec![], + opt: Default::default(), + config_defaults: None, + oauth2_authorizer: None, +}); + +// sonic.md: sonic.net +static P_SONIC: Lazy = Lazy::new(|| Provider { + id: "sonic", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/sonic", + server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1121,9 +1118,8 @@ static P_SYSTEMAUSFALL_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1150,9 +1146,8 @@ static P_SYSTEMLI_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1168,9 +1163,8 @@ static P_T_ONLINE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "secureimap.t-online.de", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "securesmtp.t-online.de", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1205,6 +1199,7 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::BccSelf, @@ -1219,8 +1214,6 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1247,9 +1240,8 @@ static P_TISCALI_IT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1263,9 +1255,8 @@ static P_TUTANOTA: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/tutanota", server: vec![ ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1278,9 +1269,8 @@ static P_UKR_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/ukr-net", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1307,9 +1297,8 @@ static P_UNDERNET_UY: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1321,9 +1310,8 @@ static P_VFEMAIL: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/vfemail", server: vec![], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1350,9 +1338,8 @@ static P_VIVALDI: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1379,9 +1366,8 @@ static P_VODAFONE_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1398,9 +1384,8 @@ static P_WEB_DE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Starttls, hostname: "imap.web.de", port: 143, username_pattern: Emaillocalpart }, Server { protocol: Smtp, socket: Starttls, hostname: "smtp.web.de", port: 587, username_pattern: Emaillocalpart }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1417,9 +1402,8 @@ static P_YAHOO: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1447,9 +1431,8 @@ static P_YANDEX_RU: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Yandex), }); @@ -1465,11 +1448,10 @@ static P_YGGMAIL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email }, Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email }, ], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::MvboxMove, value: "0" }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, } }); @@ -1497,9 +1479,8 @@ static P_ZIGGO_NL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1526,9 +1507,8 @@ static P_ZOHO: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, }); @@ -1822,6 +1802,7 @@ pub(crate) static PROVIDER_DATA: Lazy> ("foxmail.com", &*P_QQ), ("riseup.net", &*P_RISEUP_NET), ("rogers.com", &*P_ROGERS_COM), + ("sonic.net", &*P_SONIC), ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG), ("solidaris.me", &*P_SYSTEMAUSFALL_ORG), ("systemli.org", &*P_SYSTEMLI_ORG), @@ -1946,6 +1927,7 @@ pub(crate) static PROVIDER_IDS: Lazy> = ("qq", &*P_QQ), ("riseup.net", &*P_RISEUP_NET), ("rogers.com", &*P_ROGERS_COM), + ("sonic", &*P_SONIC), ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG), ("systemli.org", &*P_SYSTEMLI_ORG), ("t-online", &*P_T_ONLINE), @@ -1970,4 +1952,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 1, 6).unwrap()); + Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 21).unwrap()); diff --git a/src/provider/update.py b/src/provider/update.py index b67e9fb59..304b80e74 100755 --- a/src/provider/update.py +++ b/src/provider/update.py @@ -40,6 +40,23 @@ def file2url(f): return "https://providers.delta.chat/" + f +def process_opt(data): + if not "opt" in data: + return "Default::default()" + opt = "ProviderOptions {\n" + opt_data = data.get("opt", "") + for key in opt_data: + value = str(opt_data[key]) + if key == "max_smtp_rcpt_to": + value = "Some(" + value + ")" + if value in {"True", "False"}: + value = value.lower() + opt += " " + key + ": " + value + ",\n" + opt += " ..Default::default()\n" + opt += " }" + return opt + + def process_config_defaults(data): if not "config_defaults" in data: return "None" @@ -106,14 +123,9 @@ def process_data(data, file): server += (" Server { protocol: " + protocol.capitalize() + ", socket: " + socket.capitalize() + ", hostname: \"" + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern.capitalize() + " },\n") + opt = process_opt(data) config_defaults = process_config_defaults(data) - strict_tls = data.get("strict_tls", True) - strict_tls = "true" if strict_tls else "false" - - max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0) - max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None" - oauth2 = data.get("oauth2", "") oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None" @@ -128,9 +140,8 @@ def process_data(data, file): provider += " after_login_hint: \"" + after_login_hint + "\",\n" provider += " overview_page: \"" + file2url(file) + "\",\n" provider += " server: vec![\n" + server + " ],\n" + provider += " opt: " + opt + ",\n" provider += " config_defaults: " + config_defaults + ",\n" - provider += " strict_tls: " + strict_tls + ",\n" - provider += " max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n" provider += " oauth2_authorizer: " + oauth2 + ",\n" provider += "});\n\n" else: @@ -174,7 +185,9 @@ if __name__ == "__main__": "use crate::provider::Protocol::*;\n" "use crate::provider::Socket::*;\n" "use crate::provider::UsernamePattern::*;\n" - "use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};\n" + "use crate::provider::{\n" + " Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,\n" + "};\n" "use std::collections::HashMap;\n\n" "use once_cell::sync::Lazy;\n\n") diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 95db04d9c..2540d7442 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -342,11 +342,12 @@ pub(crate) async fn receive_imf_inner( if received_msg.needs_delete_job || (delete_server_after == Some(0) && is_partial_download.is_none()) { + let target = context.get_delete_msgs_target().await?; context .sql .execute( - "UPDATE imap SET target='' WHERE rfc724_mid=?", - paramsv![rfc724_mid], + "UPDATE imap SET target=? WHERE rfc724_mid=?", + paramsv![target, rfc724_mid], ) .await?; } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() { @@ -1084,8 +1085,6 @@ async fn add_parts( let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len()); - let conn = context.sql.get_conn().await?; - for part in &mime_parser.parts { if part.is_reaction { set_msg_reaction( @@ -1117,39 +1116,6 @@ async fn add_parts( } let mut txt_raw = "".to_string(); - let mut stmt = conn.prepare_cached( - r#" -INSERT INTO msgs - ( - id, - rfc724_mid, chat_id, - from_id, to_id, timestamp, timestamp_sent, - timestamp_rcvd, type, state, msgrmsg, - txt, subject, txt_raw, param, - bytes, mime_headers, mime_in_reply_to, - mime_references, mime_modified, error, ephemeral_timer, - ephemeral_timestamp, download_state, hop_info - ) - VALUES ( - ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ? - ) -ON CONFLICT (id) DO UPDATE -SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, - from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent, - timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg, - txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param, - bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, - mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, - ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info -"#, - )?; - let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg { (better_msg, Viewtype::Text) } else { @@ -1183,7 +1149,38 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && msg.is_empty()); - stmt.execute(paramsv![ + let row_id = context.sql.insert( + r#" +INSERT INTO msgs + ( + id, + rfc724_mid, chat_id, + from_id, to_id, timestamp, timestamp_sent, + timestamp_rcvd, type, state, msgrmsg, + txt, subject, txt_raw, param, + bytes, mime_headers, mime_in_reply_to, + mime_references, mime_modified, error, ephemeral_timer, + ephemeral_timestamp, download_state, hop_info + ) + VALUES ( + ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ? + ) +ON CONFLICT (id) DO UPDATE +SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, + from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent, + timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg, + txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param, + bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, + mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, + ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info +"#, + paramsv![ replace_msg_id, rfc724_mid, if trash { DC_CHAT_ID_TRASH } else { chat_id }, @@ -1222,17 +1219,14 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, DownloadState::Done }, mime_parser.hop_info - ])?; + ]).await?; // We only replace placeholder with a first part, // afterwards insert additional parts. replace_msg_id = None; - let row_id = conn.last_insert_rowid(); - drop(stmt); created_db_entries.push(MsgId::new(u32::try_from(row_id)?)); } - drop(conn); // check all parts whether they contain a new logging webxdc for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) { diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index b2564fc75..50aa285ce 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -5,7 +5,7 @@ use crate::aheader::EncryptPreference; use crate::chat::get_chat_contacts; use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility}; use crate::chatlist::Chatlist; -use crate::constants::{ShowEmails, DC_GCL_NO_SPECIALS}; +use crate::constants::DC_GCL_NO_SPECIALS; use crate::imap::prefetch_should_download; use crate::message::Message; use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; @@ -647,15 +647,11 @@ async fn test_parse_ndn( // Check that the ndn would be downloaded: let headers = mailparse::parse_mail(raw_ndn).unwrap().headers; - assert!(prefetch_should_download( - &t, - &headers, - "some-other-message-id", - std::iter::empty(), - ShowEmails::Off, - ) - .await - .unwrap()); + assert!( + prefetch_should_download(&t, &headers, "some-other-message-id", std::iter::empty(),) + .await + .unwrap() + ); receive_imf(&t, raw_ndn, false).await.unwrap(); let msg = Message::load_from_db(&t, msg_id).await.unwrap(); diff --git a/src/scheduler.rs b/src/scheduler.rs index 356c40e07..df9312c4e 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,6 +1,8 @@ +use std::iter::{self, once}; + use anyhow::{bail, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; -use futures::try_join; +use futures::future::try_join_all; use futures_lite::FutureExt; use tokio::task; @@ -9,7 +11,7 @@ use crate::config::Config; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; use crate::ephemeral::{self, delete_expired_imap_messages}; -use crate::imap::Imap; +use crate::imap::{FolderMeaning, Imap}; use crate::job; use crate::location; use crate::log::LogExt; @@ -20,15 +22,19 @@ use crate::tools::{duration_to_str, maybe_add_time_based_warnings}; pub(crate) mod connectivity; +#[derive(Debug)] +struct SchedBox { + meaning: FolderMeaning, + conn_state: ImapConnectionState, + handle: task::JoinHandle<()>, +} + /// Job and connection scheduler. #[derive(Debug)] pub(crate) struct Scheduler { - inbox: ImapConnectionState, - inbox_handle: task::JoinHandle<()>, - mvbox: ImapConnectionState, - mvbox_handle: Option>, - sentbox: ImapConnectionState, - sentbox_handle: Option>, + inbox: SchedBox, + /// Optional boxes -- mvbox, sentbox. + oboxes: Vec, smtp: SmtpConnectionState, smtp_handle: task::JoinHandle<()>, ephemeral_handle: task::JoinHandle<()>, @@ -161,7 +167,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne } } - info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await; + info = fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await; } } } @@ -182,7 +188,20 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne /// handling all the errors. In case of an error, it is logged, but not propagated upwards. If /// critical operation fails such as fetching new messages fails, connection is reset via /// `trigger_reconnect`, so a fresh one can be opened. -async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) -> InterruptInfo { +async fn fetch_idle( + ctx: &Context, + connection: &mut Imap, + folder_meaning: FolderMeaning, +) -> InterruptInfo { + let folder_config = match folder_meaning.to_config() { + Some(c) => c, + None => { + error!(ctx, "Bad folder meaning: {}", folder_meaning); + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; + } + }; let folder = match ctx.get_config(folder_config).await { Ok(folder) => folder, Err(err) => { @@ -190,7 +209,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) ctx, "Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err ); - return connection.fake_idle(ctx, None).await; + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; } }; @@ -199,7 +220,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) } else { connection.connectivity.set_not_configured(ctx).await; info!(ctx, "Can not watch {} folder, not set", folder_config); - return connection.fake_idle(ctx, None).await; + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; }; // connect and fake idle if unable to connect @@ -210,7 +233,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) { warn!(ctx, "{:#}", err); connection.trigger_reconnect(ctx); - return connection.fake_idle(ctx, Some(watch_folder)).await; + return connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await; } if folder_config == Config::ConfiguredInboxFolder { @@ -227,7 +252,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) // Fetch the watched folder. if let Err(err) = connection - .fetch_move_delete(ctx, &watch_folder, false) + .fetch_move_delete(ctx, &watch_folder, folder_meaning) .await .context("fetch_move_delete") { @@ -265,7 +290,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) // no new messages. We want to select the watched folder anyway before going IDLE // there, so this does not take additional protocol round-trip. if let Err(err) = connection - .fetch_move_delete(ctx, &watch_folder, false) + .fetch_move_delete(ctx, &watch_folder, folder_meaning) .await .context("fetch_move_delete after scan_folders") { @@ -293,7 +318,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) ctx, "IMAP session does not support IDLE, going to fake idle." ); - return connection.fake_idle(ctx, Some(watch_folder)).await; + return connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await; } info!(ctx, "IMAP session supports IDLE, using it."); @@ -318,7 +345,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) } } else { warn!(ctx, "No IMAP session, going to fake idle."); - connection.fake_idle(ctx, Some(watch_folder)).await + connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await } } @@ -326,11 +355,11 @@ async fn simple_imap_loop( ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers, - folder_config: Config, + folder_meaning: FolderMeaning, ) { use futures::future::FutureExt; - info!(ctx, "starting simple loop for {}", folder_config); + info!(ctx, "starting simple loop for {}", folder_meaning); let ImapConnectionHandlers { mut connection, stop_receiver, @@ -346,7 +375,7 @@ async fn simple_imap_loop( } loop { - fetch_idle(&ctx, &mut connection, folder_config).await; + fetch_idle(&ctx, &mut connection, folder_meaning).await; } }; @@ -443,75 +472,56 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect impl Scheduler { /// Start the scheduler. pub async fn start(ctx: Context) -> Result { - let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?; - let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?; let (smtp, smtp_handlers) = SmtpConnectionState::new(); - let (inbox, inbox_handlers) = ImapConnectionState::new(&ctx).await?; - let (inbox_start_send, inbox_start_recv) = channel::bounded(1); - let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1); - let mut mvbox_handle = None; - let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1); - let mut sentbox_handle = None; let (smtp_start_send, smtp_start_recv) = channel::bounded(1); let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1); let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1); - let inbox_handle = { + let mut oboxes = Vec::new(); + let mut start_recvs = Vec::new(); + + let (conn_state, inbox_handlers) = ImapConnectionState::new(&ctx).await?; + let (inbox_start_send, inbox_start_recv) = channel::bounded(1); + let handle = { let ctx = ctx.clone(); task::spawn(async move { inbox_loop(ctx, inbox_start_send, inbox_handlers).await }) }; + let inbox = SchedBox { + meaning: FolderMeaning::Inbox, + conn_state, + handle, + }; + start_recvs.push(inbox_start_recv); - if ctx.should_watch_mvbox().await? { - let ctx = ctx.clone(); - mvbox_handle = Some(task::spawn(async move { - simple_imap_loop( - ctx, - mvbox_start_send, - mvbox_handlers, - Config::ConfiguredMvboxFolder, - ) - .await - })); - } else { - mvbox_start_send - .send(()) - .await - .context("mvbox start send, missing receiver")?; - mvbox_handlers - .connection - .connectivity - .set_not_configured(&ctx) - .await - } - - if ctx.get_config_bool(Config::SentboxWatch).await? { - let ctx = ctx.clone(); - sentbox_handle = Some(task::spawn(async move { - simple_imap_loop( - ctx, - sentbox_start_send, - sentbox_handlers, - Config::ConfiguredSentboxFolder, - ) - .await - })); - } else { - sentbox_start_send - .send(()) - .await - .context("sentbox start send, missing receiver")?; - sentbox_handlers - .connection - .connectivity - .set_not_configured(&ctx) - .await + for (meaning, should_watch) in [ + (FolderMeaning::Mvbox, ctx.should_watch_mvbox().await), + ( + FolderMeaning::Sent, + ctx.get_config_bool(Config::SentboxWatch).await, + ), + ] { + if should_watch? { + let (conn_state, handlers) = ImapConnectionState::new(&ctx).await?; + let (start_send, start_recv) = channel::bounded(1); + let ctx = ctx.clone(); + let handle = task::spawn(async move { + simple_imap_loop(ctx, start_send, handlers, meaning).await + }); + oboxes.push(SchedBox { + meaning, + conn_state, + handle, + }); + start_recvs.push(start_recv); + } } let smtp_handle = { let ctx = ctx.clone(); task::spawn(async move { smtp_loop(ctx, smtp_start_send, smtp_handlers).await }) }; + start_recvs.push(smtp_start_recv); let ephemeral_handle = { let ctx = ctx.clone(); @@ -531,12 +541,8 @@ impl Scheduler { let res = Self { inbox, - mvbox, - sentbox, + oboxes, smtp, - inbox_handle, - mvbox_handle, - sentbox_handle, smtp_handle, ephemeral_handle, ephemeral_interrupt_send, @@ -546,12 +552,7 @@ impl Scheduler { }; // wait for all loops to be started - if let Err(err) = try_join!( - inbox_start_recv.recv(), - mvbox_start_recv.recv(), - sentbox_start_recv.recv(), - smtp_start_recv.recv() - ) { + if let Err(err) = try_join_all(start_recvs.iter().map(|r| r.recv())).await { bail!("failed to start scheduler: {}", err); } @@ -559,30 +560,26 @@ impl Scheduler { Ok(res) } + fn boxes(&self) -> iter::Chain, std::slice::Iter<'_, SchedBox>> { + once(&self.inbox).chain(self.oboxes.iter()) + } + fn maybe_network(&self) { - self.interrupt_inbox(InterruptInfo::new(true)); - self.interrupt_mvbox(InterruptInfo::new(true)); - self.interrupt_sentbox(InterruptInfo::new(true)); + for b in self.boxes() { + b.conn_state.interrupt(InterruptInfo::new(true)); + } self.interrupt_smtp(InterruptInfo::new(true)); } fn maybe_network_lost(&self) { - self.interrupt_inbox(InterruptInfo::new(false)); - self.interrupt_mvbox(InterruptInfo::new(false)); - self.interrupt_sentbox(InterruptInfo::new(false)); + for b in self.boxes() { + b.conn_state.interrupt(InterruptInfo::new(false)); + } self.interrupt_smtp(InterruptInfo::new(false)); } fn interrupt_inbox(&self, info: InterruptInfo) { - self.inbox.interrupt(info); - } - - fn interrupt_mvbox(&self, info: InterruptInfo) { - self.mvbox.interrupt(info); - } - - fn interrupt_sentbox(&self, info: InterruptInfo) { - self.sentbox.interrupt(info); + self.inbox.conn_state.interrupt(info); } fn interrupt_smtp(&self, info: InterruptInfo) { @@ -605,29 +602,17 @@ impl Scheduler { /// /// It consumes the scheduler and never fails to stop it. In the worst case, long-running tasks /// are forcefully terminated if they cannot shutdown within the timeout. - pub(crate) async fn stop(mut self, context: &Context) { + pub(crate) async fn stop(self, context: &Context) { // Send stop signals to tasks so they can shutdown cleanly. - self.inbox.stop().await.ok_or_log(context); - if self.mvbox_handle.is_some() { - self.mvbox.stop().await.ok_or_log(context); - } - if self.sentbox_handle.is_some() { - self.sentbox.stop().await.ok_or_log(context); + for b in self.boxes() { + b.conn_state.stop().await.ok_or_log(context); } self.smtp.stop().await.ok_or_log(context); // Actually shutdown tasks. let timeout_duration = std::time::Duration::from_secs(30); - tokio::time::timeout(timeout_duration, self.inbox_handle) - .await - .ok_or_log(context); - if let Some(mvbox_handle) = self.mvbox_handle.take() { - tokio::time::timeout(timeout_duration, mvbox_handle) - .await - .ok_or_log(context); - } - if let Some(sentbox_handle) = self.sentbox_handle.take() { - tokio::time::timeout(timeout_duration, sentbox_handle) + for b in once(self.inbox).chain(self.oboxes.into_iter()) { + tokio::time::timeout(timeout_duration, b.handle) .await .ok_or_log(context); } diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index 6ddc9eede..6ede2074f 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -1,18 +1,18 @@ use core::fmt; -use std::{ops::Deref, sync::Arc}; +use std::{iter::once, ops::Deref, sync::Arc}; use anyhow::{anyhow, Result}; use humansize::{format_size, BINARY}; use tokio::sync::{Mutex, RwLockReadGuard}; use crate::events::EventType; -use crate::imap::scan_folders::get_watched_folder_configs; +use crate::imap::{scan_folders::get_watched_folder_configs, FolderMeaning}; use crate::quota::{ QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE, }; use crate::tools::time; -use crate::{config::Config, scheduler::Scheduler, stock_str, tools}; use crate::{context::Context, log::LogExt}; +use crate::{scheduler::Scheduler, stock_str, tools}; #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)] pub enum Connectivity { @@ -157,17 +157,14 @@ impl ConnectivityStore { /// Called during `dc_maybe_network()` to make sure that `dc_accounts_all_work_done()` /// returns false immediately after `dc_maybe_network()`. pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option>) { - let [inbox, mvbox, sentbox] = match &*scheduler { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [ - inbox.state.connectivity.clone(), - mvbox.state.connectivity.clone(), - sentbox.state.connectivity.clone(), - ], + let (inbox, oboxes) = match &*scheduler { + Some(Scheduler { inbox, oboxes, .. }) => ( + inbox.conn_state.state.connectivity.clone(), + oboxes + .iter() + .map(|b| b.conn_state.state.connectivity.clone()) + .collect::>(), + ), None => return, }; drop(scheduler); @@ -185,7 +182,7 @@ pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option>, ) { - let stores = match &*scheduler { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [ - inbox.state.connectivity.clone(), - mvbox.state.connectivity.clone(), - sentbox.state.connectivity.clone(), - ], + let stores: Vec<_> = match &*scheduler { + Some(sched) => sched + .boxes() + .map(|b| b.conn_state.state.connectivity.clone()) + .collect(), None => return, }; drop(scheduler); @@ -260,14 +251,9 @@ impl Context { pub async fn get_connectivity(&self) -> Connectivity { let lock = self.scheduler.read().await; let stores: Vec<_> = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [&inbox.state, &mvbox.state, &sentbox.state] - .iter() - .map(|state| state.connectivity.clone()) + Some(sched) => sched + .boxes() + .map(|b| b.conn_state.state.connectivity.clone()) .collect(), None => return Connectivity::NotConnected, }; @@ -348,28 +334,12 @@ impl Context { let lock = self.scheduler.read().await; let (folders_states, smtp) = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - smtp, - .. - }) => ( - [ - ( - Config::ConfiguredInboxFolder, - inbox.state.connectivity.clone(), - ), - ( - Config::ConfiguredMvboxFolder, - mvbox.state.connectivity.clone(), - ), - ( - Config::ConfiguredSentboxFolder, - sentbox.state.connectivity.clone(), - ), - ], - smtp.state.connectivity.clone(), + Some(sched) => ( + sched + .boxes() + .map(|b| (b.meaning, b.conn_state.state.connectivity.clone())) + .collect::>(), + sched.smtp.state.connectivity.clone(), ), None => { return Err(anyhow!("Not started")); @@ -390,8 +360,8 @@ impl Context { for (folder, state) in &folders_states { let mut folder_added = false; - if watched_folders.contains(folder) { - let f = self.get_config(*folder).await.ok_or_log(self).flatten(); + if let Some(config) = folder.to_config().filter(|c| watched_folders.contains(c)) { + let f = self.get_config(config).await.ok_or_log(self).flatten(); if let Some(foldername) = f { let detailed = &state.get_detailed().await; @@ -407,7 +377,7 @@ impl Context { } } - if !folder_added && folder == &Config::ConfiguredInboxFolder { + if !folder_added && folder == &FolderMeaning::Inbox { let detailed = &state.get_detailed().await; if let DetailedConnectivity::Error(_) = detailed { // On the inbox thread, we also do some other things like scan_folders and run jobs @@ -535,14 +505,10 @@ impl Context { pub async fn all_work_done(&self) -> bool { let lock = self.scheduler.read().await; let stores: Vec<_> = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - smtp, - .. - }) => [&inbox.state, &mvbox.state, &sentbox.state, &smtp.state] - .iter() + Some(sched) => sched + .boxes() + .map(|b| &b.conn_state.state) + .chain(once(&sched.smtp.state)) .map(|state| state.connectivity.clone()) .collect(), None => return false, diff --git a/src/smtp.rs b/src/smtp.rs index 5cf0dd73f..cda3aff05 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -101,8 +101,9 @@ impl Smtp { &lp.smtp, &lp.socks5_config, &lp.addr, - lp.provider - .map_or(lp.socks5_config.is_some(), |provider| provider.strict_tls), + lp.provider.map_or(lp.socks5_config.is_some(), |provider| { + provider.opt.strict_tls + }), ) .await } diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 3cd5c0567..938deb5fc 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -47,7 +47,7 @@ impl Smtp { let chunk_size = context .get_configured_provider() .await? - .and_then(|provider| provider.max_smtp_rcpt_to) + .and_then(|provider| provider.opt.max_smtp_rcpt_to) .map_or(DEFAULT_MAX_SMTP_RCPT_TO, usize::from); for recipients_chunk in recipients.chunks(chunk_size) { diff --git a/src/sql.rs b/src/sql.rs index 4b4b190eb..163d4e989 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -2,8 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{bail, Context as _, Result}; use rusqlite::{self, config::DbConfig, Connection, OpenFlags, TransactionBehavior}; @@ -49,7 +48,7 @@ pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator Result<()> { let path_str = path .to_str() - .with_context(|| format!("path {path:?} is not valid unicode"))?; - let conn = self.get_conn().await?; + .with_context(|| format!("path {path:?} is not valid unicode"))? + .to_string(); + let res = self + .call(move |conn| { + // Check that backup passphrase is correct before resetting our database. + conn.execute( + "ATTACH DATABASE ? AS backup KEY ?", + paramsv![path_str, passphrase], + ) + .context("failed to attach backup database")?; + if let Err(err) = conn + .query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(())) + .context("backup passphrase is not correct") + { + conn.execute("DETACH DATABASE backup", []) + .context("failed to detach backup database")?; + return Err(err); + } - tokio::task::block_in_place(move || { - // Check that backup passphrase is correct before resetting our database. - conn.execute( - "ATTACH DATABASE ? AS backup KEY ?", - paramsv![path_str, passphrase], - ) - .context("failed to attach backup database")?; - if let Err(err) = conn - .query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(())) - .context("backup passphrase is not correct") - { + // Reset the database without reopening it. We don't want to reopen the database because we + // don't have main database passphrase at this point. + // See for documentation. + // Without resetting import may fail due to existing tables. + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true) + .context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?; + conn.execute("VACUUM", []) + .context("failed to vacuum the database")?; + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false) + .context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?; + let res = conn + .query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| { + Ok(()) + }) + .context("failed to import from attached backup database"); conn.execute("DETACH DATABASE backup", []) .context("failed to detach backup database")?; - return Err(err); - } + res?; + Ok(()) + }) + .await; - // Reset the database without reopening it. We don't want to reopen the database because we - // don't have main database passphrase at this point. - // See for documentation. - // Without resetting import may fail due to existing tables. - conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true) - .context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?; - conn.execute("VACUUM", []) - .context("failed to vacuum the database")?; - conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false) - .context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?; - let res = conn - .query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| { - Ok(()) - }) - .context("failed to import from attached backup database"); - conn.execute("DETACH DATABASE backup", []) - .context("failed to detach backup database")?; - res?; - Ok(()) - }) + // The config cache is wrong now that we have a different database + self.config_cache.write().await.clear(); + + res } /// Creates a new connection pool. @@ -294,22 +299,41 @@ impl Sql { } } + /// Allocates a connection and calls given function with the connection. + /// + /// Returns the result of the function. + pub async fn call<'a, F, R>(&'a self, function: F) -> Result + where + F: 'a + FnOnce(&mut Connection) -> Result + Send, + R: Send + 'static, + { + let lock = self.pool.read().await; + let pool = lock.as_ref().context("no SQL connection")?; + let mut conn = pool.get().await?; + let res = tokio::task::block_in_place(move || function(&mut conn))?; + Ok(res) + } + /// Execute the given query, returning the number of affected rows. - pub async fn execute(&self, query: &str, params: impl rusqlite::Params) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + pub async fn execute( + &self, + query: &str, + params: impl rusqlite::Params + Send, + ) -> Result { + self.call(move |conn| { let res = conn.execute(query, params)?; Ok(res) }) + .await } /// Executes the given query, returning the last inserted row ID. - pub async fn insert(&self, query: &str, params: impl rusqlite::Params) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + pub async fn insert(&self, query: &str, params: impl rusqlite::Params + Send) -> Result { + self.call(move |conn| { conn.execute(query, params)?; Ok(conn.last_insert_rowid()) }) + .await } /// Prepares and executes the statement and maps a function over the resulting rows. @@ -318,40 +342,32 @@ impl Sql { pub async fn query_map( &self, sql: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, mut g: G, ) -> Result where - F: FnMut(&rusqlite::Row) -> rusqlite::Result, - G: FnMut(rusqlite::MappedRows) -> Result, + F: Send + FnMut(&rusqlite::Row) -> rusqlite::Result, + G: Send + FnMut(rusqlite::MappedRows) -> Result, + H: Send + 'static, { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut stmt = conn.prepare(sql)?; let res = stmt.query_map(params, f)?; g(res) }) - } - - /// Allocates a connection from the connection pool and returns it. - pub(crate) async fn get_conn(&self) -> Result { - let lock = self.pool.read().await; - let pool = lock.as_ref().context("no SQL connection")?; - let conn = pool.get().await?; - - Ok(conn) + .await } /// Used for executing `SELECT COUNT` statements only. Returns the resulting count. - pub async fn count(&self, query: &str, params: impl rusqlite::Params) -> Result { + pub async fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result { let count: isize = self.query_row(query, params, |row| row.get(0)).await?; Ok(usize::try_from(count)?) } /// Used for executing `SELECT COUNT` statements only. Returns `true`, if the count is at least /// one, `false` otherwise. - pub async fn exists(&self, sql: &str, params: impl rusqlite::Params) -> Result { + pub async fn exists(&self, sql: &str, params: impl rusqlite::Params + Send) -> Result { let count = self.count(sql, params).await?; Ok(count > 0) } @@ -360,17 +376,18 @@ impl Sql { pub async fn query_row( &self, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, ) -> Result where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: FnOnce(&rusqlite::Row) -> rusqlite::Result + Send, + T: Send + 'static, { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let res = conn.query_row(query, params, f)?; Ok(res) }) + .await } /// Execute the function inside a transaction. @@ -388,8 +405,7 @@ impl Sql { H: Send + 'static, G: Send + FnOnce(&mut rusqlite::Transaction<'_>) -> Result, { - let mut conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let ret = callback(&mut transaction); @@ -404,12 +420,12 @@ impl Sql { } } }) + .await } /// Query the database if the requested table already exists. pub async fn table_exists(&self, name: &str) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut exists = false; conn.pragma(None, "table_info", name.to_string(), |_row| { // will only be executed if the info was found @@ -419,12 +435,12 @@ impl Sql { Ok(exists) }) + .await } /// Check if a column exists in a given table. pub async fn col_exists(&self, table_name: &str, col_name: &str) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut exists = false; // `PRAGMA table_info` returns one row per column, // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value @@ -438,29 +454,27 @@ impl Sql { Ok(exists) }) + .await } /// Execute a query which is expected to return zero or one row. pub async fn query_row_optional( &self, sql: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, ) -> Result> where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result, + T: Send + 'static, { - let conn = self.get_conn().await?; - let res = - tokio::task::block_in_place(move || match conn.query_row(sql.as_ref(), params, f) { - Ok(res) => Ok(Some(res)), - Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), - Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => { - Ok(None) - } - Err(err) => Err(err), - })?; - Ok(res) + self.call(move |conn| match conn.query_row(sql.as_ref(), params, f) { + Ok(res) => Ok(Some(res)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => Ok(None), + Err(err) => Err(err.into()), + }) + .await } /// Executes a query which is expected to return one row and one @@ -469,10 +483,10 @@ impl Sql { pub async fn query_get_value( &self, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, ) -> Result> where - T: rusqlite::types::FromSql, + T: rusqlite::types::FromSql + Send + 'static, { self.query_row_optional(query, params, |row| row.get::<_, T>(0)) .await @@ -935,11 +949,16 @@ mod tests { async fn test_auto_vacuum() -> Result<()> { let t = TestContext::new().await; - let conn = t.sql.get_conn().await?; - let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| { - let auto_vacuum: i32 = row.get(0)?; - Ok(auto_vacuum) - })?; + let auto_vacuum = t + .sql + .call(|conn| { + let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| { + let auto_vacuum: i32 = row.get(0)?; + Ok(auto_vacuum) + })?; + Ok(auto_vacuum) + }) + .await?; // auto_vacuum=2 is the same as auto_vacuum=INCREMENTAL assert_eq!(auto_vacuum, 2); diff --git a/src/sql/pool.rs b/src/sql/pool.rs index fc7bf05bf..b7459976a 100644 --- a/src/sql/pool.rs +++ b/src/sql/pool.rs @@ -1,10 +1,18 @@ -//! Connection pool. +//! # SQLite connection pool. +//! +//! The connection pool holds a number of SQLite connections and allows to allocate them. +//! When allocated connection is dropped, underlying connection is returned back to the pool. +//! +//! The pool is organized as a stack. It always allocates the most recently used connection. +//! Each SQLite connection has its own page cache, so allocating recently used connections +//! improves the performance compared to, for example, organizing the pool as a queue +//! and returning the least recently used connection each time. use std::ops::{Deref, DerefMut}; use std::sync::{Arc, Weak}; use anyhow::{Context, Result}; -use crossbeam_queue::ArrayQueue; +use parking_lot::Mutex; use rusqlite::Connection; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -12,7 +20,7 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore}; #[derive(Debug)] struct InnerPool { /// Available connections. - connections: ArrayQueue, + connections: Mutex>, /// Counts the number of available connections. semaphore: Arc, @@ -23,7 +31,9 @@ impl InnerPool { /// /// The connection could be new or returned back. fn put(&self, connection: Connection) { - self.connections.force_push(connection); + let mut connections = self.connections.lock(); + connections.push(connection); + drop(connections); } } @@ -74,22 +84,19 @@ pub struct Pool { impl Pool { /// Creates a new connection pool. pub fn new(connections: Vec) -> Self { + let semaphore = Arc::new(Semaphore::new(connections.len())); let inner = Arc::new(InnerPool { - connections: ArrayQueue::new(connections.len()), - semaphore: Arc::new(Semaphore::new(connections.len())), + connections: Mutex::new(connections), + semaphore, }); - for connection in connections { - inner.connections.force_push(connection); - } Pool { inner } } /// Retrieves a connection from the pool. pub async fn get(&self) -> Result { let permit = self.inner.semaphore.clone().acquire_owned().await?; - let conn = self - .inner - .connections + let mut connections = self.inner.connections.lock(); + let conn = connections .pop() .context("got a permit when there are no connections in the pool")?; let conn = PooledConnection {