fix: try executing migration stepwise

This requires currently too much memory, crashing on larger instances

.github/workflows/ci.yml

@@ -78,13 +78,13 @@ jobs:
             rust: 1.68.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.64.0
+          # Minimum Supported Rust Version = 1.63.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.64.0
+            rust: 1.63.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:

CHANGELOG.md

@@ -3,11 +3,10 @@
 ## Unreleased
 
 ### Changes
+- Drop unused SQL columns #4141
 - "full message view" not needed because of footers that go to contact status #4151
 - Pick up system's light/dark mode in generated message HTML #4150
 - Support non-persistent configuration with DELTACHAT_* env
-- Print deltachat-repl errors with causes. #4166
-- Increase MSRV to 1.64. #4167
 
 ### Fixes
 - Fix segmentation fault if `dc_context_unref()` is called during

Cargo.lock

@@ -177,12 +177,13 @@ dependencies = [
 
 [[package]]
 name = "async-smtp"
-version = "0.9.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8709c0d4432be428a88a06746689a9cb543e8e27ef7f61ca4d0455003a3d8c5b"
+checksum = "7384febcabdd07a498c9f4fbaa7e488ff4eb60d0ade14b47b09ec44b8f645301"
 dependencies = [
  "anyhow",
  "base64 0.13.1",
+ "bufstream",
  "futures",
  "hostname",
  "log",
@@ -400,6 +401,12 @@ dependencies = [
  "safemem",
 ]
 
+[[package]]
+name = "bufstream"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
+
 [[package]]
 name = "bumpalo"
 version = "3.12.0"

Cargo.toml

@@ -3,7 +3,7 @@ name = "deltachat"
 version = "1.111.0"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.64"
+rust-version = "1.63"
 
 [profile.dev]
 debug = 0
@@ -33,7 +33,7 @@ anyhow = "1"
 async-channel = "1.8.0"
 async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
-async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.8", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
 backtrace = "0.3"
 base64 = "0.21"

deltachat-repl/src/main.rs

@@ -359,7 +359,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                                 false
                             }
                             Err(err) => {
-                                println!("Error: {err:#}");
+                                println!("Error: {err}");
                                 true
                             }
                         }
@@ -374,7 +374,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                     break;
                 }
                 Err(err) => {
-                    println!("Error: {err:#}");
+                    println!("Error: {err}");
                     break;
                 }
             }

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.68.0
+RUST_VERSION=1.64.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/create-provider-data-rs.py

@@ -220,13 +220,13 @@ if __name__ == "__main__":
 
     process_dir(Path(sys.argv[1]))
 
-    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
+    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains
-    out_all += "]));\n\n"
+    out_all += "].iter().copied().collect());\n\n"
 
-    out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
+    out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_ids
-    out_all += "]));\n\n"
+    out_all += "].iter().copied().collect());\n\n"
 
     if len(sys.argv) < 3:
         now = datetime.datetime.utcnow()

src/imap/client.rs

@@ -6,7 +6,7 @@ use std::{
 use anyhow::{Context as _, Result};
 use async_imap::Client as ImapClient;
 use async_imap::Session as ImapSession;
-use tokio::io::BufStream;
+use tokio::io::BufWriter;
 
 use super::capabilities::Capabilities;
 use super::session::Session;
@@ -96,7 +96,7 @@ impl Client {
     ) -> Result<Self> {
         let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?;
         let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?;
-        let buffered_stream = BufStream::new(tls_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = ImapClient::new(session_stream);
 
@@ -110,7 +110,7 @@ impl Client {
 
     pub async fn connect_insecure(context: &Context, hostname: &str, port: u16) -> Result<Self> {
         let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, false).await?;
-        let buffered_stream = BufStream::new(tcp_stream);
+        let buffered_stream = BufWriter::new(tcp_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = ImapClient::new(session_stream);
         let _greeting = client
@@ -145,7 +145,7 @@ impl Client {
             .await
             .context("STARTTLS upgrade failed")?;
 
-        let buffered_stream = BufStream::new(tls_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = ImapClient::new(session_stream);
 
@@ -163,7 +163,7 @@ impl Client {
             .connect(context, domain, port, IMAP_TIMEOUT, strict_tls)
             .await?;
         let tls_stream = wrap_tls(strict_tls, domain, socks5_stream).await?;
-        let buffered_stream = BufStream::new(tls_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = ImapClient::new(session_stream);
         let _greeting = client
@@ -183,7 +183,7 @@ impl Client {
         let socks5_stream = socks5_config
             .connect(context, domain, port, IMAP_TIMEOUT, false)
             .await?;
-        let buffered_stream = BufStream::new(socks5_stream);
+        let buffered_stream = BufWriter::new(socks5_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = ImapClient::new(session_stream);
         let _greeting = client
@@ -220,7 +220,7 @@ impl Client {
         let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream)
             .await
             .context("STARTTLS upgrade failed")?;
-        let buffered_stream = BufStream::new(tls_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = ImapClient::new(session_stream);
 

src/net/session.rs

@@ -2,7 +2,7 @@ use async_native_tls::TlsStream;
 use fast_socks5::client::Socks5Stream;
 use std::pin::Pin;
 use std::time::Duration;
-use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, BufStream};
+use tokio::io::{AsyncRead, AsyncWrite, BufWriter};
 use tokio_io_timeout::TimeoutStream;
 
 pub(crate) trait SessionStream:
@@ -22,7 +22,7 @@ impl<T: SessionStream> SessionStream for TlsStream<T> {
         self.get_mut().set_read_timeout(timeout);
     }
 }
-impl<T: SessionStream> SessionStream for BufStream<T> {
+impl<T: SessionStream> SessionStream for BufWriter<T> {
     fn set_read_timeout(&mut self, timeout: Option<Duration>) {
         self.get_mut().set_read_timeout(timeout);
     }
@@ -39,8 +39,3 @@ impl<T: SessionStream> SessionStream for Socks5Stream<T> {
         self.get_socket_mut().set_read_timeout(timeout)
     }
 }
-
-/// Session stream with a read buffer.
-pub(crate) trait SessionBufStream: SessionStream + AsyncBufRead {}
-
-impl<T: SessionStream + AsyncBufRead> SessionBufStream for T {}

src/provider/data.rs

@@ -1513,7 +1513,7 @@ static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
 });
 
 pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
-    HashMap::from([
+    [
         ("163.com", &*P_163),
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
@@ -1875,11 +1875,14 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("zohomail.eu", &*P_ZOHO),
         ("zohomail.com", &*P_ZOHO),
         ("zoho.com", &*P_ZOHO),
-    ])
+    ]
+    .iter()
+    .copied()
+    .collect()
 });
 
 pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
-    HashMap::from([
+    [
         ("163", &*P_163),
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol", &*P_AOL),
@@ -1942,7 +1945,10 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
         ("zoho", &*P_ZOHO),
-    ])
+    ]
+    .iter()
+    .copied()
+    .collect()
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =

src/smtp.rs

@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
 use anyhow::{bail, format_err, Context as _, Error, Result};
 use async_smtp::response::{Category, Code, Detail};
 use async_smtp::{self as smtp, EmailAddress, SmtpTransport};
-use tokio::io::BufStream;
+use tokio::io::BufWriter;
 use tokio::task;
 
 use crate::config::Config;
@@ -18,7 +18,7 @@ use crate::message::Message;
 use crate::message::{self, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::net::connect_tcp;
-use crate::net::session::SessionBufStream;
+use crate::net::session::SessionStream;
 use crate::net::tls::wrap_tls;
 use crate::oauth2::get_oauth2_access_token;
 use crate::provider::Socket;
@@ -32,7 +32,7 @@ const SMTP_TIMEOUT: Duration = Duration::from_secs(30);
 #[derive(Default)]
 pub(crate) struct Smtp {
     /// SMTP connection.
-    transport: Option<SmtpTransport<Box<dyn SessionBufStream>>>,
+    transport: Option<SmtpTransport<Box<dyn SessionStream>>>,
 
     /// Email address we are sending from.
     from: Option<EmailAddress>,
@@ -116,13 +116,13 @@ impl Smtp {
         port: u16,
         strict_tls: bool,
         socks5_config: Socks5Config,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let socks5_stream = socks5_config
             .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls)
             .await?;
         let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream).await?;
-        let buffered_stream = BufStream::new(tls_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true);
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)
@@ -135,20 +135,20 @@ impl Smtp {
         port: u16,
         strict_tls: bool,
         socks5_config: Socks5Config,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let socks5_stream = socks5_config
             .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls)
             .await?;
 
         // Run STARTTLS command and convert the client back into a stream.
         let client = smtp::SmtpClient::new().smtp_utf8(true);
-        let transport = SmtpTransport::new(client, BufStream::new(socks5_stream)).await?;
+        let transport = SmtpTransport::new(client, socks5_stream).await?;
         let tcp_stream = transport.starttls().await?;
         let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
             .await
             .context("STARTTLS upgrade failed")?;
-        let buffered_stream = BufStream::new(tls_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true).without_greeting();
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)
@@ -160,12 +160,12 @@ impl Smtp {
         hostname: &str,
         port: u16,
         socks5_config: Socks5Config,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let socks5_stream = socks5_config
             .connect(context, hostname, port, SMTP_TIMEOUT, false)
             .await?;
-        let buffered_stream = BufStream::new(socks5_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(socks5_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true);
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)
@@ -177,11 +177,11 @@ impl Smtp {
         hostname: &str,
         port: u16,
         strict_tls: bool,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?;
         let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?;
-        let buffered_stream = BufStream::new(tls_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true);
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)
@@ -193,18 +193,18 @@ impl Smtp {
         hostname: &str,
         port: u16,
         strict_tls: bool,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, strict_tls).await?;
 
         // Run STARTTLS command and convert the client back into a stream.
         let client = smtp::SmtpClient::new().smtp_utf8(true);
-        let transport = SmtpTransport::new(client, BufStream::new(tcp_stream)).await?;
+        let transport = SmtpTransport::new(client, tcp_stream).await?;
         let tcp_stream = transport.starttls().await?;
         let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
             .await
             .context("STARTTLS upgrade failed")?;
-        let buffered_stream = BufStream::new(tls_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true).without_greeting();
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)
@@ -215,10 +215,10 @@ impl Smtp {
         context: &Context,
         hostname: &str,
         port: u16,
-    ) -> Result<SmtpTransport<Box<dyn SessionBufStream>>> {
+    ) -> Result<SmtpTransport<Box<dyn SessionStream>>> {
         let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?;
-        let buffered_stream = BufStream::new(tcp_stream);
-        let session_stream: Box<dyn SessionBufStream> = Box::new(buffered_stream);
+        let buffered_stream = BufWriter::new(tcp_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let client = smtp::SmtpClient::new().smtp_utf8(true);
         let transport = SmtpTransport::new(client, session_stream).await?;
         Ok(transport)

src/sql/migrations.rs

@@ -691,18 +691,20 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         sql.set_db_version(98).await?;
     }
     if dbversion < 99 {
-        // sql.execute_migration(
-        //     "ALTER TABLE msgs DROP COLUMN server_folder;
-        //      ALTER TABLE msgs DROP COLUMN server_uid;
-        //      ALTER TABLE msgs DROP COLUMN move_state;
-        //      ALTER TABLE chats DROP COLUMN draft_timestamp;
-        //      ALTER TABLE chats DROP COLUMN draft_txt",
-        //     99,
-        // )
-        // .await?;
+        sql.execute_migration_parts(
+            [
+                "ALTER TABLE msgs DROP COLUMN server_folder;",
+                "ALTER TABLE msgs DROP COLUMN server_uid;",
+                "ALTER TABLE msgs DROP COLUMN move_state;",
+                "ALTER TABLE chats DROP COLUMN draft_timestamp;",
+                "ALTER TABLE chats DROP COLUMN draft_txt",
+            ],
+            99,
+        )
+        .await?;
 
         // Reverted above, as it requires to load the whole DB in memory.
-        sql.set_db_version(99).await?;
+        // sql.set_db_version(99).await?;
     }
 
     let new_version = sql
@@ -756,4 +758,37 @@ impl Sql {
 
         Ok(())
     }
+
+    async fn execute_migration_parts(
+        &self,
+        queries: impl IntoIterator<Item = &'static str>,
+        version: i32,
+    ) -> Result<()> {
+        for query in queries {
+            self.transaction(move |transaction| {
+                transaction.execute_batch(query)?;
+
+                Ok(())
+            })
+            .await
+            .with_context(|| format!("execute_migration failed for version {version}"))?;
+        }
+
+        self.transaction(move |transaction| {
+            // set raw config inside the transaction
+            transaction.execute(
+                "UPDATE config SET value=? WHERE keyname=?;",
+                paramsv![format!("{version}"), VERSION_CFG],
+            )?;
+            Ok(())
+        })
+        .await
+        .context("failed to set version")?;
+
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
+        drop(lock);
+
+        Ok(())
+    }
 }