Compare commits

...
11 Commits
Author SHA1 Message Date
holger krekel 55ada1cd89 some more logging 2019-11-11 10:16:02 +01:00
dignifiedquire 1077cf5e99 try use a different rust version 2019-11-10 21:22:57 +01:00
dignifiedquire ca698f9164 update deps 2019-11-10 21:22:57 +01:00
dignifiedquire 99201027e2 bust ci cache 2019-11-10 21:22:57 +01:00
dignifiedquire d607d35abc refactor: drop native-tls 2019-11-10 21:22:57 +01:00
dignifiedquire 3d790cbfca update docker image 2019-11-10 21:22:57 +01:00
dignifiedquire 68b2707d12 cleanup imap impl 2019-11-10 21:22:57 +01:00
dignifiedquire 6066821b50 update async-imap 2019-11-10 21:22:57 +01:00
dignifiedquire fe695c0f95 implement idle again 2019-11-10 21:22:57 +01:00
dignifiedquire 7bf13f3f89 remove local dependency 2019-11-10 21:22:57 +01:00
dignifiedquire 3b3992daed it compiles with async-imap 2019-11-10 21:22:57 +01:00
17 changed files with 1776 additions and 1222 deletions
+7 -5
View File
@@ -15,7 +15,7 @@ restore-workspace: &restore-workspace
restore-cache: &restore-cache
restore_cache:
keys:
- cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
- cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
- repo-source-{{ .Branch }}-{{ .Revision }}
commands:
@@ -44,7 +44,7 @@ jobs:
command: cargo generate-lockfile
- restore_cache:
keys:
- cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
- cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
- run: rustup install $(cat rust-toolchain)
- run: rustup default $(cat rust-toolchain)
- run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
@@ -60,7 +60,7 @@ jobs:
paths:
- crate
- save_cache:
key: cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
paths:
- "~/.cargo"
- "~/.rustup"
@@ -121,7 +121,7 @@ jobs:
steps:
- checkout
- run: bash ci_scripts/run-doxygen.sh
- run: mkdir -p workspace/c-docs
- run: mkdir -p workspace/c-docs
- run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
- persist_to_workspace:
root: workspace
@@ -138,6 +138,8 @@ jobs:
steps:
- *restore-workspace
- *restore-cache
- run: rustup install $(cat rust-toolchain)
- run: rustup default $(cat rust-toolchain)
- run:
name: build docs, run tests and build wheels
command: ci_scripts/run-python.sh
@@ -189,7 +191,7 @@ workflows:
- upload_docs_wheels:
requires:
- build_test_docs_wheel
- build_doxygen
- build_doxygen
- rustfmt:
requires:
- cargo_fetch
Generated
+732 -215
View File
File diff suppressed because it is too large Load Diff
+9 -5
View File
@@ -15,12 +15,13 @@ hex = "0.3.2"
sha2 = "0.8.0"
rand = "0.6.5"
smallvec = "0.6.9"
reqwest = "0.9.15"
reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
num-derive = "0.2.5"
num-traits = "0.2.6"
native-tls = "0.2.3"
lettre = { git = "https://github.com/deltachat/lettre", branch = "master" }
imap = { git = "https://github.com/deltachat/rust-imap", branch = "master" }
lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/rustls" }
async-imap = { git = "https://github.com/dignifiedquire/async-imap", branch = "master" }
async-tls = { git = "https://github.com/async-rs/async-tls", branch = "master" }
async-std = { git = "https://github.com/async-rs/async-std", branch = "master", features = ["unstable"] }
base64 = "0.10"
charset = "0.1"
percent-encoding = "2.0"
@@ -49,6 +50,9 @@ bitflags = "1.1.0"
jetscii = "0.4.4"
debug_stub_derive = "0.3.0"
sanitize-filename = "0.2.1"
stop-token = { version = "0.1.1", features = ["unstable"] }
rustls = "0.16.0"
webpki-roots = "0.18.0"
[dev-dependencies]
tempfile = "3.0"
@@ -74,6 +78,6 @@ path = "examples/repl/main.rs"
[features]
default = ["nightly", "ringbuf"]
vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
vendored = []
nightly = ["pgp/nightly"]
ringbuf = ["pgp/ringbuf"]
+9 -10
View File
@@ -5,16 +5,6 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
ENV PIP_DISABLE_PIP_VERSION_CHECK 1
# Install python tools (auditwheels,tox, ...)
ADD deps/build_python.sh /builder/build_python.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
# Install Rust nightly
ADD deps/build_rust.sh /builder/build_rust.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
# Install a recent Perl, needed to install OpenSSL
ADD deps/build_perl.sh /builder/build_perl.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -23,3 +13,12 @@ RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
ADD deps/build_openssl.sh /builder/build_openssl.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
ENV PIP_DISABLE_PIP_VERSION_CHECK 1
# Install python tools (auditwheels,tox, ...)
ADD deps/build_python.sh /builder/build_python.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
# Install Rust nightly
ADD deps/build_rust.sh /builder/build_rust.sh
RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
@@ -1,11 +1,11 @@
#!/bin/bash
PERL_VERSION=5.28.0
PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
PERL_VERSION=5.30.0
# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
echo "${PERL_SHA256} perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
tar xzf perl-${PERL_VERSION}.tar.gz
cd perl-${PERL_VERSION}
# echo "${PERL_SHA256} perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
tar -xzf perl-${PERL_VERSION}.tar.gz
cd perl-${PERL_VERSION}
./Configure -de
make
@@ -1,11 +1,8 @@
#!/bin/bash
set -e -x
set -e -x
# Install Rust
curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y
# Install Rust
curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
export PATH=/root/.cargo/bin:$PATH
rustc --version
# remove some 300-400 MB that we don't need for automated builds
rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/
+5 -1
View File
@@ -427,15 +427,19 @@ class TestOnlineAccount:
assert self_addr not in ev[2]
ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
def test_mvbox_sentbox_threads(self, acfactory):
def test_mvbox_sentbox_threads(self, acfactory, lp):
ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)
ac2 = acfactory.get_online_configuring_account()
wait_configuration_progress(ac2, 1000)
wait_configuration_progress(ac1, 1000)
lp.sec("configuration completed for both accounts")
chat = self.get_chat(ac1, ac2)
lp.sec("ac1: send text to chat with ac2")
chat.send_text("message1")
lp.sec("ac2: wait for incoming message")
ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
lp.sec("message arrived, test ends")
def test_move_works(self, acfactory):
ac1 = acfactory.get_online_configuring_account()
+1 -1
View File
@@ -1 +1 @@
nightly-2019-08-13
nightly-2019-11-06
+1 -1
View File
@@ -569,7 +569,7 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
pub fn dc_connect_to_configured_imap(context: &Context, imap: &Imap) -> libc::c_int {
let mut ret_connected = 0;
if imap.is_connected() {
if async_std::task::block_on(async move { imap.is_connected().await }) {
ret_connected = 1
} else if !context.sql.get_raw_config_bool(context, "configured") {
warn!(context, "Not configured, cannot connect.",);
+1 -1
View File
@@ -498,7 +498,7 @@ fn decrypt_if_autocrypt_message(
public_keyring_for_validate: &Keyring,
ret_valid_signatures: &mut HashSet<String>,
ret_gossip_headers: *mut *mut mailimf_fields,
) -> Result<(bool)> {
) -> Result<bool> {
/* The returned bool is true if we detected an Autocrypt-encrypted
message and successfully decrypted it. Decryption then modifies the
passed in mime structure in place. The returned bool is false
+670 -937
View File
File diff suppressed because it is too large Load Diff
+294
View File
@@ -0,0 +1,294 @@
use async_imap::{
error::{Error as ImapError, Result as ImapResult},
extensions::idle::Handle as ImapIdleHandle,
types::{Capabilities, Fetch, Mailbox, Name},
Client as ImapClient, Session as ImapSession,
};
use async_std::net::{self, TcpStream};
use async_std::prelude::*;
use async_tls::client::TlsStream;
use crate::login_param::CertificateChecks;
const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
#[derive(Debug)]
pub(crate) enum Client {
Secure(ImapClient<TlsStream<TcpStream>>),
Insecure(ImapClient<TcpStream>),
}
#[derive(Debug)]
pub(crate) enum Session {
Secure(ImapSession<TlsStream<TcpStream>>),
Insecure(ImapSession<TcpStream>),
}
#[derive(Debug)]
pub(crate) enum IdleHandle {
Secure(ImapIdleHandle<TlsStream<TcpStream>>),
Insecure(ImapIdleHandle<TcpStream>),
}
impl Client {
pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
addr: A,
domain: S,
_certificate_checks: CertificateChecks,
) -> ImapResult<Self> {
let stream = TcpStream::connect(addr).await?;
let tls = async_tls::TlsConnector::new();
let tls_stream = tls.connect(domain.as_ref(), stream)?.await?;
let mut client = ImapClient::new(tls_stream);
if std::env::var(DCC_IMAP_DEBUG).is_ok() {
client.debug = true;
}
let _greeting = client
.read_response()
.await
.expect("failed to read greeting");
Ok(Client::Secure(client))
}
pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
let stream = TcpStream::connect(addr).await?;
let mut client = ImapClient::new(stream);
if std::env::var(DCC_IMAP_DEBUG).is_ok() {
client.debug = true;
}
let _greeting = client
.read_response()
.await
.expect("failed to read greeting");
Ok(Client::Insecure(client))
}
pub async fn secure<S: AsRef<str>>(
self,
domain: S,
_certificate_checks: CertificateChecks,
) -> ImapResult<Client> {
match self {
Client::Insecure(client) => {
let tls = async_tls::TlsConnector::new();
let client_sec = client.secure(domain, &tls).await?;
Ok(Client::Secure(client_sec))
}
// Nothing to do
Client::Secure(_) => Ok(self),
}
}
pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
self,
auth_type: S,
authenticator: &A,
) -> Result<Session, (ImapError, Client)> {
match self {
Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
Ok(session) => Ok(Session::Secure(session)),
Err((err, c)) => Err((err, Client::Secure(c))),
},
Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
Ok(session) => Ok(Session::Insecure(session)),
Err((err, c)) => Err((err, Client::Insecure(c))),
},
}
}
pub async fn login<U: AsRef<str>, P: AsRef<str>>(
self,
username: U,
password: P,
) -> Result<Session, (ImapError, Client)> {
match self {
Client::Secure(i) => match i.login(username, password).await {
Ok(session) => Ok(Session::Secure(session)),
Err((err, c)) => Err((err, Client::Secure(c))),
},
Client::Insecure(i) => match i.login(username, password).await {
Ok(session) => Ok(Session::Insecure(session)),
Err((err, c)) => Err((err, Client::Insecure(c))),
},
}
}
}
impl Session {
pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
let res = match self {
Session::Secure(i) => i.capabilities().await?,
Session::Insecure(i) => i.capabilities().await?,
};
Ok(res)
}
pub async fn list(
&mut self,
reference_name: Option<&str>,
mailbox_pattern: Option<&str>,
) -> ImapResult<Vec<Name>> {
let res = match self {
Session::Secure(i) => {
i.list(reference_name, mailbox_pattern)
.await?
.collect::<ImapResult<_>>()
.await?
}
Session::Insecure(i) => {
i.list(reference_name, mailbox_pattern)
.await?
.collect::<ImapResult<_>>()
.await?
}
};
Ok(res)
}
pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
match self {
Session::Secure(i) => i.create(mailbox_name).await?,
Session::Insecure(i) => i.create(mailbox_name).await?,
}
Ok(())
}
pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
match self {
Session::Secure(i) => i.subscribe(mailbox).await?,
Session::Insecure(i) => i.subscribe(mailbox).await?,
}
Ok(())
}
pub async fn close(&mut self) -> ImapResult<()> {
match self {
Session::Secure(i) => i.close().await?,
Session::Insecure(i) => i.close().await?,
}
Ok(())
}
pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
let mbox = match self {
Session::Secure(i) => i.select(mailbox_name).await?,
Session::Insecure(i) => i.select(mailbox_name).await?,
};
Ok(mbox)
}
pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
where
S1: AsRef<str>,
S2: AsRef<str>,
{
let res = match self {
Session::Secure(i) => {
i.fetch(sequence_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
Session::Insecure(i) => {
i.fetch(sequence_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
};
Ok(res)
}
pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
where
S1: AsRef<str>,
S2: AsRef<str>,
{
let res = match self {
Session::Secure(i) => {
i.uid_fetch(uid_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
Session::Insecure(i) => {
i.uid_fetch(uid_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
};
Ok(res)
}
pub fn idle(self) -> IdleHandle {
match self {
Session::Secure(i) => {
let h = i.idle();
IdleHandle::Secure(h)
}
Session::Insecure(i) => {
let h = i.idle();
IdleHandle::Insecure(h)
}
}
}
pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
where
S1: AsRef<str>,
S2: AsRef<str>,
{
let res = match self {
Session::Secure(i) => {
i.uid_store(uid_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
Session::Insecure(i) => {
i.uid_store(uid_set, query)
.await?
.collect::<ImapResult<_>>()
.await?
}
};
Ok(res)
}
pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
&mut self,
uid_set: S1,
mailbox_name: S2,
) -> ImapResult<()> {
match self {
Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
}
Ok(())
}
pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
&mut self,
uid_set: S1,
mailbox_name: S2,
) -> ImapResult<()> {
match self {
Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
}
Ok(())
}
}
+9 -9
View File
@@ -245,10 +245,10 @@ impl Job {
&dest_folder,
&mut dest_uid,
) {
ImapResult::RetryLater => {
ImapActionResult::RetryLater => {
self.try_again_later(3i32, None);
}
ImapResult::Success => {
ImapActionResult::Success => {
message::update_server_uid(
context,
&msg.rfc724_mid,
@@ -256,7 +256,7 @@ impl Job {
dest_uid,
);
}
ImapResult::Failed | ImapResult::AlreadyDone => {}
ImapActionResult::Failed | ImapActionResult::AlreadyDone => {}
}
}
}
@@ -280,7 +280,7 @@ impl Job {
let mid = msg.rfc724_mid;
let server_folder = msg.server_folder.as_ref().unwrap();
let res = inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid);
if res == ImapResult::RetryLater {
if res == ImapActionResult::RetryLater {
self.try_again_later(-1i32, None);
return;
}
@@ -313,11 +313,11 @@ impl Job {
if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
let folder = msg.server_folder.as_ref().unwrap();
match inbox.set_seen(context, folder, msg.server_uid) {
ImapResult::RetryLater => {
ImapActionResult::RetryLater => {
self.try_again_later(3i32, None);
}
ImapResult::AlreadyDone => {}
ImapResult::Success | ImapResult::Failed => {
ImapActionResult::AlreadyDone => {}
ImapActionResult::Success | ImapActionResult::Failed => {
// XXX the message might just have been moved
// we want to send out an MDN anyway
// The job will not be retried so locally
@@ -343,7 +343,7 @@ impl Job {
.to_string();
let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32;
let inbox = context.inbox.read().unwrap();
if inbox.set_seen(context, &folder, uid) == ImapResult::RetryLater {
if inbox.set_seen(context, &folder, uid) == ImapActionResult::RetryLater {
self.try_again_later(3i32, None);
return;
}
@@ -361,7 +361,7 @@ impl Job {
.get_raw_config(context, "configured_mvbox_folder");
if let Some(dest_folder) = dest_folder {
let mut dest_uid = 0;
if ImapResult::RetryLater
if ImapActionResult::RetryLater
== inbox.mv(context, &folder, uid, &dest_folder, &mut dest_uid)
{
self.try_again_later(3, None);
+1 -1
View File
@@ -107,7 +107,7 @@ impl JobThread {
}
fn connect_to_imap(&self, context: &Context) -> bool {
if self.imap.is_connected() {
if async_std::task::block_on(async move { self.imap.is_connected().await }) {
return true;
}
+1
View File
@@ -40,6 +40,7 @@ pub mod contact;
pub mod context;
mod e2ee;
mod imap;
mod imap_client;
pub mod imex;
pub mod job;
mod job_thread;
+22 -22
View File
@@ -251,28 +251,28 @@ fn get_readable_flags(flags: i32) -> String {
res
}
pub fn dc_build_tls(
certificate_checks: CertificateChecks,
) -> Result<native_tls::TlsConnector, native_tls::Error> {
let mut tls_builder = native_tls::TlsConnector::builder();
match certificate_checks {
CertificateChecks::Automatic => {
// Same as AcceptInvalidCertificates for now.
// TODO: use provider database when it becomes available
tls_builder
.danger_accept_invalid_hostnames(true)
.danger_accept_invalid_certs(true)
}
CertificateChecks::Strict => &mut tls_builder,
CertificateChecks::AcceptInvalidHostnames => {
tls_builder.danger_accept_invalid_hostnames(true)
}
CertificateChecks::AcceptInvalidCertificates => tls_builder
.danger_accept_invalid_hostnames(true)
.danger_accept_invalid_certs(true),
}
.build()
}
// pub fn dc_build_tls(
// certificate_checks: CertificateChecks,
// ) -> Result<native_tls::TlsConnector, native_tls::Error> {
// let mut tls_builder = native_tls::TlsConnector::builder();
// match certificate_checks {
// CertificateChecks::Automatic => {
// // Same as AcceptInvalidCertificates for now.
// // TODO: use provider database when it becomes available
// tls_builder
// .danger_accept_invalid_hostnames(true)
// .danger_accept_invalid_certs(true)
// }
// CertificateChecks::Strict => &mut tls_builder,
// CertificateChecks::AcceptInvalidHostnames => {
// tls_builder.danger_accept_invalid_hostnames(true)
// }
// CertificateChecks::AcceptInvalidCertificates => tls_builder
// .danger_accept_invalid_hostnames(true)
// .danger_accept_invalid_certs(true),
// }
// .build()
// }
#[cfg(test)]
mod tests {
+6 -3
View File
@@ -5,7 +5,7 @@ use crate::constants::*;
use crate::context::Context;
use crate::error::Error;
use crate::events::Event;
use crate::login_param::{dc_build_tls, LoginParam};
use crate::login_param::LoginParam;
use crate::oauth2::*;
#[derive(DebugStub)]
@@ -65,8 +65,11 @@ impl Smtp {
let domain = &lp.send_server;
let port = lp.send_port as u16;
let tls = dc_build_tls(lp.smtp_certificate_checks).unwrap();
let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls);
let mut tls_config = rustls::ClientConfig::new();
tls_config
.root_store
.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
// oauth2