also make smtp respect CertificateChecks setting roughly

cleanup select_folder and fix idle/termination issues

.github/workflows/rust.yml

@@ -1,30 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-  push:
-
-env:
-  RUSTFLAGS: -Dwarnings
-
-jobs:
-  build:
-    name: 3.7 python tests against core
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-
-      - uses: actions-rs/toolchain@v1
-        with:
-           profile: minimal
-           toolchain: nightly
-           override: true
-           components: rustfmt
-
-      - name: Setup python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.x
-          architecture: x64
-
-      - run: bash ci_scripts/run-python.sh 

ci_scripts/run-python.sh

@@ -13,6 +13,7 @@ export TOXWORKDIR=.docker-tox
 
 export PATH=/root/.cargo/bin:$PATH
 cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
@@ -21,27 +22,36 @@ export DCC_RS_DEV=$(pwd)
 # needed by tox below.
 export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
-
-pushd python
-# prepare a clean tox run
-rm -rf tests/__pycache__
-rm -rf src/deltachat/__pycache__
-export PYTHONDONTWRITEBYTECODE=1
-
-# run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
-# allows running of "liveconfig" tests but for speed reasons
-# we run them only for the highest python version we support
-
-# we split out qr-tests run to minimize likelyness of flaky tests
-# (some qr tests are pretty heavy in terms of send/received
-# messages and rust's imap code likely has concurrency problems) 
-tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
-tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
-unset DCC_PY_LIVECONFIG
-#tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
-#tox --workdir "$TOXWORKDIR" -e auditwheels
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
 popd
 
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
 # if [ -n "$DOCS" ]; then
 #     echo -----------------------
 #     echo generating python docs

src/imap.rs

@@ -312,21 +312,18 @@ impl Imap {
                 return false;
             }
 
-            let teardown = match &mut *self.session.lock().await {
+            let (teardown, can_idle, has_xlist) = match &mut *self.session.lock().await {
                 Some(ref mut session) => match session.capabilities().await {
                     Ok(caps) => {
                         if !context.sql.is_open() {
                             warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.mail_user,);
-                            true
+                            (true, false, false)
                         } else {
                             let can_idle = caps.has_str("IDLE");
                             let has_xlist = caps.has_str("XLIST");
                             let caps_list = caps
                                 .iter()
                                 .fold(String::new(), |s, c| s + &format!(" {:?}", c));
-                            self.config.write().await.can_idle = can_idle;
-                            self.config.write().await.has_xlist = has_xlist;
-                            *self.connected.lock().await = true;
                             emit_event!(
                                 context,
                                 Event::ImapConnected(format!(
@@ -334,22 +331,25 @@ impl Imap {
                                     lp.mail_user, caps_list,
                                 ))
                             );
-                            false
+                            (false, can_idle, has_xlist)
                         }
                     }
                     Err(err) => {
                         info!(context, "CAPABILITY command error: {}", err);
-                        true
+                        (true, false, false)
                     }
                 },
-                None => true,
+                None => (true, false, false),
             };
 
             if teardown {
-                self.disconnect(context);
-
+                self.unsetup_handle(context).await;
+                self.free_connect_params().await;
                 false
             } else {
+                self.config.write().await.can_idle = can_idle;
+                self.config.write().await.has_xlist = has_xlist;
+                *self.connected.lock().await = true;
                 true
             }
         })
@@ -746,13 +746,10 @@ impl Imap {
     pub fn idle(&self, context: &Context) {
         task::block_on(async move {
             if self.config.read().await.selected_folder.is_none() {
-                // this probably means that we are in teardown
-                // in any case we can't perform any idling
                 return;
             }
-
             if !self.config.read().await.can_idle {
-                self.fake_idle(context, true).await;
+                self.fake_idle(context).await;
                 return;
             }
 
@@ -768,7 +765,7 @@ impl Imap {
                         "idle select_folder failed {:?}",
                         watch_folder.as_ref()
                     );
-                    self.fake_idle(context, true).await;
+                    self.fake_idle(context).await;
                     return;
                 }
             }
@@ -839,7 +836,7 @@ impl Imap {
         });
     }
 
-    pub(crate) async fn fake_idle(&self, context: &Context, use_network: bool) {
+    async fn fake_idle(&self, context: &Context) {
         // Idle using timeouts. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job
         let fake_idle_start_time = SystemTime::now();
@@ -849,20 +846,13 @@ impl Imap {
         task::block_on(async move {
             let interrupt = stop_token::StopSource::new();
 
-            // we use 1000 minutes if we are told to not try network
-            // which can happen because the watch_folder is not defined
-            // but clients are still calling us in a loop.
-            // if we are to use network, we check every minute if there
-            // is new mail -- TODO: make this more flexible
-            let secs = if use_network { 60 } else { 60000 };
-            let interval = async_std::stream::interval(Duration::from_secs(secs));
+            // TODO: More flexible interval
+            let interval = async_std::stream::interval(Duration::from_secs(10));
             let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
             *self.interrupt.lock().await = Some(interrupt);
 
             while let Some(_) = interrupt_interval.next().await {
-                if !use_network {
-                    continue;
-                }
+                // check if we want to finish fake-idling.
                 if !self.is_connected().await {
                     // try to connect with proper login params
                     // (setup_handle_if_needed might not know about them if we
@@ -1164,6 +1154,13 @@ impl Imap {
                     .list_folders(session, context)
                     .await
                     .expect("no folders found");
+                let delimiter = self.config.read().await.imap_delimiter;
+                let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+
+                let mut mvbox_folder = folders
+                    .iter()
+                    .find(|folder| folder.name() == "DeltaChat" || folder.name() == fallback_folder)
+                    .map(|n| n.name().to_string());
 
                 let sentbox_folder =
                     folders
@@ -1172,15 +1169,6 @@ impl Imap {
                             FolderMeaning::SentObjects => true,
                             _ => false,
                         });
-                info!(context, "sentbox folder is {:?}", sentbox_folder);
-
-                let delimiter = self.config.read().await.imap_delimiter;
-                let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
-
-                let mut mvbox_folder = folders
-                    .iter()
-                    .find(|folder| folder.name() == "DeltaChat" || folder.name() == fallback_folder)
-                    .map(|n| n.name().to_string());
 
                 if mvbox_folder.is_none() && 0 != (flags as usize & DC_CREATE_MVBOX) {
                     info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);

src/job_thread.rs

@@ -110,14 +110,9 @@ impl JobThread {
         if async_std::task::block_on(async move { self.imap.is_connected().await }) {
             return true;
         }
-        let watch_folder_name = match context.sql.get_raw_config(context, self.folder_config_name) {
-            Some(name) => name,
-            None => {
-                return false;
-            }
-        };
 
-        let ret_connected = dc_connect_to_configured_imap(context, &self.imap) != 0;
+        let mut ret_connected = dc_connect_to_configured_imap(context, &self.imap) != 0;
+
         if ret_connected {
             if context
                 .sql
@@ -128,7 +123,12 @@ impl JobThread {
                 self.imap.configure_folders(context, 0x1);
             }
 
-            self.imap.set_watch_folder(watch_folder_name);
+            if let Some(mvbox_name) = context.sql.get_raw_config(context, self.folder_config_name) {
+                self.imap.set_watch_folder(mvbox_name);
+            } else {
+                self.imap.disconnect(context);
+                ret_connected = false;
+            }
         }
 
         ret_connected
@@ -170,18 +170,10 @@ impl JobThread {
             }
         }
 
-        if self.connect_to_imap(context) {
-            info!(context, "{}-IDLE started...", self.name,);
-            self.imap.idle(context);
-            info!(context, "{}-IDLE ended.", self.name);
-        } else {
-            // It's probably wrong that the thread even runs
-            // but let's call fake_idle and tell it to not try network at all.
-            // (once we move to rust-managed threads this problem goes away)
-            info!(context, "{}-IDLE not connected, fake-idling", self.name);
-            async_std::task::block_on(async move { self.imap.fake_idle(context, false).await });
-            info!(context, "{}-IDLE fake-idling finished", self.name);
-        }
+        self.connect_to_imap(context);
+        info!(context, "{}-IDLE started...", self.name,);
+        self.imap.idle(context);
+        info!(context, "{}-IDLE ended.", self.name);
 
         self.state.0.lock().unwrap().using_handle = false;
     }

src/login_param.rs

@@ -6,7 +6,6 @@ use crate::error::Error;
 use async_std::sync::Arc;
 use rustls;
 use webpki;
-use webpki_roots;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive)]
 #[repr(i32)]
@@ -271,10 +270,6 @@ impl rustls::ServerCertVerifier for NoCertificateVerification {
 
 pub fn dc_build_tls_config(certificate_checks: CertificateChecks) -> rustls::ClientConfig {
     let mut config = rustls::ClientConfig::new();
-    config
-        .root_store
-        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
-
     match certificate_checks {
         CertificateChecks::Strict => {}
         CertificateChecks::Automatic => {