Add Quota to Connectivity View (#2612)

* add imap::get_quota_roots()

* schedule quote-checking job on getting connectivity-html

* get quota and debug print it

* basic quota output

* update quota at most once per minute, emit event on changes

* use more meaningful names

* add some comments, move update_recent_quota() to quota.rs

* show root name only if there are several roots

* make clippy happy, some refactorings

* allow only one update-quota job per time

* add now supported QUOTA to standards.md

Cargo.lock

@@ -238,8 +238,7 @@ dependencies = [
 [[package]]
 name = "async-imap"
 version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/async-email/async-imap#4ce7da455618c387b87b2905a80935107bc69afc"
-checksum = "bbb2df4b37a99456360a9ab475b723e3a499d51e060ab1bdd8d7565d23dcb74b"
 dependencies = [
  "async-native-tls",
  "async-std",
@@ -250,7 +249,7 @@ dependencies = [
  "imap-proto",
  "lazy_static",
  "log",
- "nom 5.1.2",
+ "nom 6.2.1",
  "pin-utils",
  "rental",
  "stop-token",
@@ -522,6 +521,18 @@ version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1"
 
+[[package]]
+name = "bitvec"
+version = "0.19.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -1134,6 +1145,7 @@ dependencies = [
  "futures",
  "futures-lite",
  "hex",
+ "humansize",
  "image",
  "indexmap",
  "itertools 0.10.1",
@@ -1586,6 +1598,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "funty"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
+
 [[package]]
 name = "futures"
 version = "0.3.16"
@@ -1916,6 +1934,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "humansize"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026"
+
 [[package]]
 name = "humantime"
 version = "1.3.0"
@@ -1961,11 +1985,11 @@ dependencies = [
 
 [[package]]
 name = "imap-proto"
-version = "0.11.0"
+version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3091b99ee5b80f9b010eb6f962af9495ad06561bf662126b077e8ca30e463182"
+checksum = "3ad9b46a79efb6078e578ae04e51463d7c3e8767864687f7e63095b3cbefafbb"
 dependencies = [
- "nom 5.1.2",
+ "nom 6.2.1",
 ]
 
 [[package]]
@@ -2343,6 +2367,19 @@ dependencies = [
  "version_check 0.9.3",
 ]
 
+[[package]]
+name = "nom"
+version = "6.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c5c51b9083a3c620fa67a2a635d1ce7d95b897e957d6b28ff9a5da960a103a6"
+dependencies = [
+ "bitvec",
+ "funty",
+ "lexical-core",
+ "memchr",
+ "version_check 0.9.3",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.2.6"
@@ -2874,6 +2911,12 @@ dependencies = [
  "rusqlite",
 ]
 
+[[package]]
+name = "radium"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
+
 [[package]]
 name = "radix_trie"
 version = "0.2.1"
@@ -3651,6 +3694,12 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "tempfile"
 version = "3.2.0"
@@ -4166,6 +4215,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "wyz"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
+
 [[package]]
 name = "x25519-dalek"
 version = "1.1.0"

Cargo.toml

@@ -16,7 +16,7 @@ deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1.0.42"
-async-imap = "0.5.0"
+async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3.3" }
 async-smtp = { git = "https://github.com/async-email/async-smtp", rev="c8800625f7cf29f437143ac7e720ac2730a0962f" }
 async-std-resolver = "0.20.3"
@@ -72,6 +72,7 @@ thiserror = "1.0.26"
 toml = "0.5.6"
 url = "2.2.2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+humansize = "1.1.1"
 
 [dev-dependencies]
 ansi_term = "0.12.0"

src/context.rs

@@ -22,6 +22,7 @@ use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
+use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
@@ -62,6 +63,10 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
+    /// Recently loaded quota information, if any.
+    /// Set to `None` if quota was never tried to load.
+    pub(crate) quota: RwLock<Option<QuotaInfo>>,
+
     pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
 
     /// ID for this `Context` in the current process.
@@ -139,6 +144,7 @@ impl Context {
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
+            quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
         };

src/imap.rs

@@ -5,10 +5,10 @@
 
 use std::{cmp, cmp::max, collections::BTreeMap};
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{anyhow, bail, format_err, Context as _, Result};
 use async_imap::{
     error::Result as ImapResult,
-    types::{Fetch, Flag, Mailbox, Name, NameAttribute, UnsolicitedResponse},
+    types::{Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse},
 };
 use async_std::channel::Receiver;
 use async_std::prelude::*;
@@ -153,6 +153,10 @@ struct ImapConfig {
     /// True if the server has MOVE capability as defined in
     /// <https://tools.ietf.org/html/rfc6851>
     pub can_move: bool,
+
+    /// True if the server has QUOTA capability as defined in
+    /// <https://tools.ietf.org/html/rfc2087>
+    pub can_check_quota: bool,
 }
 
 impl Imap {
@@ -186,6 +190,7 @@ impl Imap {
             selected_folder_needs_expunge: false,
             can_idle: false,
             can_move: false,
+            can_check_quota: false,
         };
 
         let imap = Imap {
@@ -362,6 +367,7 @@ impl Imap {
                 Ok(caps) => {
                     self.config.can_idle = caps.has_str("IDLE");
                     self.config.can_move = caps.has_str("MOVE");
+                    self.config.can_check_quota = caps.has_str("QUOTA");
                     self.capabilities_determined = true;
                     Ok(())
                 }
@@ -1392,6 +1398,22 @@ impl Imap {
         }
         unsolicited_exists
     }
+
+    pub fn can_check_quota(&self) -> bool {
+        self.config.can_check_quota
+    }
+
+    pub async fn get_quota_roots(
+        &mut self,
+        mailbox_name: &str,
+    ) -> Result<(Vec<QuotaRoot>, Vec<Quota>)> {
+        if let Some(session) = self.session.as_mut() {
+            let quota_roots = session.get_quota_root(mailbox_name).await?;
+            Ok(quota_roots)
+        } else {
+            Err(anyhow!("Not connected to IMAP, no session"))
+        }
+    }
 }
 
 /// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.

src/imap/scan_folders.rs

@@ -102,7 +102,7 @@ impl Imap {
     }
 }
 
-async fn get_watched_folders(context: &Context) -> Vec<String> {
+pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
     let mut res = Vec::new();
     let folder_watched_configured = &[
         (Config::SentboxWatch, Config::ConfiguredSentboxFolder),

src/job.rs

@@ -95,6 +95,9 @@ pub enum Action {
     FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
+    // this is user initiated so it should have a fairly high priority
+    UpdateRecentQuota = 140,
+
     // Moving message is prioritized lower than deletion so we don't
     // bother moving message if it is already scheduled for deletion.
     MoveMsg = 200,
@@ -130,6 +133,7 @@ impl From<Action> for Thread {
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
+            UpdateRecentQuota => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
@@ -1148,6 +1152,7 @@ async fn perform_job_action(
             sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
+        Action::UpdateRecentQuota => context.update_recent_quota(connection.inbox()).await,
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -1210,7 +1215,8 @@ pub async fn add(context: &Context, job: Job) {
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
-            | Action::MoveMsg => {
+            | Action::MoveMsg
+            | Action::UpdateRecentQuota => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))

src/lib.rs

@@ -75,6 +75,7 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
+pub mod quota;
 pub mod securejoin;
 mod simplify;
 mod smtp;

src/quota.rs

@@ -0,0 +1,100 @@
+use anyhow::{anyhow, Result};
+use async_imap::types::{Quota, QuotaResource};
+use indexmap::IndexMap;
+
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::imap::scan_folders::get_watched_folders;
+use crate::imap::Imap;
+use crate::job::{Action, Status};
+use crate::param::Params;
+use crate::{job, EventType};
+
+/// warn about a nearly full mailbox after this usage percentage is reached.
+/// quota icon is "yellow".
+pub const QUOTA_WARN_THRESHOLD_PERCENTAGE: u64 = 80;
+
+// warning is already issued at QUOTA_WARN_THRESHOLD_PERCENTAGE,
+// this threshold only makes the quota icon "red".
+pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 99;
+
+// if recent quota is older,
+// it is re-fetched on dc_get_connectivity_html()
+pub const QUOTA_MAX_AGE_SECONDS: i64 = 60;
+
+#[derive(Debug)]
+pub struct QuotaInfo {
+    /// Recently loaded quota information.
+    /// set to `Err()` if the provider does not support quota or on other errors,
+    /// set to `Ok()` for valid quota information.
+    /// Updated by `Action::UpdateRecentQuota`
+    pub(crate) recent: Result<IndexMap<String, Vec<QuotaResource>>>,
+
+    /// Timestamp when structure was modified.
+    pub(crate) modified: i64,
+}
+
+async fn get_unique_quota_roots_and_usage(
+    folders: Vec<String>,
+    imap: &mut Imap,
+) -> Result<IndexMap<String, Vec<QuotaResource>>> {
+    let mut unique_quota_roots: IndexMap<String, Vec<QuotaResource>> = IndexMap::new();
+    for folder in folders {
+        let (quota_roots, quotas) = &imap.get_quota_roots(&folder).await?;
+        // if there are new quota roots found in this imap folder, add them to the list
+        for qr_entries in quota_roots {
+            for quota_root_name in &qr_entries.quota_root_names {
+                // the quota for that quota root
+                let quota: Quota = quotas
+                    .iter()
+                    .find(|q| &q.root_name == quota_root_name)
+                    .cloned()
+                    .ok_or_else(|| anyhow!("quota_root should have a quota"))?;
+                // replace old quotas, because between fetching quotaroots for folders,
+                // messages could be recieved and so the usage could have been changed
+                *unique_quota_roots
+                    .entry(quota_root_name.clone())
+                    .or_insert(vec![]) = quota.resources;
+            }
+        }
+    }
+    Ok(unique_quota_roots)
+}
+
+impl Context {
+    // Adds a job to update `quota.recent`
+    pub(crate) async fn schedule_quota_update(&self) {
+        job::kill_action(self, Action::UpdateRecentQuota).await;
+        job::add(
+            self,
+            job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
+        )
+        .await;
+    }
+
+    /// Updates `quota.recent`, sets `quota.modified` to the current time
+    /// and emits an event to let the UIs update connectivity view.
+    ///
+    /// Called in response to `Action::UpdateRecentQuota`.
+    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(self).await {
+            warn!(self, "could not connect: {:?}", err);
+            return Status::RetryNow;
+        }
+
+        let quota = if imap.can_check_quota() {
+            let folders = get_watched_folders(self).await;
+            get_unique_quota_roots_and_usage(folders, imap).await
+        } else {
+            Err(anyhow!("Quota not supported by your provider."))
+        };
+
+        *self.quota.write().await = Some(QuotaInfo {
+            recent: quota,
+            modified: time(),
+        });
+
+        self.emit_event(EventType::ConnectivityChanged);
+        Status::Finished(Ok(()))
+    }
+}

src/scheduler/connectivity.rs

@@ -3,9 +3,14 @@ use std::{ops::Deref, sync::Arc};
 
 use async_std::sync::{Mutex, RwLockReadGuard};
 
+use crate::dc_tools::time;
 use crate::events::EventType;
+use crate::quota::{
+    QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
+};
 use crate::{config::Config, scheduler::Scheduler};
 use crate::{context::Context, log::LogExt};
+use humansize::{file_size_opts, FileSize};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)]
 pub enum Connectivity {
@@ -403,6 +408,81 @@ impl Context {
         ret += &*escaper::encode_minimal(&detailed.to_string_smtp(self));
         ret += "</li></ul>";
 
+        ret += "<h3>Quota</h3><ul>";
+        let quota = self.quota.read().await;
+        if let Some(quota) = &*quota {
+            match &quota.recent {
+                Ok(quota) => {
+                    let roots_cnt = quota.len();
+                    for (root_name, resources) in quota {
+                        use async_imap::types::QuotaResourceName::*;
+                        for resource in resources {
+                            ret += "<li>";
+
+                            let usage_percent = resource.get_usage_percentage();
+                            if usage_percent >= QUOTA_ERROR_THRESHOLD_PERCENTAGE {
+                                ret += "<span class=\"red dot\"></span> ";
+                            } else if usage_percent >= QUOTA_WARN_THRESHOLD_PERCENTAGE {
+                                ret += "<span class=\"yellow dot\"></span> ";
+                            } else {
+                                ret += "<span class=\"green dot\"></span> ";
+                            }
+
+                            // root name is empty eg. for gmail and redundant eg. for riseup.
+                            // therefore, use it only if there are really several roots.
+                            if roots_cnt > 1 && !root_name.is_empty() {
+                                ret +=
+                                    &format!("<b>{}:</b> ", &*escaper::encode_minimal(root_name));
+                            } else {
+                                info!(self, "connectivity: root name hidden: \"{}\"", root_name);
+                            }
+
+                            ret += &match &resource.name {
+                                Atom(resource_name) => {
+                                    format!(
+                                        "<b>{}:</b> {} of {} used",
+                                        &*escaper::encode_minimal(resource_name),
+                                        resource.usage.to_string(),
+                                        resource.limit.to_string(),
+                                    )
+                                }
+                                Message => {
+                                    format!(
+                                        "<b>Messages:</b> {} of {} used",
+                                        resource.usage.to_string(),
+                                        resource.limit.to_string(),
+                                    )
+                                }
+                                Storage => {
+                                    let usage = (resource.usage * 1024)
+                                        .file_size(file_size_opts::BINARY)
+                                        .unwrap_or_default();
+                                    let limit = (resource.limit * 1024)
+                                        .file_size(file_size_opts::BINARY)
+                                        .unwrap_or_default();
+                                    format!("<b>Storage:</b> {} of {} used", usage, limit)
+                                }
+                            };
+                            ret += &format!(" ({}%)", usage_percent);
+
+                            ret += "</li>";
+                        }
+                    }
+                }
+                Err(e) => {
+                    ret += format!("<li>{}</li>", e).as_str();
+                }
+            }
+
+            if quota.modified + QUOTA_MAX_AGE_SECONDS < time() {
+                self.schedule_quota_update().await;
+            }
+        } else {
+            ret += "<li>One moment...</li>";
+            self.schedule_quota_update().await;
+        }
+        ret += "</ul>";
+
         ret += "</body></html>\n";
         ret
     }

standards.md

@@ -10,6 +10,7 @@ Text and Quote encoding          | Fixed, Flowed ([RFC 3676](https://tools.ietf.
 Filename encoding                | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231))
 Identify server folders          | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154))
 Push                             | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177))
+Quota                            | IMAP QUOTA extension ([RFC 2087](https://tools.ietf.org/html/rfc2087))
 Authorization                    | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749))
 End-to-end encryption            | [Autocrypt Level 1](https://autocrypt.org/level1.html), OpenPGP ([RFC 4880](https://tools.ietf.org/html/rfc4880)), Security Multiparts for MIME ([RFC 1847](https://tools.ietf.org/html/rfc1847)) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html)
 Configuration assistance         | [Autoconfigure](https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)