From a55d7f1d3a70c5bec597c599a68912816a13a0ba Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 12 Sep 2021 00:00:00 +0000 Subject: [PATCH] scheduler: rate limit IMAP loops Avoid hitting the server with repeated requests or reconnections if some operation, such as FETCH, keeps failing. --- Cargo.lock | 71 ++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/scheduler.rs | 11 ++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 040ee87d3..e9cfd147c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,12 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + [[package]] name = "ahash" version = "0.7.2" @@ -1114,6 +1120,7 @@ dependencies = [ "fast-socks5", "futures", "futures-lite", + "governor", "hex", "humansize", "image", @@ -1675,6 +1682,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.17" @@ -1767,19 +1780,46 @@ dependencies = [ "web-sys", ] +[[package]] +name = "governor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c5d2f987ee8f6dff3fa1a352058dc59b990e447e4c7846aa7d804971314f7b" +dependencies = [ + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "quanta", + "rand 0.8.3", + "smallvec", +] + [[package]] name = "half" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62aca2aba2d62b4a7f5b33f3712cb1b0692779a56fb510499d5c0aa594daeaf3" +[[package]] +name = "hashbrown" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" +dependencies = [ + "ahash 0.3.8", + "autocfg 1.0.1", +] + [[package]] name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash", + "ahash 0.7.2", ] [[package]] @@ -1788,7 +1828,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -1982,7 +2022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg 1.0.1", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -2320,6 +2360,15 @@ dependencies = [ "libc", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +dependencies = [ + "hashbrown 0.8.2", +] + [[package]] name = "nom" version = "4.2.3" @@ -2354,6 +2403,12 @@ dependencies = [ "version_check 0.9.3", ] +[[package]] +name = "nonzero_ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44a1290799eababa63ea60af0cbc3f03363e328e58f32fb0294798ed3e85f444" + [[package]] name = "num-bigint" version = "0.2.6" @@ -2829,6 +2884,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "quanta" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98dc777a7a39b76b1a26ae9d3f691f4c1bc0455090aa0b64dfa8cb7fc34c135" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" diff --git a/Cargo.toml b/Cargo.toml index dcb7c1336..bf61db6ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" } encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" } escaper = "0.1.1" futures = "0.3.17" +governor = "0.3.2" hex = "0.4.0" image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] } indexmap = "1.7.0" diff --git a/src/scheduler.rs b/src/scheduler.rs index bd9065ff2..94370f023 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -4,6 +4,7 @@ use async_std::{ channel::{self, Receiver, Sender}, task, }; +use governor; use crate::config::Config; use crate::context::Context; @@ -70,6 +71,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne shutdown_sender, } = inbox_handlers; + let quota = governor::Quota::per_minute(std::num::NonZeroU32::new(1).unwrap()) + .allow_burst(std::num::NonZeroU32::new(5).unwrap()); + let rate_limiter = governor::RateLimiter::direct(quota); + let ctx1 = ctx.clone(); let fut = async move { started @@ -82,6 +87,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne let mut jobs_loaded = 0; let mut info = InterruptInfo::default(); loop { + rate_limiter.until_ready().await; match job::load_next(&ctx, Thread::Imap, &info) .await .ok() @@ -247,6 +253,10 @@ async fn simple_imap_loop( let ctx1 = ctx.clone(); + let quota = governor::Quota::per_minute(std::num::NonZeroU32::new(2).unwrap()) + .allow_burst(std::num::NonZeroU32::new(5).unwrap()); + let rate_limiter = governor::RateLimiter::direct(quota); + let fut = async move { started .send(()) @@ -255,6 +265,7 @@ async fn simple_imap_loop( let ctx = ctx1; loop { + rate_limiter.until_ready().await; fetch_idle(&ctx, &mut connection, folder).await; } };