Merge v1.112.8

src/configure.rs

@@ -11,7 +11,6 @@
 
 mod auto_mozilla;
 mod auto_outlook;
-mod read_url;
 mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};

src/configure/auto_mozilla.rs

@@ -6,10 +6,10 @@ use std::str::FromStr;
 
 use quick_xml::events::{BytesStart, Event};
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 #[derive(Debug)]

src/configure/auto_outlook.rs

@@ -7,9 +7,9 @@ use std::io::BufRead;
 
 use quick_xml::events::Event;
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 /// Result of parsing a single `Protocol` tag.

src/configure/read_url.rs

@@ -1,44 +0,0 @@
-use anyhow::{anyhow, format_err};
-
-use crate::context::Context;
-use crate::socks::Socks5Config;
-
-pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
-    match read_url_inner(context, url).await {
-        Ok(s) => {
-            info!(context, "Successfully read url {}", url);
-            Ok(s)
-        }
-        Err(e) => {
-            info!(context, "Can't read URL {}: {:#}", url, e);
-            Err(format_err!("Can't read URL {}: {:#}", url, e))
-        }
-    }
-}
-
-pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result<String> {
-    let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let client = crate::http::get_client(socks5_config)?;
-    let mut url = url.to_string();
-
-    // Follow up to 10 http-redirects
-    for _i in 0..10 {
-        let response = client.get(&url).send().await?;
-        if response.status().is_redirection() {
-            let headers = response.headers();
-            let header = headers
-                .get_all("location")
-                .iter()
-                .last()
-                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
-                .to_str()?;
-            info!(context, "Following redirect to {}", header);
-            url = header.to_string();
-            continue;
-        }
-
-        return response.text().await.map_err(Into::into);
-    }
-
-    Err(format_err!("Followed 10 redirections"))
-}

src/http.rs

@@ -1,24 +0,0 @@
-//! # HTTP module.
-
-use std::time::Duration;
-
-use anyhow::Result;
-
-use crate::socks::Socks5Config;
-
-const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
-
-pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
-    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
-    let builder = if let Some(socks5_config) = socks5_config {
-        let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
-        builder.proxy(proxy)
-    } else {
-        // Disable usage of "system" proxy configured via environment variables.
-        // It is enabled by default in `reqwest`, see
-        // <https://docs.rs/reqwest/0.11.14/reqwest/struct.ClientBuilder.html#method.no_proxy>
-        // for documentation.
-        builder.no_proxy()
-    };
-    Ok(builder.build()?)
-}

src/lib.rs

@@ -63,7 +63,6 @@ mod decrypt;
 pub mod download;
 mod e2ee;
 pub mod ephemeral;
-mod http;
 mod imap;
 pub mod imex;
 pub mod release;
@@ -100,7 +99,7 @@ mod dehtml;
 mod authres;
 mod color;
 pub mod html;
-mod net;
+pub mod net;
 pub mod plaintext;
 pub mod summary;
 

src/net.rs

@@ -1,4 +1,4 @@
-///! # Common network utilities.
+//! # Common network utilities.
 use std::net::{IpAddr, SocketAddr};
 use std::pin::Pin;
 use std::str::FromStr;
@@ -12,9 +12,12 @@ use tokio_io_timeout::TimeoutStream;
 use crate::context::Context;
 use crate::tools::time;
 
+pub(crate) mod http;
 pub(crate) mod session;
 pub(crate) mod tls;
 
+pub use http::{read_url, read_url_blob, Response as HttpResponse};
+
 async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result<TcpStream> {
     let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))
         .await

src/net/http.rs

@@ -0,0 +1,94 @@
+//! # HTTP module.
+
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use mime::Mime;
+
+use crate::context::Context;
+use crate::socks::Socks5Config;
+
+const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// HTTP(S) GET response.
+#[derive(Debug)]
+pub struct Response {
+    /// Response body.
+    pub blob: Vec<u8>,
+
+    /// MIME type exntracted from the `Content-Type` header, if any.
+    pub mimetype: Option<String>,
+
+    /// Encoding extracted from the `Content-Type` header, if any.
+    pub encoding: Option<String>,
+}
+
+/// Retrieves the text contents of URL using HTTP GET request.
+pub async fn read_url(context: &Context, url: &str) -> Result<String> {
+    Ok(read_url_inner(context, url).await?.text().await?)
+}
+
+/// Retrieves the binary contents of URL using HTTP GET request.
+pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
+    let response = read_url_inner(context, url).await?;
+    let content_type = response
+        .headers()
+        .get(reqwest::header::CONTENT_TYPE)
+        .and_then(|value| value.to_str().ok())
+        .and_then(|value| value.parse::<Mime>().ok());
+    let mimetype = content_type
+        .as_ref()
+        .map(|mime| mime.essence_str().to_string());
+    let encoding = content_type.as_ref().and_then(|mime| {
+        mime.get_param(mime::CHARSET)
+            .map(|charset| charset.as_str().to_string())
+    });
+    let blob: Vec<u8> = response.bytes().await?.into();
+    Ok(Response {
+        blob,
+        mimetype,
+        encoding,
+    })
+}
+
+async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Response> {
+    let socks5_config = Socks5Config::from_database(&context.sql).await?;
+    let client = get_client(socks5_config)?;
+    let mut url = url.to_string();
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let response = client.get(&url).send().await?;
+        if response.status().is_redirection() {
+            let headers = response.headers();
+            let header = headers
+                .get_all("location")
+                .iter()
+                .last()
+                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
+                .to_str()?;
+            info!(context, "Following redirect to {}", header);
+            url = header.to_string();
+            continue;
+        }
+
+        return Ok(response);
+    }
+
+    Err(anyhow!("Followed 10 redirections"))
+}
+
+pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
+    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
+    let builder = if let Some(socks5_config) = socks5_config {
+        let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
+        builder.proxy(proxy)
+    } else {
+        // Disable usage of "system" proxy configured via environment variables.
+        // It is enabled by default in `reqwest`, see
+        // <https://docs.rs/reqwest/0.11.14/reqwest/struct.ClientBuilder.html#method.no_proxy>
+        // for documentation.
+        builder.no_proxy()
+    };
+    Ok(builder.build()?)
+}

src/oauth2.rs

@@ -160,7 +160,7 @@ pub(crate) async fn get_oauth2_access_token(
 
         // ... and POST
         let socks5_config = Socks5Config::from_database(&context.sql).await?;
-        let client = crate::http::get_client(socks5_config)?;
+        let client = crate::net::http::get_client(socks5_config)?;
 
         let response: Response = match client.post(post_url).form(&post_param).send().await {
             Ok(resp) => match resp.json().await {
@@ -291,7 +291,7 @@ impl Oauth2 {
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
         let socks5_config = Socks5Config::from_database(&context.sql).await.ok()?;
-        let client = match crate::http::get_client(socks5_config) {
+        let client = match crate::net::http::get_client(socks5_config) {
             Ok(cl) => cl,
             Err(err) => {
                 warn!(context, "failed to get HTTP client: {}", err);

src/qr.rs

@@ -526,7 +526,7 @@ struct CreateAccountErrorResponse {
 async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
     let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let response = crate::http::get_client(socks5_config)?
+    let response = crate::net::http::get_client(socks5_config)?
         .post(url_str)
         .send()
         .await?;