Merge v1.112.8

CHANGELOG.md

@@ -38,6 +38,12 @@
 - Fix python bindings README documentation on installing the bindings from source.
 - Remove confusing log line "ignoring unsolicited response Recent(…)". #3934
 
+## [1.112.8] - 2023-04-20
+
+### Changes
+- Add `get_http_response` JSON-RPC API.
+- Add C API to get HTTP responses.
+
 ## [1.112.7] - 2023-04-17
 
 ### Fixes

Cargo.lock

@@ -1175,6 +1175,7 @@ dependencies = [
  "libc",
  "log",
  "mailparse",
+ "mime",
  "num-derive",
  "num-traits",
  "num_cpus",
@@ -2806,9 +2807,9 @@ dependencies = [
 
 [[package]]
 name = "mime"
-version = "0.3.16"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "minimal-lexical"

Cargo.toml

@@ -57,6 +57,7 @@ kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
 mailparse = "0.14"
+mime = "0.3.17"
 num_cpus = "1.15"
 num-derive = "0.3"
 num-traits = "0.2"

deltachat-ffi/deltachat.h

@@ -25,6 +25,7 @@ typedef struct _dc_event     dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 typedef struct _dc_backup_provider dc_backup_provider_t;
+typedef struct _dc_http_response dc_http_response_t;
 
 // Alias for backwards compatibility, use dc_event_emitter_t instead.
 typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
@@ -5127,6 +5128,72 @@ int             dc_provider_get_status                (const dc_provider_t* prov
 void            dc_provider_unref                     (dc_provider_t* provider);
 
 
+/**
+ * Return an HTTP(S) GET response.
+ * This function can be used to download remote content for HTML emails.
+ *
+ * @memberof dc_context_t
+ * @param context The context object to take proxy settings from.
+ * @param url HTTP or HTTPS URL.
+ * @return The response must be released using dc_http_response_unref() after usage.
+ *     NULL is returned on errors.
+ */
+dc_http_response_t*     dc_get_http_response      (const dc_context_t* context, const char* url);
+
+
+/**
+ * @class dc_http_response_t
+ *
+ * An object containing an HTTP(S) GET response.
+ * Created by dc_get_http_response().
+ */
+
+
+/**
+ * Returns HTTP response MIME type as a string, e.g. "text/plain" or "text/html".
+ *
+ * @memberof dc_http_response_t
+ * @param response HTTP response as returned by dc_get_http_response().
+ * @return The string which must be released using dc_str_unref() after usage. May be NULL.
+ */
+char*                   dc_http_response_get_mimetype (const dc_http_response_t* response);
+
+/**
+ * Returns HTTP response encoding, e.g. "utf-8".
+ *
+ * @memberof dc_http_response_t
+ * @param response HTTP response as returned by dc_get_http_response().
+ * @return The string which must be released using dc_str_unref() after usage. May be NULL.
+ */
+char*                   dc_http_response_get_encoding (const dc_http_response_t* response);
+
+/**
+ * Returns HTTP response contents.
+ *
+ * @memberof dc_http_response_t
+ * @param response HTTP response as returned by dc_get_http_response().
+ * @return The blob which must be released using dc_str_unref() after usage. NULL is never returned.
+ */
+uint8_t*                dc_http_response_get_blob     (const dc_http_response_t* response);
+
+/**
+ * Returns HTTP response content size.
+ *
+ * @memberof dc_http_response_t
+ * @param response HTTP response as returned by dc_get_http_response().
+ * @return The blob size.
+ */
+size_t                  dc_http_response_get_size     (const dc_http_response_t* response);
+
+/**
+ * Free an HTTP response object.
+ *
+ * @memberof dc_http_response_t
+ * @param response HTTP response as returned by dc_get_http_response().
+ */
+void                    dc_http_response_unref        (const dc_http_response_t* response);
+
+
 /**
  * @class dc_lot_t
  *
@@ -5604,7 +5671,6 @@ void            dc_reactions_unref       (dc_reactions_t* reactions);
  */
 
 
-
 /**
  * @class dc_jsonrpc_instance_t
  *

deltachat-ffi/src/lib.rs

@@ -31,6 +31,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
+use deltachat::net::read_url_blob;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
@@ -4572,6 +4573,93 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // this may change once we start localizing string.
 }
 
+// dc_http_response_t
+
+pub type dc_http_response_t = net::HttpResponse;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_http_response(
+    context: *const dc_context_t,
+    url: *const libc::c_char,
+) -> *mut dc_http_response_t {
+    if context.is_null() || url.is_null() {
+        eprintln!("ignoring careless call to dc_get_http_response()");
+        return ptr::null_mut();
+    }
+
+    let context = &*context;
+    let url = to_string_lossy(url);
+    if let Ok(response) = block_on(read_url_blob(context, &url)).log_err(context, "read_url_blob") {
+        Box::into_raw(Box::new(response))
+    } else {
+        ptr::null_mut()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_http_response_get_mimetype(
+    response: *const dc_http_response_t,
+) -> *mut libc::c_char {
+    if response.is_null() {
+        eprintln!("ignoring careless call to dc_http_response_get_mimetype()");
+        return ptr::null_mut();
+    }
+
+    let response = &*response;
+    response.mimetype.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_http_response_get_encoding(
+    response: *const dc_http_response_t,
+) -> *mut libc::c_char {
+    if response.is_null() {
+        eprintln!("ignoring careless call to dc_http_response_get_encoding()");
+        return ptr::null_mut();
+    }
+
+    let response = &*response;
+    response.encoding.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_http_response_get_blob(
+    response: *const dc_http_response_t,
+) -> *mut libc::c_char {
+    if response.is_null() {
+        eprintln!("ignoring careless call to dc_http_response_get_blob()");
+        return ptr::null_mut();
+    }
+
+    let response = &*response;
+    let blob_len = response.blob.len();
+    let ptr = libc::malloc(blob_len);
+    libc::memcpy(ptr, response.blob.as_ptr() as *mut libc::c_void, blob_len);
+    ptr as *mut libc::c_char
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_http_response_get_size(
+    response: *const dc_http_response_t,
+) -> libc::size_t {
+    if response.is_null() {
+        eprintln!("ignoring careless call to dc_http_response_get_size()");
+        return 0;
+    }
+
+    let response = &*response;
+    response.blob.len()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_http_response_unref(response: *mut dc_http_response_t) {
+    if response.is_null() {
+        eprintln!("ignoring careless call to dc_http_response_unref()");
+        return;
+    }
+    drop(Box::from_raw(response));
+}
+
 // -- Accounts
 
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using

deltachat-jsonrpc/src/api/mod.rs

@@ -43,6 +43,7 @@ use types::account::Account;
 use types::chat::FullChat;
 use types::chat_list::ChatListEntry;
 use types::contact::ContactObject;
+use types::http::HttpResponse;
 use types::message::MessageData;
 use types::message::MessageObject;
 use types::provider_info::ProviderInfo;
@@ -1656,6 +1657,15 @@ impl CommandApi {
         Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
     }
 
+    /// Makes an HTTP GET request and returns a response.
+    ///
+    /// `url` is the HTTP or HTTPS URL.
+    async fn get_http_response(&self, account_id: u32, url: String) -> Result<HttpResponse> {
+        let ctx = self.get_context(account_id).await?;
+        let response = deltachat::net::read_url_blob(&ctx, &url).await?.into();
+        Ok(response)
+    }
+
     /// Forward messages to another chat.
     ///
     /// All types of messages can be forwarded,

deltachat-jsonrpc/src/api/types/http.rs

@@ -0,0 +1,29 @@
+use deltachat::net::HttpResponse as CoreHttpResponse;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+pub struct HttpResponse {
+    /// base64-encoded response body.
+    blob: String,
+
+    /// MIME type, e.g. "text/plain" or "text/html".
+    mimetype: Option<String>,
+
+    /// Encoding, e.g. "utf-8".
+    encoding: Option<String>,
+}
+
+impl From<CoreHttpResponse> for HttpResponse {
+    fn from(response: CoreHttpResponse) -> Self {
+        use base64::{engine::general_purpose, Engine as _};
+        let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob);
+        let mimetype = response.mimetype;
+        let encoding = response.encoding;
+        HttpResponse {
+            blob,
+            mimetype,
+            encoding,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -2,6 +2,7 @@ pub mod account;
 pub mod chat;
 pub mod chat_list;
 pub mod contact;
+pub mod http;
 pub mod location;
 pub mod message;
 pub mod provider_info;

release-date.in

@@ -1 +1 @@
-2023-04-18
+2023-04-20

src/configure.rs

@@ -11,7 +11,6 @@
 
 mod auto_mozilla;
 mod auto_outlook;
-mod read_url;
 mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};

src/configure/auto_mozilla.rs

@@ -6,10 +6,10 @@ use std::str::FromStr;
 
 use quick_xml::events::{BytesStart, Event};
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 #[derive(Debug)]

src/configure/auto_outlook.rs

@@ -7,9 +7,9 @@ use std::io::BufRead;
 
 use quick_xml::events::Event;
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 /// Result of parsing a single `Protocol` tag.

src/configure/read_url.rs

@@ -1,44 +0,0 @@
-use anyhow::{anyhow, format_err};
-
-use crate::context::Context;
-use crate::socks::Socks5Config;
-
-pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
-    match read_url_inner(context, url).await {
-        Ok(s) => {
-            info!(context, "Successfully read url {}", url);
-            Ok(s)
-        }
-        Err(e) => {
-            info!(context, "Can't read URL {}: {:#}", url, e);
-            Err(format_err!("Can't read URL {}: {:#}", url, e))
-        }
-    }
-}
-
-pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result<String> {
-    let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let client = crate::http::get_client(socks5_config)?;
-    let mut url = url.to_string();
-
-    // Follow up to 10 http-redirects
-    for _i in 0..10 {
-        let response = client.get(&url).send().await?;
-        if response.status().is_redirection() {
-            let headers = response.headers();
-            let header = headers
-                .get_all("location")
-                .iter()
-                .last()
-                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
-                .to_str()?;
-            info!(context, "Following redirect to {}", header);
-            url = header.to_string();
-            continue;
-        }
-
-        return response.text().await.map_err(Into::into);
-    }
-
-    Err(format_err!("Followed 10 redirections"))
-}

src/http.rs

@@ -1,24 +0,0 @@
-//! # HTTP module.
-
-use std::time::Duration;
-
-use anyhow::Result;
-
-use crate::socks::Socks5Config;
-
-const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
-
-pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
-    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
-    let builder = if let Some(socks5_config) = socks5_config {
-        let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
-        builder.proxy(proxy)
-    } else {
-        // Disable usage of "system" proxy configured via environment variables.
-        // It is enabled by default in `reqwest`, see
-        // <https://docs.rs/reqwest/0.11.14/reqwest/struct.ClientBuilder.html#method.no_proxy>
-        // for documentation.
-        builder.no_proxy()
-    };
-    Ok(builder.build()?)
-}

src/lib.rs

@@ -63,7 +63,6 @@ mod decrypt;
 pub mod download;
 mod e2ee;
 pub mod ephemeral;
-mod http;
 mod imap;
 pub mod imex;
 pub mod release;
@@ -100,7 +99,7 @@ mod dehtml;
 mod authres;
 mod color;
 pub mod html;
-mod net;
+pub mod net;
 pub mod plaintext;
 pub mod summary;
 

src/net.rs

@@ -1,4 +1,4 @@
-///! # Common network utilities.
+//! # Common network utilities.
 use std::net::{IpAddr, SocketAddr};
 use std::pin::Pin;
 use std::str::FromStr;
@@ -12,9 +12,12 @@ use tokio_io_timeout::TimeoutStream;
 use crate::context::Context;
 use crate::tools::time;
 
+pub(crate) mod http;
 pub(crate) mod session;
 pub(crate) mod tls;
 
+pub use http::{read_url, read_url_blob, Response as HttpResponse};
+
 async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result<TcpStream> {
     let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))
         .await

src/net/http.rs

@@ -0,0 +1,94 @@
+//! # HTTP module.
+
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use mime::Mime;
+
+use crate::context::Context;
+use crate::socks::Socks5Config;
+
+const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// HTTP(S) GET response.
+#[derive(Debug)]
+pub struct Response {
+    /// Response body.
+    pub blob: Vec<u8>,
+
+    /// MIME type exntracted from the `Content-Type` header, if any.
+    pub mimetype: Option<String>,
+
+    /// Encoding extracted from the `Content-Type` header, if any.
+    pub encoding: Option<String>,
+}
+
+/// Retrieves the text contents of URL using HTTP GET request.
+pub async fn read_url(context: &Context, url: &str) -> Result<String> {
+    Ok(read_url_inner(context, url).await?.text().await?)
+}
+
+/// Retrieves the binary contents of URL using HTTP GET request.
+pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
+    let response = read_url_inner(context, url).await?;
+    let content_type = response
+        .headers()
+        .get(reqwest::header::CONTENT_TYPE)
+        .and_then(|value| value.to_str().ok())
+        .and_then(|value| value.parse::<Mime>().ok());
+    let mimetype = content_type
+        .as_ref()
+        .map(|mime| mime.essence_str().to_string());
+    let encoding = content_type.as_ref().and_then(|mime| {
+        mime.get_param(mime::CHARSET)
+            .map(|charset| charset.as_str().to_string())
+    });
+    let blob: Vec<u8> = response.bytes().await?.into();
+    Ok(Response {
+        blob,
+        mimetype,
+        encoding,
+    })
+}
+
+async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Response> {
+    let socks5_config = Socks5Config::from_database(&context.sql).await?;
+    let client = get_client(socks5_config)?;
+    let mut url = url.to_string();
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let response = client.get(&url).send().await?;
+        if response.status().is_redirection() {
+            let headers = response.headers();
+            let header = headers
+                .get_all("location")
+                .iter()
+                .last()
+                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
+                .to_str()?;
+            info!(context, "Following redirect to {}", header);
+            url = header.to_string();
+            continue;
+        }
+
+        return Ok(response);
+    }
+
+    Err(anyhow!("Followed 10 redirections"))
+}
+
+pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
+    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
+    let builder = if let Some(socks5_config) = socks5_config {
+        let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
+        builder.proxy(proxy)
+    } else {
+        // Disable usage of "system" proxy configured via environment variables.
+        // It is enabled by default in `reqwest`, see
+        // <https://docs.rs/reqwest/0.11.14/reqwest/struct.ClientBuilder.html#method.no_proxy>
+        // for documentation.
+        builder.no_proxy()
+    };
+    Ok(builder.build()?)
+}

src/oauth2.rs

@@ -160,7 +160,7 @@ pub(crate) async fn get_oauth2_access_token(
 
         // ... and POST
         let socks5_config = Socks5Config::from_database(&context.sql).await?;
-        let client = crate::http::get_client(socks5_config)?;
+        let client = crate::net::http::get_client(socks5_config)?;
 
         let response: Response = match client.post(post_url).form(&post_param).send().await {
             Ok(resp) => match resp.json().await {
@@ -291,7 +291,7 @@ impl Oauth2 {
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
         let socks5_config = Socks5Config::from_database(&context.sql).await.ok()?;
-        let client = match crate::http::get_client(socks5_config) {
+        let client = match crate::net::http::get_client(socks5_config) {
             Ok(cl) => cl,
             Err(err) => {
                 warn!(context, "failed to get HTTP client: {}", err);

src/qr.rs

@@ -526,7 +526,7 @@ struct CreateAccountErrorResponse {
 async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
     let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let response = crate::http::get_client(socks5_config)?
+    let response = crate::net::http::get_client(socks5_config)?
         .post(url_str)
         .send()
         .await?;