diff --git a/CHANGELOG.md b/CHANGELOG.md index b06c8e826..dc145bb4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,12 @@ - Fix python bindings README documentation on installing the bindings from source. - Remove confusing log line "ignoring unsolicited response Recent(…)". #3934 +## [1.112.8] - 2023-04-20 + +### Changes +- Add `get_http_response` JSON-RPC API. +- Add C API to get HTTP responses. + ## [1.112.7] - 2023-04-17 ### Fixes diff --git a/Cargo.lock b/Cargo.lock index 6bc23f2f8..e462de808 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,6 +1175,7 @@ dependencies = [ "libc", "log", "mailparse", + "mime", "num-derive", "num-traits", "num_cpus", @@ -2806,9 +2807,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" diff --git a/Cargo.toml b/Cargo.toml index 60ba547a2..4d3d42494 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" mailparse = "0.14" +mime = "0.3.17" num_cpus = "1.15" num-derive = "0.3" num-traits = "0.2" diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index c76a5891a..3125344a4 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -25,6 +25,7 @@ typedef struct _dc_event dc_event_t; typedef struct _dc_event_emitter dc_event_emitter_t; typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t; typedef struct _dc_backup_provider dc_backup_provider_t; +typedef struct _dc_http_response dc_http_response_t; // Alias for backwards compatibility, use dc_event_emitter_t instead. typedef struct _dc_event_emitter dc_accounts_event_emitter_t; @@ -5127,6 +5128,72 @@ int dc_provider_get_status (const dc_provider_t* prov void dc_provider_unref (dc_provider_t* provider); +/** + * Return an HTTP(S) GET response. + * This function can be used to download remote content for HTML emails. + * + * @memberof dc_context_t + * @param context The context object to take proxy settings from. + * @param url HTTP or HTTPS URL. + * @return The response must be released using dc_http_response_unref() after usage. + * NULL is returned on errors. + */ +dc_http_response_t* dc_get_http_response (const dc_context_t* context, const char* url); + + +/** + * @class dc_http_response_t + * + * An object containing an HTTP(S) GET response. + * Created by dc_get_http_response(). + */ + + +/** + * Returns HTTP response MIME type as a string, e.g. "text/plain" or "text/html". + * + * @memberof dc_http_response_t + * @param response HTTP response as returned by dc_get_http_response(). + * @return The string which must be released using dc_str_unref() after usage. May be NULL. + */ +char* dc_http_response_get_mimetype (const dc_http_response_t* response); + +/** + * Returns HTTP response encoding, e.g. "utf-8". + * + * @memberof dc_http_response_t + * @param response HTTP response as returned by dc_get_http_response(). + * @return The string which must be released using dc_str_unref() after usage. May be NULL. + */ +char* dc_http_response_get_encoding (const dc_http_response_t* response); + +/** + * Returns HTTP response contents. + * + * @memberof dc_http_response_t + * @param response HTTP response as returned by dc_get_http_response(). + * @return The blob which must be released using dc_str_unref() after usage. NULL is never returned. + */ +uint8_t* dc_http_response_get_blob (const dc_http_response_t* response); + +/** + * Returns HTTP response content size. + * + * @memberof dc_http_response_t + * @param response HTTP response as returned by dc_get_http_response(). + * @return The blob size. + */ +size_t dc_http_response_get_size (const dc_http_response_t* response); + +/** + * Free an HTTP response object. + * + * @memberof dc_http_response_t + * @param response HTTP response as returned by dc_get_http_response(). + */ +void dc_http_response_unref (const dc_http_response_t* response); + + /** * @class dc_lot_t * @@ -5604,7 +5671,6 @@ void dc_reactions_unref (dc_reactions_t* reactions); */ - /** * @class dc_jsonrpc_instance_t * diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 6872c6d38..4b1f859e9 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -31,6 +31,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer; use deltachat::imex::BackupProvider; use deltachat::key::DcKey; use deltachat::message::MsgId; +use deltachat::net::read_url_blob; use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg}; use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions}; use deltachat::stock_str::StockMessage; @@ -4572,6 +4573,93 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) { // this may change once we start localizing string. } +// dc_http_response_t + +pub type dc_http_response_t = net::HttpResponse; + +#[no_mangle] +pub unsafe extern "C" fn dc_get_http_response( + context: *const dc_context_t, + url: *const libc::c_char, +) -> *mut dc_http_response_t { + if context.is_null() || url.is_null() { + eprintln!("ignoring careless call to dc_get_http_response()"); + return ptr::null_mut(); + } + + let context = &*context; + let url = to_string_lossy(url); + if let Ok(response) = block_on(read_url_blob(context, &url)).log_err(context, "read_url_blob") { + Box::into_raw(Box::new(response)) + } else { + ptr::null_mut() + } +} + +#[no_mangle] +pub unsafe extern "C" fn dc_http_response_get_mimetype( + response: *const dc_http_response_t, +) -> *mut libc::c_char { + if response.is_null() { + eprintln!("ignoring careless call to dc_http_response_get_mimetype()"); + return ptr::null_mut(); + } + + let response = &*response; + response.mimetype.strdup() +} + +#[no_mangle] +pub unsafe extern "C" fn dc_http_response_get_encoding( + response: *const dc_http_response_t, +) -> *mut libc::c_char { + if response.is_null() { + eprintln!("ignoring careless call to dc_http_response_get_encoding()"); + return ptr::null_mut(); + } + + let response = &*response; + response.encoding.strdup() +} + +#[no_mangle] +pub unsafe extern "C" fn dc_http_response_get_blob( + response: *const dc_http_response_t, +) -> *mut libc::c_char { + if response.is_null() { + eprintln!("ignoring careless call to dc_http_response_get_blob()"); + return ptr::null_mut(); + } + + let response = &*response; + let blob_len = response.blob.len(); + let ptr = libc::malloc(blob_len); + libc::memcpy(ptr, response.blob.as_ptr() as *mut libc::c_void, blob_len); + ptr as *mut libc::c_char +} + +#[no_mangle] +pub unsafe extern "C" fn dc_http_response_get_size( + response: *const dc_http_response_t, +) -> libc::size_t { + if response.is_null() { + eprintln!("ignoring careless call to dc_http_response_get_size()"); + return 0; + } + + let response = &*response; + response.blob.len() +} + +#[no_mangle] +pub unsafe extern "C" fn dc_http_response_unref(response: *mut dc_http_response_t) { + if response.is_null() { + eprintln!("ignoring careless call to dc_http_response_unref()"); + return; + } + drop(Box::from_raw(response)); +} + // -- Accounts /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 5418862c9..41dff098b 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -43,6 +43,7 @@ use types::account::Account; use types::chat::FullChat; use types::chat_list::ChatListEntry; use types::contact::ContactObject; +use types::http::HttpResponse; use types::message::MessageData; use types::message::MessageObject; use types::provider_info::ProviderInfo; @@ -1656,6 +1657,15 @@ impl CommandApi { Ok(general_purpose::STANDARD_NO_PAD.encode(blob)) } + /// Makes an HTTP GET request and returns a response. + /// + /// `url` is the HTTP or HTTPS URL. + async fn get_http_response(&self, account_id: u32, url: String) -> Result { + let ctx = self.get_context(account_id).await?; + let response = deltachat::net::read_url_blob(&ctx, &url).await?.into(); + Ok(response) + } + /// Forward messages to another chat. /// /// All types of messages can be forwarded, diff --git a/deltachat-jsonrpc/src/api/types/http.rs b/deltachat-jsonrpc/src/api/types/http.rs new file mode 100644 index 000000000..3b9d29509 --- /dev/null +++ b/deltachat-jsonrpc/src/api/types/http.rs @@ -0,0 +1,29 @@ +use deltachat::net::HttpResponse as CoreHttpResponse; +use serde::Serialize; +use typescript_type_def::TypeDef; + +#[derive(Serialize, TypeDef)] +pub struct HttpResponse { + /// base64-encoded response body. + blob: String, + + /// MIME type, e.g. "text/plain" or "text/html". + mimetype: Option, + + /// Encoding, e.g. "utf-8". + encoding: Option, +} + +impl From for HttpResponse { + fn from(response: CoreHttpResponse) -> Self { + use base64::{engine::general_purpose, Engine as _}; + let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob); + let mimetype = response.mimetype; + let encoding = response.encoding; + HttpResponse { + blob, + mimetype, + encoding, + } + } +} diff --git a/deltachat-jsonrpc/src/api/types/mod.rs b/deltachat-jsonrpc/src/api/types/mod.rs index 2d783990e..ace49f14e 100644 --- a/deltachat-jsonrpc/src/api/types/mod.rs +++ b/deltachat-jsonrpc/src/api/types/mod.rs @@ -2,6 +2,7 @@ pub mod account; pub mod chat; pub mod chat_list; pub mod contact; +pub mod http; pub mod location; pub mod message; pub mod provider_info; diff --git a/release-date.in b/release-date.in index 98684ad68..faca32070 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2023-04-18 \ No newline at end of file +2023-04-20 diff --git a/src/configure.rs b/src/configure.rs index 4f8f1781a..7acd6e4d2 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -11,7 +11,6 @@ mod auto_mozilla; mod auto_outlook; -mod read_url; mod server_params; use anyhow::{bail, ensure, Context as _, Result}; diff --git a/src/configure/auto_mozilla.rs b/src/configure/auto_mozilla.rs index eb93b03e9..505691814 100644 --- a/src/configure/auto_mozilla.rs +++ b/src/configure/auto_mozilla.rs @@ -6,10 +6,10 @@ use std::str::FromStr; use quick_xml::events::{BytesStart, Event}; -use super::read_url::read_url; use super::{Error, ServerParams}; use crate::context::Context; use crate::login_param::LoginParam; +use crate::net::read_url; use crate::provider::{Protocol, Socket}; #[derive(Debug)] diff --git a/src/configure/auto_outlook.rs b/src/configure/auto_outlook.rs index 8d42fd353..c1cfbe416 100644 --- a/src/configure/auto_outlook.rs +++ b/src/configure/auto_outlook.rs @@ -7,9 +7,9 @@ use std::io::BufRead; use quick_xml::events::Event; -use super::read_url::read_url; use super::{Error, ServerParams}; use crate::context::Context; +use crate::net::read_url; use crate::provider::{Protocol, Socket}; /// Result of parsing a single `Protocol` tag. diff --git a/src/configure/read_url.rs b/src/configure/read_url.rs deleted file mode 100644 index d164a7007..000000000 --- a/src/configure/read_url.rs +++ /dev/null @@ -1,44 +0,0 @@ -use anyhow::{anyhow, format_err}; - -use crate::context::Context; -use crate::socks::Socks5Config; - -pub async fn read_url(context: &Context, url: &str) -> anyhow::Result { - match read_url_inner(context, url).await { - Ok(s) => { - info!(context, "Successfully read url {}", url); - Ok(s) - } - Err(e) => { - info!(context, "Can't read URL {}: {:#}", url, e); - Err(format_err!("Can't read URL {}: {:#}", url, e)) - } - } -} - -pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result { - let socks5_config = Socks5Config::from_database(&context.sql).await?; - let client = crate::http::get_client(socks5_config)?; - let mut url = url.to_string(); - - // Follow up to 10 http-redirects - for _i in 0..10 { - let response = client.get(&url).send().await?; - if response.status().is_redirection() { - let headers = response.headers(); - let header = headers - .get_all("location") - .iter() - .last() - .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))? - .to_str()?; - info!(context, "Following redirect to {}", header); - url = header.to_string(); - continue; - } - - return response.text().await.map_err(Into::into); - } - - Err(format_err!("Followed 10 redirections")) -} diff --git a/src/http.rs b/src/http.rs deleted file mode 100644 index 8eed8b55e..000000000 --- a/src/http.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! # HTTP module. - -use std::time::Duration; - -use anyhow::Result; - -use crate::socks::Socks5Config; - -const HTTP_TIMEOUT: Duration = Duration::from_secs(30); - -pub(crate) fn get_client(socks5_config: Option) -> Result { - let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT); - let builder = if let Some(socks5_config) = socks5_config { - let proxy = reqwest::Proxy::all(socks5_config.to_url())?; - builder.proxy(proxy) - } else { - // Disable usage of "system" proxy configured via environment variables. - // It is enabled by default in `reqwest`, see - // - // for documentation. - builder.no_proxy() - }; - Ok(builder.build()?) -} diff --git a/src/lib.rs b/src/lib.rs index 38c8d0d47..28778eb19 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -63,7 +63,6 @@ mod decrypt; pub mod download; mod e2ee; pub mod ephemeral; -mod http; mod imap; pub mod imex; pub mod release; @@ -100,7 +99,7 @@ mod dehtml; mod authres; mod color; pub mod html; -mod net; +pub mod net; pub mod plaintext; pub mod summary; diff --git a/src/net.rs b/src/net.rs index c6cc6f539..c4b0f3686 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,4 +1,4 @@ -///! # Common network utilities. +//! # Common network utilities. use std::net::{IpAddr, SocketAddr}; use std::pin::Pin; use std::str::FromStr; @@ -12,9 +12,12 @@ use tokio_io_timeout::TimeoutStream; use crate::context::Context; use crate::tools::time; +pub(crate) mod http; pub(crate) mod session; pub(crate) mod tls; +pub use http::{read_url, read_url_blob, Response as HttpResponse}; + async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) .await diff --git a/src/net/http.rs b/src/net/http.rs new file mode 100644 index 000000000..cbee57959 --- /dev/null +++ b/src/net/http.rs @@ -0,0 +1,94 @@ +//! # HTTP module. + +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use mime::Mime; + +use crate::context::Context; +use crate::socks::Socks5Config; + +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +/// HTTP(S) GET response. +#[derive(Debug)] +pub struct Response { + /// Response body. + pub blob: Vec, + + /// MIME type exntracted from the `Content-Type` header, if any. + pub mimetype: Option, + + /// Encoding extracted from the `Content-Type` header, if any. + pub encoding: Option, +} + +/// Retrieves the text contents of URL using HTTP GET request. +pub async fn read_url(context: &Context, url: &str) -> Result { + Ok(read_url_inner(context, url).await?.text().await?) +} + +/// Retrieves the binary contents of URL using HTTP GET request. +pub async fn read_url_blob(context: &Context, url: &str) -> Result { + let response = read_url_inner(context, url).await?; + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + let mimetype = content_type + .as_ref() + .map(|mime| mime.essence_str().to_string()); + let encoding = content_type.as_ref().and_then(|mime| { + mime.get_param(mime::CHARSET) + .map(|charset| charset.as_str().to_string()) + }); + let blob: Vec = response.bytes().await?.into(); + Ok(Response { + blob, + mimetype, + encoding, + }) +} + +async fn read_url_inner(context: &Context, url: &str) -> Result { + let socks5_config = Socks5Config::from_database(&context.sql).await?; + let client = get_client(socks5_config)?; + let mut url = url.to_string(); + + // Follow up to 10 http-redirects + for _i in 0..10 { + let response = client.get(&url).send().await?; + if response.status().is_redirection() { + let headers = response.headers(); + let header = headers + .get_all("location") + .iter() + .last() + .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))? + .to_str()?; + info!(context, "Following redirect to {}", header); + url = header.to_string(); + continue; + } + + return Ok(response); + } + + Err(anyhow!("Followed 10 redirections")) +} + +pub(crate) fn get_client(socks5_config: Option) -> Result { + let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT); + let builder = if let Some(socks5_config) = socks5_config { + let proxy = reqwest::Proxy::all(socks5_config.to_url())?; + builder.proxy(proxy) + } else { + // Disable usage of "system" proxy configured via environment variables. + // It is enabled by default in `reqwest`, see + // + // for documentation. + builder.no_proxy() + }; + Ok(builder.build()?) +} diff --git a/src/oauth2.rs b/src/oauth2.rs index 2fe8795b4..f3e5dfe5e 100644 --- a/src/oauth2.rs +++ b/src/oauth2.rs @@ -160,7 +160,7 @@ pub(crate) async fn get_oauth2_access_token( // ... and POST let socks5_config = Socks5Config::from_database(&context.sql).await?; - let client = crate::http::get_client(socks5_config)?; + let client = crate::net::http::get_client(socks5_config)?; let response: Response = match client.post(post_url).form(&post_param).send().await { Ok(resp) => match resp.json().await { @@ -291,7 +291,7 @@ impl Oauth2 { // "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg" // } let socks5_config = Socks5Config::from_database(&context.sql).await.ok()?; - let client = match crate::http::get_client(socks5_config) { + let client = match crate::net::http::get_client(socks5_config) { Ok(cl) => cl, Err(err) => { warn!(context, "failed to get HTTP client: {}", err); diff --git a/src/qr.rs b/src/qr.rs index d13366860..e11b74f56 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -526,7 +526,7 @@ struct CreateAccountErrorResponse { async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> { let url_str = &qr[DCACCOUNT_SCHEME.len()..]; let socks5_config = Socks5Config::from_database(&context.sql).await?; - let response = crate::http::get_client(socks5_config)? + let response = crate::net::http::get_client(socks5_config)? .post(url_str) .send() .await?;