Compare commits

...

6 Commits

Author SHA1 Message Date
holger krekel
0dc87e9a0f remove one attr from mimeparser 2019-12-07 22:41:43 +01:00
holger krekel
c011a8cfef run lint without installing the package 2019-12-07 21:44:34 +01:00
holger krekel
2d7d79fc44 allow to use a different buildhost :) 2019-12-07 21:44:34 +01:00
holger krekel
119d4fa689 - test and fix receiving text/html attachment in multipart/mixed situations
They are now preserved as attachment, instead of diving into parsing-html
  and simplifying.

- adapt mime-debugging
2019-12-07 21:44:34 +01:00
holger krekel
b7a9f73329 passes test but needs cleanup 2019-12-07 21:44:34 +01:00
holger krekel
a52a3e0d24 try fix incoming text/html in multipart/mixed 2019-12-07 21:44:34 +01:00
10 changed files with 246 additions and 130 deletions

View File

@@ -2,7 +2,7 @@
export BRANCH=${CIRCLE_BRANCH:?branch to build}
export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
export SSHTARGET=ci@b1.delta.chat
export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
# we construct the BUILDDIR such that we can easily share the
# CARGO_TARGET_DIR between runs ("..")

View File

@@ -2,7 +2,7 @@
export BRANCH=${CIRCLE_BRANCH:?branch to build}
export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
export SSHTARGET=ci@b1.delta.chat
export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
# we construct the BUILDDIR such that we can easily share the
# CARGO_TARGET_DIR between runs ("..")

View File

@@ -477,6 +477,30 @@ class TestOnlineAccount:
assert msg2.filename.endswith("html.zip")
assert msg.filename != msg2.filename
def test_send_file_html_attachment(self, tmpdir, acfactory, lp):
ac1, ac2 = acfactory.get_two_online_accounts()
chat = self.get_chat(ac1, ac2)
basename = "test.html"
content = "<html><body>text</body>data"
p = os.path.join(tmpdir.strpath, basename)
with open(p, "w") as f:
# write wrong html to see if core tries to parse it
# (it shouldn't as it's a file attachment)
f.write(content)
lp.sec("ac1: prepare and send attachment + text to ac2")
chat.send_file(p, mime_type="text/html")
lp.sec("ac2: receive message")
ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
msg = ac2.get_message_by_id(ev[2])
assert open(msg.filename).read() == content
assert msg.filename.endswith(basename)
def test_mvbox_sentbox_threads(self, acfactory, lp):
lp.sec("ac1: start with mvbox thread")
ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)

View File

@@ -32,7 +32,7 @@ commands =
[testenv:lint]
skipsdist = True
usedevelop = True
skip_install = True
deps =
flake8
# pygments required by rst-lint

View File

@@ -47,7 +47,7 @@ pub fn dc_receive_imf(
server_uid,
);
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" {
info!(context, "dc_receive_imf: incoming message mime-body:");
println!("{}", String::from_utf8_lossy(imf_raw));
}
@@ -337,7 +337,7 @@ fn add_parts(
}
// 1 or 0 for yes/no
msgrmsg = mime_parser.is_send_by_messenger as _;
msgrmsg = mime_parser.has_chat_version() as _;
if msgrmsg == 0 && 0 != dc_is_reply_to_messenger_message(context, mime_parser) {
// 2=no, but is reply to messenger message
msgrmsg = 2;
@@ -1112,7 +1112,7 @@ fn create_or_lookup_group(
// the only critical situation is if the user hits "Reply" instead
// of "Reply all" in a non-messenger-client */
if to_ids_cnt == 1
&& !mime_parser.is_send_by_messenger
&& !mime_parser.has_chat_version()
&& chat::get_chat_contact_cnt(context, chat_id) > 3
{
// to_ids_cnt==1 may be "From: A, To: B, SELF" as SELF is not counted in to_ids_cnt.

View File

@@ -247,13 +247,13 @@ fn decrypt_if_autocrypt_message<'a>(
// Errors are returned for failures related to decryption of AC-messages.
let encrypted_data_part = match wrapmime::get_autocrypt_mime(mail) {
Err(err) => {
// not a proper autocrypt message, abort and ignore
info!(context, "Not an autocrypt message: {:?}", err);
Err(_) => {
// not an autocrypt mime message, abort and ignore
return Ok(None);
}
Ok(res) => res,
};
info!(context, "Detected Autocrypt-mime message");
decrypt_part(
context,

View File

@@ -525,6 +525,12 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
outer_message = outer_message.header(header);
}
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(self.context, "mimefactory: outgoing message mime:");
let raw_message = message.clone().build().as_string();
println!("{}", raw_message);
}
let encrypted =
encrypt_helper.encrypt(self.context, min_verified, message, &peerstates)?;

View File

@@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet};
use deltachat_derive::{FromSql, ToSql};
use lettre_email::mime::{self, Mime};
use mailparse::MailHeaderMap;
use mailparse::{DispositionType, MailHeaderMap};
use crate::aheader::Aheader;
use crate::blob::BlobObject;
@@ -22,6 +22,7 @@ use crate::param::*;
use crate::peerstate::Peerstate;
use crate::securejoin::handle_degrade_event;
use crate::stock::StockMessage;
use crate::{bail, ensure};
#[derive(Debug)]
pub struct MimeParser<'a> {
@@ -29,7 +30,6 @@ pub struct MimeParser<'a> {
pub parts: Vec<Part>,
pub header: HashMap<String, String>,
pub subject: Option<String>,
pub is_send_by_messenger: bool,
pub decrypting_failed: bool,
pub encrypted: bool,
pub signatures: HashSet<String>,
@@ -74,7 +74,6 @@ impl<'a> MimeParser<'a> {
parts: Vec::new(),
header: Default::default(),
subject: None,
is_send_by_messenger: false,
decrypting_failed: false,
encrypted: false,
signatures: Default::default(),
@@ -109,6 +108,10 @@ impl<'a> MimeParser<'a> {
if let Some(raw) = raw {
mail_raw = raw;
let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(context, "decrypted message mime-body:");
println!("{}", String::from_utf8_lossy(&mail_raw));
}
// Handle any gossip headers if the mail was encrypted. See section
// "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
@@ -148,10 +151,6 @@ impl<'a> MimeParser<'a> {
self.subject = Some(field.clone());
}
if self.lookup_field("Chat-Version").is_some() {
self.is_send_by_messenger = true
}
if self.lookup_field("Autocrypt-Setup-Message").is_some() {
let has_setup_file = self.parts.iter().any(|p| {
p.mimetype.is_some() && p.mimetype.as_ref().unwrap().as_ref() == MIME_AC_SETUP_FILE
@@ -198,7 +197,7 @@ impl<'a> MimeParser<'a> {
}
}
if self.is_send_by_messenger && self.parts.len() == 2 {
if self.has_chat_version() && self.parts.len() == 2 {
let need_drop = {
let textpart = &self.parts[0];
let filepart = &self.parts[1];
@@ -232,7 +231,7 @@ impl<'a> MimeParser<'a> {
let colon = subject.find(':');
if colon == Some(2)
|| colon == Some(3)
|| self.is_send_by_messenger
|| self.has_chat_version()
|| subject.contains("Chat:")
{
prepend_subject = 0i32
@@ -318,7 +317,7 @@ impl<'a> MimeParser<'a> {
part.typ = Viewtype::Text;
if let Some(ref subject) = self.subject {
if !self.is_send_by_messenger {
if !self.has_chat_version() {
part.msg = subject.to_string();
}
}
@@ -337,6 +336,10 @@ impl<'a> MimeParser<'a> {
self.parts.iter_mut().rev().find(|part| !part.is_meta)
}
pub(crate) fn has_chat_version(&self) -> bool {
self.header.contains_key("chat-version")
}
pub fn lookup_field(&self, field_name: &str) -> Option<&String> {
self.header.get(&field_name.to_lowercase())
}
@@ -487,41 +490,10 @@ impl<'a> MimeParser<'a> {
}
}
_ => {
// Add all parts (in fact,
// AddSinglePartIfKnown() later check if the parts are really supported)
// HACK: the following lines are a hack for clients who use
// multipart/mixed instead of multipart/alternative for
// combined text/html messages (eg. Stock Android "Mail" does so).
// So, if we detect such a message below, we skip the Html
// part. However, not sure, if there are useful situations to use
// plain+html in multipart/mixed - if so, we should disable the hack.
let mut skip_part = -1;
let mut html_part = -1;
let mut plain_cnt = 0;
let mut html_cnt = 0;
for (i, cur_data) in mail.subparts.iter().enumerate() {
match get_mime_type(cur_data)?.0.type_() {
mime::TEXT => {
plain_cnt += 1;
}
mime::HTML => {
html_part = i as isize;
html_cnt += 1;
}
_ => {}
}
}
if plain_cnt == 1 && html_cnt == 1 {
warn!(
self.context,
"HACK: multipart/mixed message found with Plain and HTML, we\'ll skip the HTML part as this seems to be unwanted."
);
skip_part = html_part;
}
for (i, cur_data) in mail.subparts.iter().enumerate() {
if i as isize != skip_part && self.parse_mime_recursive(cur_data)? {
// Add all parts (in fact, AddSinglePartIfKnown() later check if
// the parts are really supported)
for cur_data in mail.subparts.iter() {
if self.parse_mime_recursive(cur_data)? {
any_part_added = true;
}
}
@@ -536,84 +508,60 @@ impl<'a> MimeParser<'a> {
let (mime_type, msg_type) = get_mime_type(mail)?;
let raw_mime = mail.ctype.mimetype.to_lowercase();
let filename = get_attachment_filename(mail);
info!(
self.context,
"add_single_part_if_known {:?} {:?} {:?}", mime_type, msg_type, filename
);
let old_part_count = self.parts.len();
// regard `Content-Transfer-Encoding:`
match mime_type.type_() {
mime::TEXT | mime::HTML => {
let decoded_data = match mail.get_body() {
Ok(decoded_data) => decoded_data,
Err(err) => {
warn!(self.context, "Invalid body parsed {:?}", err);
// Note that it's not always an error - might be no data
return Ok(false);
}
};
// check header directly as is_send_by_messenger is not yet set up
let is_msgrmsg = self.lookup_field("Chat-Version").is_some();
let mut simplifier = Simplify::new();
let simplified_txt = if decoded_data.is_empty() {
"".into()
} else {
let is_html = mime_type == mime::TEXT_HTML;
simplifier.simplify(&decoded_data, is_html, is_msgrmsg)
};
if !simplified_txt.is_empty() {
let mut part = Part::default();
part.typ = Viewtype::Text;
part.mimetype = Some(mime_type);
part.msg = simplified_txt;
part.msg_raw = Some(decoded_data);
self.do_add_single_part(part);
if let Ok(filename) = filename {
self.do_add_single_file_part(
msg_type,
mime_type,
&raw_mime,
&mail.get_body_raw()?,
&filename,
);
} else {
match mime_type.type_() {
mime::IMAGE | mime::AUDIO | mime::VIDEO | mime::APPLICATION => {
bail!("missing attachment");
}
mime::TEXT | mime::HTML => {
let decoded_data = match mail.get_body() {
Ok(decoded_data) => decoded_data,
Err(err) => {
warn!(self.context, "Invalid body parsed {:?}", err);
// Note that it's not always an error - might be no data
return Ok(false);
}
};
if simplifier.is_forwarded {
self.is_forwarded = true;
}
}
mime::IMAGE | mime::AUDIO | mime::VIDEO | mime::APPLICATION => {
// try to get file name from
// `Content-Disposition: ... filename*=...`
// or `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
// or `Content-Disposition: ... filename=...`
let ct = mail.get_content_disposition()?;
let mut desired_filename = ct
.params
.iter()
.filter(|(key, _value)| key.starts_with("filename"))
.fold(String::new(), |mut acc, (_key, value)| {
acc += value;
acc
});
if desired_filename.is_empty() {
if let Some(param) = ct.params.get("name") {
// might be a wrongly encoded filename
desired_filename = param.to_string();
}
}
// if there is still no filename, guess one
if desired_filename.is_empty() {
if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
desired_filename = format!("file.{}", subtype,);
let mut simplifier = Simplify::new();
let simplified_txt = if decoded_data.is_empty() {
"".into()
} else {
return Ok(false);
let is_html = mime_type == mime::TEXT_HTML;
simplifier.simplify(&decoded_data, is_html, self.has_chat_version())
};
if !simplified_txt.is_empty() {
let mut part = Part::default();
part.typ = Viewtype::Text;
part.mimetype = Some(mime_type);
part.msg = simplified_txt;
part.msg_raw = Some(decoded_data);
self.do_add_single_part(part);
}
if simplifier.is_forwarded {
self.is_forwarded = true;
}
}
self.do_add_single_file_part(
msg_type,
mime_type,
&raw_mime,
&mail.get_body_raw()?,
&desired_filename,
);
_ => {}
}
_ => {}
}
// add object? (we do not add all objects, eg. signatures etc. are ignored)
@@ -662,6 +610,7 @@ impl<'a> MimeParser<'a> {
return;
}
};
info!(self.context, "added blobfile: {:?}", blob.as_name());
/* create and register Mime part referencing the new Blob object */
let mut part = Part::default();
@@ -812,11 +761,11 @@ impl<'a> MimeParser<'a> {
mdn_consumed = true;
}
if self.is_send_by_messenger || mdn_consumed {
if self.has_chat_version() || mdn_consumed {
let mut param = Params::new();
param.set(Param::ServerFolder, server_folder.as_ref());
param.set_int(Param::ServerUid, server_uid as i32);
if self.is_send_by_messenger && self.context.get_config_bool(Config::MvboxMove) {
if self.has_chat_version() && self.context.get_config_bool(Config::MvboxMove) {
param.set_int(Param::AlsoMove, 1);
}
job_add(self.context, Action::MarkseenMdnOnImap, 0, param, 0);
@@ -913,12 +862,13 @@ pub struct Part {
pub param: Params,
}
/// return mimetype and viewtype for a parsed mail
fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> {
let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
let viewtype = match mimetype.type_() {
mime::TEXT => {
if !mailmime_is_attachment_disposition(mail) {
if !is_attachment_disposition(mail) {
match mimetype.subtype() {
mime::PLAIN | mime::HTML => Viewtype::Text,
_ => Viewtype::File,
@@ -952,14 +902,62 @@ fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> {
Ok((mimetype, viewtype))
}
fn mailmime_is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool {
if let Some(ct) = mail.ctype.params.get("Content-Disposition") {
return ct.to_lowercase().starts_with("attachment");
fn is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool {
if let Ok(ct) = mail.get_content_disposition() {
return ct.disposition == DispositionType::Attachment
&& ct
.params
.iter()
.any(|(key, _value)| key.starts_with("filename"));
}
false
}
fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<String> {
// try to get file name from
// `Content-Disposition: ... filename*=...`
// or `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
// or `Content-Disposition: ... filename=...`
let ct = mail.get_content_disposition()?;
ensure!(
ct.disposition == DispositionType::Attachment,
"disposition not an attachment: {:?}",
ct.disposition
);
let mut desired_filename = ct
.params
.iter()
.filter(|(key, _value)| key.starts_with("filename"))
.fold(String::new(), |mut acc, (_key, value)| {
acc += value;
acc
});
println!("get_attachment_filename1: {:?}", desired_filename);
if desired_filename.is_empty() {
if let Some(param) = ct.params.get("name") {
// might be a wrongly encoded filename
desired_filename = param.to_string();
}
}
println!("get_attachment_filename2: {:?}", desired_filename);
// if there is still no filename, guess one
if desired_filename.is_empty() {
if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
desired_filename = format!("file.{}", subtype,);
} else {
bail!("could not determine filename: {:?}", ct.disposition);
}
}
println!("get_attachment_filename3: {:?}", desired_filename);
Ok(desired_filename)
}
// returned addresses are normalized.
fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> HashSet<String> {
let mut recipients: HashSet<String> = Default::default();
@@ -1064,6 +1062,29 @@ mod tests {
assert_eq!(recipients.len(), 2);
}
#[test]
fn test_is_attachment() {
let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
let mail = mailparse::parse_mail(raw).unwrap();
assert!(!is_attachment_disposition(&mail));
let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
let mail = mailparse::parse_mail(raw).unwrap();
assert!(!is_attachment_disposition(&mail));
assert!(!is_attachment_disposition(&mail.subparts[0]));
assert!(is_attachment_disposition(&mail.subparts[1]));
}
#[test]
fn test_get_attachment_filename() {
let raw = include_bytes!("../test-data/message/html_attach.eml");
let mail = mailparse::parse_mail(raw).unwrap();
assert!(get_attachment_filename(&mail).is_err());
assert!(get_attachment_filename(&mail.subparts[0]).is_err());
let filename = get_attachment_filename(&mail.subparts[1]).unwrap();
assert_eq!(filename, "test.html")
}
#[test]
fn test_mailparse_content_type() {
let ctype =

View File

@@ -0,0 +1,25 @@
Chat-Disposition-Notification-To: tmp_6272287793210918@testrun.org
Subject: =?utf-8?q?Chat=3A_File_=E2=80=93_test=2Ehtml?=
Message-ID: Mr.XA6y3og8-az.WGbH9_dNcQx@testrun.org
Date: Sat, 07 Dec 2019 19:00:27 +0000
X-Mailer: Delta Chat Core 1.0.0-beta.12/DcFFI
Chat-Version: 1.0
To: <tmp_5890965001269692@testrun.org>
From: "=?utf-8?q??=" <tmp_6272287793210918@testrun.org>
Content-Type: multipart/mixed; boundary="mwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z"
--mwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z
Content-Type: text/plain; charset=utf-8
--
Sent with my Delta Chat Messenger: https://delta.chat
--mwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z
Content-Type: text/html
Content-Disposition: attachment; filename="test.html"
Content-Transfer-Encoding: base64
PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
--mwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z--

View File

@@ -0,0 +1,40 @@
From holger@merlinux.eu Sat Dec 7 11:53:58 2019
Return-Path: <holger@merlinux.eu>
X-Original-To: holger+test@merlinux.eu
Delivered-To: holger+test@merlinux.eu
Received: from [127.0.0.1] (localhost [127.0.0.1])
(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))
(No client certificate requested)
by mail.merlinux.eu (Postfix) with ESMTPSA id 61825100531;
Sat, 7 Dec 2019 10:53:58 +0000 (UTC)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=merlinux.eu;
s=default; t=1575716038;
bh=xhaPssQzVOHRafcciTQqDnZ1Zi4GMwsmg9pHLH3i8P8=;
h=Date:From:To:Subject;
b=U4HxGDZ8RwLwRPFtIvRsb+x5BiyICnbbY2ZOGlZdLt12MuDTfiYi/phHiQUC402EY
GXb8dYgYr5+0PDiPBa7dyt2VQLC/h9QRfOA82tb1vpJYC+KksSAH0nYQqJvs7XrqCN
i95/jwZnsWrV7w72+xsrO5qPujIE68TmM5I9Cyec=
Received: by beto.merlinux.eu (Postfix, from userid 1000)
id 229D3820070; Sat, 7 Dec 2019 11:53:58 +0100 (CET)
Date: Sat, 7 Dec 2019 11:53:57 +0100
From: holger krekel <holger@merlinux.eu>
To: holger+test@merlinux.eu
Subject: hello
Message-ID: <20191207105357.GA6266@beto>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="YiEDa0DAkWCtVeE4"
Content-Disposition: inline
--YiEDa0DAkWCtVeE4
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
siehe anhang
--YiEDa0DAkWCtVeE4
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="x.txt"
hello
--YiEDa0DAkWCtVeE4--