add option to access original message (#2125)

* draft API to deal with uncut message texts

* add column mime_modified

* add mime_modified flag to MimeParser and save it in the database

* save mime_headers also when mime_modified is set

* cargo fmt

* set mime_modified on parsed html-texts and when there are multiple alternative-parts; add test for that

* prototype functions, add to repl and ffi

* use correct mime_modified flag

* basically parse Mime-Structure to HTML

* add basic tests for HTML-parsing

* convert text/plain to html for getting original

* respect charset for plain texts

* make test more specific

* fix handling non-utf-8 charsets for plain messages

* add test for plain_to_html()

* add failing test for plaintext linkify

* linkify urls in plain text

* fix regex

* plain text linkify: add failing test for encapsulated links as <https://domain.com>

* plain text linkify: make encapsulated links as <https://domain.com> work

* plain text linkify: require word boundary at beginning of link, add tests for that

* plain text linkify: linkify emails

* plain text: support format=flowed

* plain text: support quotes

* make clippy happy

* set mime-modified also when simplify() cuts non-html messages, add tests for that

* streamline mime recursion

* repl tool: write original html to file for further processing

* convert cid:- to data:-protocol

* add a test for cid: to data: conversion

* make clippy happy

* fix html-tests to work with windows-lineends

* clarify what the returned html-code may contain

* add some more detailed doc comments

* add mime_modified column only if not exist

this additional check is needed
as the column may added with another dbversion in
some shipped beta-versions.

* incorporate documentation suggestions from review

* rename get_original_mime_html() to more simple get_html()

* rename api is_mime_modified() to more simple has_html(); internally, mime_modified-flag stays as-is, however

* rename MimeS to MimeMultipartType

* do not set mime-modified flag for encrypted messages that need extra-handling for saved mime-structure

* fix typo

* move get_msg_html() to MsgId.get_html()

* incorporate more documentation suggestions from review

* remove unused return value from collect_texts_recursive()

* avoid mime_modified being mutable in write-parts-loop

* move 'use futures::future::FutureExt' atop of html.rs

* move attributes defining plain-text to a dedicated structure

* more PlainText to separate file

* escape cid when building regex

* let dc_get_msg_html() return NULL when calling with bad param

src/sql.rs

@@ -224,6 +224,31 @@ impl Sql {
         .await
     }
 
+    /// Check if a column exists in a given table.
+    pub async fn col_exists(
+        &self,
+        table_name: impl AsRef<str>,
+        col_name: impl AsRef<str>,
+    ) -> Result<bool> {
+        let table_name = table_name.as_ref().to_string();
+        let col_name = col_name.as_ref().to_string();
+        self.with_conn(move |conn| {
+            let mut exists = false;
+            // `PRAGMA table_info` returns one row per column,
+            // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value
+            conn.pragma(None, "table_info", &table_name, |row| {
+                let curr_name: String = row.get(1)?;
+                if col_name == curr_name {
+                    exists = true;
+                }
+                Ok(())
+            })?;
+
+            Ok(exists)
+        })
+        .await
+    }
+
     /// Execute a query which is expected to return zero or one row.
     pub async fn query_row_optional<T, F>(
         &self,
@@ -1413,6 +1438,17 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
 
             sql.set_raw_config_int(context, "dbversion", 71).await?;
         }
+        if dbversion < 72 {
+            info!(context, "[migration] v72");
+            if !sql.col_exists("msgs", "mime_modified").await? {
+                sql.execute(
+                    "ALTER TABLE msgs ADD COLUMN mime_modified INTEGER DEFAULT 0;",
+                    paramsv![],
+                )
+                .await?;
+            }
+            sql.set_raw_config_int(context, "dbversion", 72).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1468,6 +1504,7 @@ async fn prune_tombstones(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_maybe_add_file() {
@@ -1494,4 +1531,19 @@ mod test {
         assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
         assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
+
+    #[async_std::test]
+    async fn test_table_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.table_exists("msgs").await.unwrap());
+        assert!(!t.ctx.sql.table_exists("foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_col_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.col_exists("msgs", "mime_modified").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("msgs", "foobar").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("foobar", "foobar").await.unwrap());
+    }
 }