feat: client version information (#8557)

> we aim to inform about updates for installations outside of any
appstore soon.
>
> there is already a PR on android at
https://github.com/deltachat/deltachat-android/pull/4582, however, the
information about "update available" is a mockup there.
>
> in general, there are 3 ideas around about how to gather the "update
available" infomation - (1) checking a central url, (2) let contacts
provide information, (3) let relay provide information. on various
one-to-one discussions, outcome is that (3) is the most reasonable way
to go.

this PR is about reading update information via IMAP metadata from the
relay.

it is up to the UI to call `get_app_version()` at a reasonable time and
frequency, see comment in the code. when called, `get_app_version()`
iterates over all known profiles and relays and checks for version
information, returning the newest for the given scope.

we do not use an event, as that is tricky wrt changes - we do not know
if other relays report later a newer version. we also do not cache
anything, to prevent bad relays avoiding us to update permanently. also
it is easier :)

<details>
<summary>outdated notes and questions</summary>

- ~~it is up to the clients to get the needed information, we could let
core filter, but it seems easy enough the other way round, and may have
debug advantages, one can iterate etc.~~ EDIT: we now filter in core,
this also makes the jsonrpc part easier, see review comments

- when is IMAP METADATA actually read? when are they ready? is that
really the correct place? i am up to change that, but beware, this is
not really my expertise, so someone else may need to take over :)
EDIT: see below, IMAP METADATA is read on connection, before fetching
starts, usually fast enough

- relay part is missing. once the format is settled and discussed
shortly here, that should be done soon as well. but this is definitely
not my expertise and needed to be done by someone else :)

- key for IMAP METADATA is `/shared/vendor/deltachat/appversions` -
shall we continue use `deltachat` for compatibility or so? `chatmail`
seems to be more correct
EDIT: we stay with the current
</details>

relay counterpart issue: https://github.com/chatmail/relay/issues/1037

cc @link2xt @Hocuri @hpk42

---------

Co-authored-by: holger krekel <holger@merlinux.eu>

src/appversions.rs

@@ -0,0 +1,344 @@
+//! Get version information of clients.
+//!
+//! Used by clients to inform about updates.
+//! The version information comes in via IMAP METADATA,
+//! (as JSON) and is parsed to `AppVersionInfo`.
+use crate::accounts::Accounts;
+use crate::log::warn;
+use anyhow::Result;
+use serde::Deserialize;
+
+/// Version information of clients as used on the wire.
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase", default)]
+struct AppVersionInfo {
+    /// Array of clients with version information.
+    clients: Vec<AppClient>,
+}
+
+/// Version information of a single client, eg. "deltachat" or "ubuntutouch".
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase", default)]
+struct AppClient {
+    /// ID how the client identifies itself, eg. "deltachat" or "ubuntutouch"
+    client_id: String,
+
+    /// Array of sources for that client.
+    sources: Vec<AppSource>,
+}
+
+/// Version information of a single source of a client, eg. "gplay" or "fdroid".
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase", default)]
+pub struct AppSource {
+    /// ID how the client identifies a source, eg. "gplay" or "fdroid"
+    source_id: String,
+
+    /// Always increasing version number.
+    pub version_integer: u32,
+
+    /// Any version string.
+    pub version_string: String,
+
+    /// Where to download that version.
+    pub download_url: String,
+}
+
+/// Get version information of a specific client and source.
+///
+/// Iterates over all accounts and all transports,
+/// checking version information set by IMAP METADATA,
+/// and returns the source with the highest `version_integer`.
+///
+/// If no matching version information is available at all, `None` is returned.
+pub async fn get_app_version(
+    accounts: &Accounts,
+    client_id: &str,
+    source_id: &str,
+) -> Result<Option<AppSource>> {
+    let mut best: Option<AppSource> = None;
+
+    for account_id in accounts.get_all() {
+        let Some(context) = accounts.get_account(account_id) else {
+            continue;
+        };
+        for metadata in context.metadata.read().await.values() {
+            let Some(json) = &metadata.app_versions else {
+                continue;
+            };
+            let app_versions: AppVersionInfo = match serde_json::from_str(json) {
+                Ok(app_versions) => app_versions,
+                Err(err) => {
+                    warn!(context, "Failed to parse appversions: {err:#}.");
+                    continue;
+                }
+            };
+            let candidate = app_versions
+                .clients
+                .into_iter()
+                .find(|c| c.client_id == client_id)
+                .and_then(|c| c.sources.into_iter().find(|s| s.source_id == source_id));
+
+            if let Some(candidate) = candidate
+                && best
+                    .as_ref()
+                    .is_none_or(|b| candidate.version_integer > b.version_integer)
+            {
+                best = Some(candidate);
+            }
+        }
+    }
+
+    Ok(best)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::PathBuf;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_app_version_info_deserialize() -> Result<()> {
+        let json = r##"{
+            "clients": [
+              {
+                "clientId": "deltachat",
+                "sources": [
+                  {
+                    "sourceId": "gplay",
+                    "versionInteger": 754,
+                    "versionString": "2.57.0",
+                    "downloadUrl": "https://github.com/deltachat/deltachat-android/releases/download/v2.57.0/deltachat-gplay-release-2.57.0.apk"
+                  }
+                ]
+              }
+            ]
+          }"##;
+        let versions: AppVersionInfo = serde_json::from_str(json)?;
+        assert_eq!(versions.clients.len(), 1);
+        assert_eq!(versions.clients[0].client_id, "deltachat");
+        assert_eq!(versions.clients[0].sources.len(), 1);
+        assert_eq!(versions.clients[0].sources[0].source_id, "gplay");
+        assert_eq!(versions.clients[0].sources[0].version_integer, 754);
+        assert_eq!(versions.clients[0].sources[0].version_string, "2.57.0");
+        assert_eq!(
+            versions.clients[0].sources[0].download_url,
+            "https://github.com/deltachat/deltachat-android/releases/download/v2.57.0/deltachat-gplay-release-2.57.0.apk"
+        );
+
+        // missing fields are set to defaults, additional fields are ignored, errors are errors
+        let json = r##"{
+            "clients": [
+              {
+                "clientId": "deltachat",
+                "xsource": "bang"
+              }
+            ],
+            "foo": "bar"
+          }"##;
+        let versions: AppVersionInfo = serde_json::from_str(json)?;
+        assert_eq!(versions.clients.len(), 1);
+        assert_eq!(versions.clients[0].client_id, "deltachat");
+        assert_eq!(versions.clients[0].sources.len(), 0);
+
+        let json = "{}";
+        let versions: AppVersionInfo = serde_json::from_str(json)?;
+        assert_eq!(versions.clients.len(), 0);
+
+        let json = "";
+        assert!(serde_json::from_str::<AppVersionInfo>(json).is_err());
+
+        let json = "bad json";
+        assert!(serde_json::from_str::<AppVersionInfo>(json).is_err());
+
+        Ok(())
+    }
+
+    async fn mockup_app_versions(
+        accounts: &Accounts,
+        account_id: u32,
+        transport_id: u32,
+        json: &str,
+    ) {
+        let context = accounts.get_account(account_id).expect("account exists");
+        let mut metadata = context.metadata.write().await;
+        metadata.entry(transport_id).or_default().app_versions = Some(json.to_string());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_app_versions() -> Result<()> {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts");
+        let writable = true;
+        let mut accounts = Accounts::new(p.clone(), writable).await.unwrap();
+
+        // no accounts configured, means no versions are reported
+        let version = get_app_version(&accounts, "non-", "existant").await?;
+        assert!(version.is_none());
+
+        // first account reports two clients, with one and two sources
+        let account_id1 = accounts.add_account().await?;
+        let json = r##"{
+            "clients": [
+              {
+                "clientId": "basta",
+                "sources": [
+                  {
+                    "sourceId": "web",
+                    "versionInteger": 754,
+                    "versionString": "2.57.0",
+                    "downloadUrl": "https://example.org/basta-2.57.0.apk"
+                  }
+                ]
+              },
+              {
+                "clientId": "foo",
+                "sources": [
+                  {
+                    "sourceId": "bar",
+                    "versionInteger": 42,
+                    "versionString": "42.0",
+                    "downloadUrl": "https://foo.bar/42.0.prg"
+                  },
+                  {
+                    "sourceId": "baz",
+                    "versionInteger": 1337,
+                    "versionString": "13.37",
+                    "downloadUrl": "https://dl.org/1337.acc"
+                  }
+                ]
+              }
+            ]
+          }"##;
+        mockup_app_versions(&accounts, account_id1, 1, json).await;
+
+        let version = get_app_version(&accounts, "basta", "web").await?.unwrap();
+        assert_eq!(version.version_integer, 754);
+        assert_eq!(version.version_string, "2.57.0");
+        assert_eq!(version.download_url, "https://example.org/basta-2.57.0.apk");
+
+        let version = get_app_version(&accounts, "foo", "bar").await?.unwrap();
+        assert_eq!(version.version_integer, 42);
+        assert_eq!(version.version_string, "42.0");
+        assert_eq!(version.download_url, "https://foo.bar/42.0.prg");
+
+        let version = get_app_version(&accounts, "foo", "baz").await?.unwrap();
+        assert_eq!(version.version_integer, 1337);
+        assert_eq!(version.version_string, "13.37");
+        assert_eq!(version.download_url, "https://dl.org/1337.acc");
+
+        let version = get_app_version(&accounts, "non-", "existant").await?;
+        assert!(version.is_none());
+
+        // a second account reports a newer version for "bar"
+        let account_id2 = accounts.add_account().await?;
+        let json = r##"{
+            "clients": [
+              {
+                "clientId": "foo",
+                "sources": [
+                  {
+                    "sourceId": "bar",
+                    "versionInteger": 43,
+                    "versionString": "43.0",
+                    "downloadUrl": "https://foo.bar/43.0-is-newer.prg"
+                  }
+                ]
+              }
+            ]
+          }"##;
+        mockup_app_versions(&accounts, account_id2, 1, json).await;
+
+        let version = get_app_version(&accounts, "basta", "web").await?.unwrap();
+        assert_eq!(version.version_integer, 754);
+        assert_eq!(version.version_string, "2.57.0");
+        assert_eq!(version.download_url, "https://example.org/basta-2.57.0.apk");
+
+        let version = get_app_version(&accounts, "foo", "bar").await?.unwrap();
+        assert_eq!(version.version_integer, 43);
+        assert_eq!(version.version_string, "43.0");
+        assert_eq!(version.download_url, "https://foo.bar/43.0-is-newer.prg");
+
+        let version = get_app_version(&accounts, "foo", "baz").await?.unwrap();
+        assert_eq!(version.version_integer, 1337);
+        assert_eq!(version.version_string, "13.37");
+        assert_eq!(version.download_url, "https://dl.org/1337.acc");
+
+        let version = get_app_version(&accounts, "non-", "existant").await?;
+        assert!(version.is_none());
+
+        // a third account reports a older version for "bar", that is ignored
+        let account_id3 = accounts.add_account().await?;
+        let json = r##"{
+            "clients": [
+              {
+                "clientId": "foo",
+                "sources": [
+                  {
+                    "sourceId": "bar",
+                    "versionInteger": 39,
+                    "versionString": "39.0",
+                    "downloadUrl": "https://foo.bar/39.0-is-too-old.prg"
+                  }
+                ]
+              }
+            ]
+          }"##;
+        mockup_app_versions(&accounts, account_id3, 1, json).await;
+
+        let version = get_app_version(&accounts, "basta", "web").await?.unwrap();
+        assert_eq!(version.version_integer, 754);
+        assert_eq!(version.version_string, "2.57.0");
+        assert_eq!(version.download_url, "https://example.org/basta-2.57.0.apk");
+
+        let version = get_app_version(&accounts, "foo", "bar").await?.unwrap();
+        assert_eq!(version.version_integer, 43);
+        assert_eq!(version.version_string, "43.0");
+        assert_eq!(version.download_url, "https://foo.bar/43.0-is-newer.prg");
+
+        let version = get_app_version(&accounts, "foo", "baz").await?.unwrap();
+        assert_eq!(version.version_integer, 1337);
+        assert_eq!(version.version_string, "13.37");
+        assert_eq!(version.download_url, "https://dl.org/1337.acc");
+
+        let version = get_app_version(&accounts, "non-", "existant").await?;
+        assert!(version.is_none());
+
+        // second account returns an invalid json, that account is skipped, but app versionss are still gathered from the other
+        let json = r"bad json!";
+        mockup_app_versions(&accounts, account_id2, 1, json).await;
+
+        let version = get_app_version(&accounts, "foo", "bar").await?.unwrap();
+        assert_eq!(version.version_integer, 42);
+        assert_eq!(version.version_string, "42.0");
+        assert_eq!(version.download_url, "https://foo.bar/42.0.prg");
+
+        // second account now has two transports, returning different version information in different orders
+        let json = r##"{
+            "clients": [
+              { "clientId": "a", "sources": [{"sourceId": "a", "versionInteger": 1, "versionString": "1.0", "downloadUrl": "https://a/1.prg"}] },
+              { "clientId": "b", "sources": [{"sourceId": "b", "versionInteger": 2, "versionString": "2.0", "downloadUrl": "https://b/2.prg"}] },
+              { "clientId": "c", "sources": [{"sourceId": "c", "versionInteger": 2, "versionString": "2.0", "downloadUrl": "https://c/2.prg"}] }
+            ]
+          }"##;
+        mockup_app_versions(&accounts, account_id2, 1, json).await;
+        let json = r##"{
+            "clients": [
+              { "clientId": "a", "sources": [{"sourceId": "a", "versionInteger": 2, "versionString": "2.0", "downloadUrl": "https://a/2.prg"}] },
+              { "clientId": "b", "sources": [{"sourceId": "b", "versionInteger": 1, "versionString": "1.0", "downloadUrl": "https://b/1.prg"}] },
+              { "clientId": "d", "sources": [{"sourceId": "d", "versionInteger": 2, "versionString": "2.0", "downloadUrl": "https://d/2.prg"}] }
+            ]
+          }"##;
+        mockup_app_versions(&accounts, account_id2, 2, json).await;
+        let version = get_app_version(&accounts, "a", "a").await?.unwrap();
+        assert_eq!(version.version_integer, 2);
+        let version = get_app_version(&accounts, "b", "b").await?.unwrap();
+        assert_eq!(version.version_integer, 2);
+        let version = get_app_version(&accounts, "c", "c").await?.unwrap();
+        assert_eq!(version.version_integer, 2);
+        let version = get_app_version(&accounts, "d", "d").await?.unwrap();
+        assert_eq!(version.version_integer, 2);
+
+        Ok(())
+    }
+}

src/imap.rs

@@ -140,6 +140,10 @@ pub(crate) struct ServerMetadata {
     /// should be fetched from the server
     /// to be ready for WebRTC calls.
     pub ice_servers_expiration_timestamp: i64,
+
+    /// App versions, as raw JSON string.
+    /// Consumed by get_app_versions().
+    pub app_versions: Option<String>,
 }
 
 struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
@@ -1292,6 +1296,10 @@ impl Session {
             let now = time();
 
             // Refresh TURN server credentials if they expire in 12 hours.
+            //
+            // Moreover, Take the chance to update `app_versions` as well.
+            // As best effort, even checking every some days is good enough -
+            // and saves one additional time get get_metadata() call.
             if now + 3600 * 12 < old_metadata.ice_servers_expiration_timestamp {
                 return Ok(());
             }
@@ -1302,7 +1310,11 @@ impl Session {
                 let mailbox = "";
                 let options = "";
                 let metadata = self
-                    .get_metadata(mailbox, options, "(/shared/vendor/deltachat/turn)")
+                    .get_metadata(
+                        mailbox,
+                        options,
+                        "(/shared/vendor/deltachat/turn /shared/vendor/deltachat/appversions)",
+                    )
                     .await?;
                 for m in metadata {
                     if m.entry == "/shared/vendor/deltachat/turn"
@@ -1318,6 +1330,8 @@ impl Session {
                                 warn!(context, "Failed to parse TURN server metadata: {err:#}.");
                             }
                         }
+                    } else if m.entry == "/shared/vendor/deltachat/appversions" {
+                        old_metadata.app_versions = m.value;
                     }
                 }
             }
@@ -1341,6 +1355,7 @@ impl Session {
         let mut max_smtp_rcpt_to = None;
         let mut ice_servers = None;
         let mut ice_servers_expiration_timestamp = 0;
+        let mut app_versions = None;
 
         let mailbox = "";
         let options = "";
@@ -1348,7 +1363,7 @@ impl Session {
             .get_metadata(
                 mailbox,
                 options,
-                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay /shared/vendor/deltachat/turn /shared/vendor/deltachat/maxsmtprecipients)",
+                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay /shared/vendor/deltachat/turn /shared/vendor/deltachat/maxsmtprecipients /shared/vendor/deltachat/appversions)",
             )
             .await?;
         for m in metadata {
@@ -1396,6 +1411,9 @@ impl Session {
                         }
                     }
                 }
+                "/shared/vendor/deltachat/appversions" => {
+                    app_versions = m.value;
+                }
                 _ => {}
             }
         }
@@ -1417,6 +1435,7 @@ impl Session {
                 supports_push: max_smtp_rcpt_to.is_some() || self.capabilities.has_xdeltapush,
                 ice_servers,
                 ice_servers_expiration_timestamp,
+                app_versions,
             },
         );
         Ok(())

src/lib.rs

@@ -54,6 +54,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+pub mod appversions;
 mod automatic_relay_management;
 pub mod blob;
 pub mod calls;