diff --git a/src/receive_imf.rs b/src/receive_imf.rs
index dfffb66e0..efbe30d73 100644
--- a/src/receive_imf.rs
+++ b/src/receive_imf.rs
@@ -346,18 +346,9 @@ pub(crate) async fn receive_imf_inner(
         },
     )
     .await?;
-    let past_ids = add_or_lookup_contacts_by_address_list(
-        context,
-        &mime_parser.past_members,
-        if !mime_parser.incoming {
-            Origin::OutgoingTo
-        } else if incoming_origin.is_known() {
-            Origin::IncomingTo
-        } else {
-            Origin::IncomingUnknownTo
-        },
-    )
-    .await?;
+    let past_ids =
+        add_or_lookup_contacts_by_address_list(context, &mime_parser.past_members, Origin::Hidden)
+            .await?;
 
     update_verified_keys(context, &mut mime_parser, from_id).await?;