diff --git a/src/mimeparser.rs b/src/mimeparser.rs
index 88569d5db..b5107b295 100644
--- a/src/mimeparser.rs
+++ b/src/mimeparser.rs
@@ -490,11 +490,13 @@ impl MimeMessage {
         let mail = mail.as_ref().map(|mail| {
             let (content, signatures_detached) = validate_detached_signature(mail, &public_keyring)
                 .unwrap_or((mail, Default::default()));
-            let signatures_detached = signatures_detached
-                .into_iter()
-                .map(|fp| (fp, Vec::new()))
-                .collect::<HashMap<_, _>>();
-            signatures.extend(signatures_detached);
+            if is_encrypted {
+                let signatures_detached = signatures_detached
+                    .into_iter()
+                    .map(|fp| (fp, Vec::new()))
+                    .collect::<HashMap<_, _>>();
+                signatures.extend(signatures_detached);
+            }
             content
         });