diff --git a/components/esp_security/src/esp32s31/esp_crypto_clk.h b/components/esp_security/src/esp32s31/esp_crypto_clk.h
index 25c3b3f80c2..aaa01a9b3db 100644
--- a/components/esp_security/src/esp32s31/esp_crypto_clk.h
+++ b/components/esp_security/src/esp32s31/esp_crypto_clk.h
@@ -15,10 +15,10 @@ void esp_crypto_common_clk_enable(bool enable);
 
 static inline void esp_crypto_clk_init(void)
 {
-    // Set crypto clock (`clk_sec`) to use 240M PLL clock
-    sec_ll_crypto_clk_src_sel(SOC_MOD_CLK_PLL_F240M);
 #if !CONFIG_ESP_CRYPTO_CLK_ON_DEMAND
     /* Keep crypto clocks always on for better crypto performance. */
     esp_crypto_common_clk_enable(true);
 #endif
+    // Set crypto clock (`clk_sec`) to use 240M PLL clock
+    sec_ll_crypto_clk_src_sel(SOC_MOD_CLK_PLL_F240M);
 }