diff --git a/components/esp_hw_support/port/esp32s31/pmu_sleep.c b/components/esp_hw_support/port/esp32s31/pmu_sleep.c
index a07bd8439bd..fd53fdd9d45 100644
--- a/components/esp_hw_support/port/esp32s31/pmu_sleep.c
+++ b/components/esp_hw_support/port/esp32s31/pmu_sleep.c
@@ -224,30 +224,26 @@ const pmu_sleep_config_t* pmu_sleep_config_default(pmu_sleep_config_t *config, p
             analog_default.hp_sys.analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
             analog_default.lp_sys[LP(SLEEP)].analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
         }
-
-        if (!(sleep_flags & PMU_SLEEP_PD_XTAL))
-        {
-            // Analog parameters in HP_SLEEP
-            analog_default.hp_sys.analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
-            analog_default.hp_sys.analog.bias_sleep = PMU_BIASSLP_SLEEP_ON;
-            analog_default.hp_sys.analog.dbg_atten = PMU_DBG_ATTEN_ACTIVE_DEFAULT;
-            analog_default.hp_sys.analog.dbias = HP_CALI_DBIAS_DEFAULT;
-
-            // Analog parameters in LP_SLEEP
-            analog_default.lp_sys[LP(SLEEP)].analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
-            analog_default.lp_sys[LP(SLEEP)].analog.bias_sleep = PMU_BIASSLP_SLEEP_ON;
-            analog_default.lp_sys[LP(SLEEP)].analog.dbg_atten = PMU_DBG_ATTEN_ACTIVE_DEFAULT;
-        }
         config->analog = analog_default;
     }
 
-    if (sleep_flags & RTC_SLEEP_XTAL_AS_RTC_FAST) {
-        // Keep XTAL on in HP_SLEEP state if it is the clock source of RTC_FAST
-        power_default.hp_sys.xtal.xpd_xtal = 1;
+    if (!(sleep_flags & PMU_SLEEP_PD_XTAL)) {
         config->analog.hp_sys.analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
-        config->analog.hp_sys.analog.bias_sleep = PMU_BIASSLP_SLEEP_ON;
+        config->analog.lp_sys[LP(SLEEP)].analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
+    }
+
+    if (!(sleep_flags & PMU_SLEEP_PD_XTAL) || !(sleep_flags & PMU_SLEEP_PD_RC_FAST)) {
         config->analog.hp_sys.analog.dbg_atten = PMU_DBG_ATTEN_ACTIVE_DEFAULT;
         config->analog.hp_sys.analog.dbias = HP_CALI_DBIAS_DEFAULT;
+        config->analog.lp_sys[LP(SLEEP)].analog.dbg_atten = PMU_DBG_ATTEN_ACTIVE_DEFAULT;
+        config->analog.lp_sys[LP(SLEEP)].analog.dbias = LP_CALI_DBIAS_DEFAULT;
+    }
+
+    if (sleep_flags & RTC_SLEEP_XTAL_AS_RTC_FAST) {
+        // Keep XTAL on in HP_SLEEP only; LP_SLEEP still follows PD_XTAL.
+        power_default.hp_sys.xtal.xpd_xtal = 1;
+        config->analog.hp_sys.analog.dbg_atten = PMU_DBG_ATTEN_ACTIVE_DEFAULT;
+        config->analog.hp_sys.analog.pd_cur = PMU_PD_CUR_SLEEP_ON;
     }
 
     config->power = power_default;
diff --git a/components/esp_system/port/soc/esp32s31/clk.c b/components/esp_system/port/soc/esp32s31/clk.c
index 810bfb43d71..8d0efc21441 100644
--- a/components/esp_system/port/soc/esp32s31/clk.c
+++ b/components/esp_system/port/soc/esp32s31/clk.c
@@ -56,7 +56,16 @@ __attribute__((weak)) void esp_clk_init(void)
     assert(rtc_clk_xtal_freq_get() == SOC_XTAL_FREQ_40M);
 
     rtc_clk_8m_enable(true);
+#if CONFIG_RTC_FAST_CLK_SRC_RC_FAST
     rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_RC_FAST);
+#elif CONFIG_RTC_FAST_CLK_SRC_XTAL
+    rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_XTAL);
+    if (esp_sleep_sub_mode_dump_config(NULL)[ESP_SLEEP_RTC_FAST_USE_XTAL_MODE] == 0) {
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_FAST_USE_XTAL_MODE, true);
+    }
+#else
+#error "No RTC fast clock source configured"
+#endif
 
 #if defined(CONFIG_BOOTLOADER_WDT_ENABLE) && SOC_RTC_WDT_SUPPORTED
     // WDT uses a SLOW_CLK clock source. After a function select_rtc_slow_clk a frequency of this source can changed.
diff --git a/components/ulp/test_apps/.build-test-rules.yml b/components/ulp/test_apps/.build-test-rules.yml
index cb62926709c..0b1752de04b 100644
--- a/components/ulp/test_apps/.build-test-rules.yml
+++ b/components/ulp/test_apps/.build-test-rules.yml
@@ -16,9 +16,6 @@ components/ulp/test_apps/lp_core/lp_core_basic_tests:
   disable:
     - if: SOC_LP_CORE_SUPPORTED != 1
     - if: CONFIG_NAME == "xtal" and SOC_CLK_LP_FAST_SUPPORT_XTAL != 1
-    - if: IDF_TARGET in ["esp32s31"] and CONFIG_NAME == "xtal"
-      temporary: true
-      reason: CONFIG_RTC_FAST_CLK_SRC_XTAL not supported yet # TODO: [ESP32S31] IDF-15576
     - if: CONFIG_NAME == "lp_vad" and SOC_LP_VAD_SUPPORTED != 1
   depends_components:
     - ulp