diff --git a/board/xilinx/zynqmp/zynqmp.c b/board/xilinx/zynqmp/zynqmp.c
index 107a3e494e..ff776f7deb 100644
--- a/board/xilinx/zynqmp/zynqmp.c
+++ b/board/xilinx/zynqmp/zynqmp.c
@@ -278,10 +278,13 @@ int board_early_init_r(void)
 {
 	u32 val;
 
+	if (current_el() != 3)
+		return 0;
+
 	val = readl(&crlapb_base->timestamp_ref_ctrl);
 	val &= ZYNQMP_CRL_APB_TIMESTAMP_REF_CTRL_CLKACT;
 
-	if (current_el() == 3 && !val) {
+	if (!val) {
 		val = readl(&crlapb_base->timestamp_ref_ctrl);
 		val |= ZYNQMP_CRL_APB_TIMESTAMP_REF_CTRL_CLKACT;
 		writel(val, &crlapb_base->timestamp_ref_ctrl);