diff --git a/include/asm-arm/elf.h b/include/asm-arm/elf.h
index a1696ba238d3..7da97a937548 100644
--- a/include/asm-arm/elf.h
+++ b/include/asm-arm/elf.h
@@ -124,6 +124,8 @@ do { \
 	if (((ex).e_flags & EF_ARM_EABI_MASK) || \
 	    ((ex).e_flags & EF_ARM_SOFT_FLOAT)) \
 		set_thread_flag(TIF_USING_IWMMXT); \
+	else \
+		clear_thread_flag(TIF_USING_IWMMXT); \
 } while (0)
 
 #endif