diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index fdb28b540e2d2..fda1d806dabc2 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -23,6 +23,27 @@ include(CheckCXXSourceCompiles)
 
 message(STATUS "System processor: ${CMAKE_SYSTEM_PROCESSOR}")
 
+set(UsingClang
+    CMAKE_C_COMPILER_ID
+    STREQUAL
+    "AppleClang"
+    OR
+    CMAKE_C_COMPILER_ID
+    STREQUAL
+    "Clang")
+set(UsingClangPlusPlus
+    CMAKE_CXX_COMPILER_ID
+    STREQUAL
+    "AppleClang"
+    OR
+    CMAKE_CXX_COMPILER_ID
+    STREQUAL
+    "Clang")
+
+if(UsingClangPlusPlus and NOT UsingClang)
+  error("When using clang++ you must also use clang")
+endif()
+
 if(NOT DEFINED ARROW_CPU_FLAG)
   if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
     set(ARROW_CPU_FLAG "emscripten")