diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile
index 4340a18f6ee..ceba8545e8b 100644
--- a/ci/docker/conda-cpp.dockerfile
+++ b/ci/docker/conda-cpp.dockerfile
@@ -34,6 +34,7 @@ RUN mamba install -q -y \
         --file arrow/ci/conda_env_cpp.txt \
         --file arrow/ci/conda_env_gandiva.txt \
         compilers \
+        conda-gcc-specs \
         doxygen \
         libnuma \
         mold \
diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
index 272141b604e..612bdac3237 100644
--- a/ci/docker/conda-integration.dockerfile
+++ b/ci/docker/conda-integration.dockerfile
@@ -37,6 +37,7 @@ RUN mamba install -q -y \
         "python < 3.12" \
         numpy \
         compilers \
+        conda-gcc-specs \
         go \
         maven=${maven} \
         nodejs=${node} \
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 05303b7bcfb..208a39dba3f 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -443,6 +443,12 @@ maybe_setup_virtualenv() {
 test_and_install_cpp() {
   show_header "Build, install and test C++ libraries"
 
+  # Ensure GCC uses libraries and headers from the Conda env
+  local conda_gcc_specs=()
+  if [ "$(uname)" = "Linux" ]; then
+    conda_gcc_specs=(conda-gcc-specs)
+  fi
+
   # Build and test C++
   maybe_setup_virtualenv numpy
   maybe_setup_conda \
@@ -452,7 +458,8 @@ test_and_install_cpp() {
     ncurses \
     numpy \
     sqlite \
-    compilers
+    compilers \
+    "${conda_gcc_specs[@]}"
 
   if [ "${USE_CONDA}" -gt 0 ]; then
     DEFAULT_DEPENDENCY_SOURCE="CONDA"