diff --git a/plugins/python-build/bin/python-build b/plugins/python-build/bin/python-build
index 0b29c4e5..8b6c97f6 100755
--- a/plugins/python-build/bin/python-build
+++ b/plugins/python-build/bin/python-build
@@ -1564,6 +1564,8 @@ use_homebrew_zlib() {
 }
 
 use_xcode_sdk_zlib() {
+  # If a custom compiler is used, including XCode SDK will likely break it
+  [[ "${CC:-clang}" != "clang" || "$(command -v clang 2>/dev/null || true)" != "/usr/bin/clang" ]] && return 1
   local xc_sdk_path="$(xcrun --show-sdk-path 2>/dev/null || true)"
   if [ -d "$xc_sdk_path" ]; then
     echo "python-build: use zlib from xcode sdk"