diff --git a/libexec/rbenv-rehash b/libexec/rbenv-rehash
index 2b5ffecc..a5c45756 100755
--- a/libexec/rbenv-rehash
+++ b/libexec/rbenv-rehash
@@ -60,6 +60,18 @@ SH
   chmod +x "$PROTOTYPE_SHIM_PATH"
 }
 
+# If the contents of the prototype shim file differ from the contents
+# of the first shim in the shims directory, assume rbenv has been
+# upgraded and the existing shims need to be removed.
+remove_outdated_shims() {
+  for shim in *; do
+    if ! diff "$PROTOTYPE_SHIM_PATH" "$shim" >/dev/null 2>&1; then
+      for shim in *; do rm -f "$shim"; done
+    fi
+    break
+  done
+}
+
 # The basename of each argument passed to `make_shims` will be
 # registered for installation as a shim. In this way, plugins may call
 # `make_shims` with a glob to register many shims at once.
@@ -146,10 +158,11 @@ remove_stale_shims() {
 
 # Change to the shims directory.
 cd "$SHIM_PATH"
+shopt -s nullglob
 
 # Create the prototype shim, then register shims for all known binaries.
 create_prototype_shim
-shopt -s nullglob
+remove_outdated_shims
 make_shims ../versions/*/bin/*
 
 # Restore the previous working directory.