Originally tested under cl/522005071, in combination with changes under pybind11_abseil:

```
--- a/pybind11_abseil/absl_casters.h
+++ b/pybind11_abseil/absl_casters.h
@@ -33,6 +33,7 @@
 #include <pybind11/cast.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
+#include <pybind11/type_caster_pyobject_ptr.h>

 #include <cmath>
 #include <cstdint>
@@ -282,8 +283,10 @@ struct type_caster<absl::CivilYear>

 // Returns {true, a span referencing the data contained by src} without copying
 // or converting the data if possible. Otherwise returns {false, an empty span}.
-template <typename T, typename std::enable_if<std::is_arithmetic<T>::value,
-                                              bool>::type = true>
+template <typename T, typename std::enable_if<
+                          std::is_arithmetic<T>::value ||
+                              is_same_ignoring_cvref<T, PyObject*>::value,
+                          bool>::type = true>
 std::tuple<bool, absl::Span<T>> LoadSpanFromBuffer(handle src) {
   Py_buffer view;
   int flags = PyBUF_STRIDES | PyBUF_FORMAT;
@@ -301,8 +304,10 @@ std::tuple<bool, absl::Span<T>> LoadSpan
   return {false, absl::Span<T>()};
 }
 // If T is not a numeric type, the buffer interface cannot be used.
-template <typename T, typename std::enable_if<!std::is_arithmetic<T>::value,
-                                              bool>::type = true>
+template <typename T, typename std::enable_if<
+                          !std::is_arithmetic<T>::value &&
+                              !is_same_ignoring_cvref<T, PyObject*>::value,
+                          bool>::type = true>
 constexpr std::tuple<bool, absl::Span<T>> LoadSpanFromBuffer(handle src) {
   return {false, absl::Span<T>()};
 }
```