[cppyy] Disallow conversion from C++ instance to function pointers

bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx

@@ -2655,7 +2655,7 @@ static PyMethodDef gWrapperCacheEraserMethodDef = {
 };
 
 static void* PyFunction_AsCPointer(PyObject* pyobject,
-    const std::string& rettype, const std::string& signature)
+    const std::string& rettype, const std::string& signature, bool allowCppInstance)
 {
 // Convert a bound C++ function pointer or callable python object to a C-style
 // function pointer. The former is direct, the latter involves a JIT-ed wrapper.
@@ -2702,8 +2702,12 @@ static void* PyFunction_AsCPointer(PyObject* pyobject,
         return fptr;
     }
 
-    if (PyCallable_Check(pyobject)) {
+
+    if (PyCallable_Check(pyobject) && (allowCppInstance || !CPPInstance_Check(pyobject))) {
     // generic python callable: create a C++ wrapper function
+    // Sometimes we don't want to take this branch if the object is a C++
+    // instance, because C++ doesn't allow converting functor objects to
+    // function pointers, but only to std::function.
         void* wpraddress = nullptr;
 
     // re-use existing wrapper if possible
@@ -2808,7 +2812,7 @@ bool CPyCppyy::FunctionPointerConverter::SetArg(
     }
 
 // normal case, get a function pointer
-    void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature);
+    void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature, fAllowCppInstance);
     if (fptr) {
         SetLifeLine(ctxt->fPyContext, pyobject, (intptr_t)this);
         para.fValue.fVoidp = fptr;
@@ -2840,7 +2844,7 @@ bool CPyCppyy::FunctionPointerConverter::ToMemory(
     }
 
 // normal case, get a function pointer
-    void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature);
+    void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature, fAllowCppInstance);
     if (fptr) {
         SetLifeLine(ctxt, pyobject, (intptr_t)address);
         *((void**)address) = fptr;

bindings/pyroot/cppyy/CPyCppyy/src/DeclareConverters.h

@@ -408,13 +408,16 @@ class FunctionPointerConverter : public Converter {
 protected:
     std::string fRetType;
     std::string fSignature;
+    bool fAllowCppInstance = false;
 };
 
 // std::function
 class StdFunctionConverter : public FunctionPointerConverter {
 public:
     StdFunctionConverter(Converter* cnv, const std::string& ret, const std::string& sig) :
-        FunctionPointerConverter(ret, sig), fConverter(cnv) {}
+        FunctionPointerConverter(ret, sig), fConverter(cnv) {
+        fAllowCppInstance = true;
+    }
     StdFunctionConverter(const StdFunctionConverter&) = delete;
     StdFunctionConverter& operator=(const StdFunctionConverter&) = delete;
     virtual ~StdFunctionConverter() { delete fConverter; }

bindings/pyroot/cppyy/cppyy/test/test_overloads.py

@@ -1,4 +1,4 @@
 import py, pytest, os
 from pytest import raises, skip, mark
 from support import setup_make, ispypy, IS_WINDOWS, IS_MAC_ARM
 
@@ -73,7 +73,7 @@
         import cppyy
         more_overloads = cppyy.gbl.more_overloads
         aa_ol = cppyy.gbl.aa_ol
         bb_ol = cppyy.gbl.bb_ol
         cc_ol = cppyy.gbl.cc_ol
         dd_ol = cppyy.gbl.dd_ol
 
@@ -130,13 +130,13 @@
     def test07_mean_overloads(self):
         """Adapted test for array overloading"""
 
         import cppyy, array
         cmean = cppyy.gbl.calc_mean
 
         numbers = [8, 2, 4, 2, 4, 2, 4, 4, 1, 5, 6, 3, 7]
         mean, median = 4.0, 4.0
 
         for l in ['f', 'd', 'i', 'h', 'l']:
             a = array.array(l, numbers)
             assert round(cmean(len(a), a) - mean, 8) == 0
 
@@ -158,7 +158,7 @@
         assert m.slice(0)          == 'non-const'
         assert m,slice(0, 0)       == 'non-const'
         assert m.slice_const(0)    ==     'const'
         assert m,slice_const(0, 0) ==     'const'
 
     def test09_bool_int_overloads(self):
         """Check bool/int overloaded calls"""
@@ -405,5 +405,51 @@
 
         assert result_instance == result_direct
 
+
+    def test14_disallow_functor_to_function_pointer(self):
+        """Make sure we're no allowing to convert C++ functors to function
+        pointers, extending the C++ language in an unnatural way that can lead
+        to wrong overload resolutions."""
+
+        import cppyy
+
+        cppyy.cppdef("""
+        class Test14Functor {
+        public:
+            double operator () (double* args, double*) {
+                return 4.0 * args[0];
+            }
+        };
+
+        int test14_foo(double (*fcn)(double*, double*)) {
+            return 0;
+        }
+
+        template<class T>
+        int test14_foo(T fcn) {
+            return 1;
+        }
+
+        int test14_bar(double (*fcn)(double*, double*)) {
+            return 0;
+        }
+
+        int test14_baz(double (*fcn)(double*, double*)) {
+            return 0;
+        }
+
+        int test14_baz(std::function<double(double*, double*)> const &fcn) {
+            return 2;
+        }
+        """)
+
+        functor = cppyy.gbl.Test14Functor()
+        assert cppyy.gbl.test14_foo(functor) == 1 # should resolve to foo(T fcn)
+        # not allowed, because there is only an overload taking a function pointer
+        raises(TypeError, cppyy.gbl.test14_bar, functor)
+        # The "baz" function has a std::function overload, which should be selected
+        assert cppyy.gbl.test14_baz(functor) == 2 # should resolve to baz(std::function)
+
+
 if __name__ == "__main__":
     exit(pytest.main(args=['-sv', '-ra', __file__]))