pandas default backend fixes

Author: evertlammerts

tests/conftest.py

@@ -2,7 +2,7 @@
 import warnings
 from importlib import import_module
 from pathlib import Path
-from typing import Any, Union
+from typing import Union
 
 import pytest
 
@@ -19,13 +19,27 @@
     pandas = None
     pyarrow_dtype = None
 
-# Check if pandas has arrow dtypes enabled
-try:
-    from pandas.compat import pa_version_under7p0
 
-    pyarrow_dtypes_enabled = not pa_version_under7p0
-except ImportError:
-    pyarrow_dtypes_enabled = False
+# Version-aware helpers for Pandas 2.x vs 3.0 compatibility
+def _get_pandas_ge_3():
+    if pandas is None:
+        return False
+    from packaging.version import Version
+
+    return Version(pandas.__version__) >= Version("3.0.0")
+
+
+PANDAS_GE_3 = _get_pandas_ge_3()
+
+
+def is_string_dtype(dtype):
+    """Check if a dtype is a string dtype (works across Pandas 2.x and 3.0).
+
+    Uses pd.api.types.is_string_dtype() which handles:
+    - Pandas 2.x: object dtype for strings
+    - Pandas 3.0+: str (StringDtype) for strings
+    """
+    return pandas.api.types.is_string_dtype(dtype)
 
 
 def import_pandas():
@@ -113,78 +127,6 @@ def pandas_supports_arrow_backend():
     return pandas_2_or_higher()
 
 
-def numpy_pandas_df(*args, **kwargs):
-    return import_pandas().DataFrame(*args, **kwargs)
-
-
-def arrow_pandas_df(*args, **kwargs):
-    df = numpy_pandas_df(*args, **kwargs)
-    return df.convert_dtypes(dtype_backend="pyarrow")
-
-
-class NumpyPandas:
-    def __init__(self) -> None:
-        self.backend = "numpy_nullable"
-        self.DataFrame = numpy_pandas_df
-        self.pandas = import_pandas()
-
-    def __getattr__(self, name: str) -> Any:  # noqa: ANN401
-        return getattr(self.pandas, name)
-
-
-def convert_arrow_to_numpy_backend(df):
-    names = df.columns
-    df_content = {}
-    for name in names:
-        df_content[name] = df[name].array.__arrow_array__()
-    # This should convert the pyarrow chunked arrays into numpy arrays
-    return import_pandas().DataFrame(df_content)
-
-
-def convert_to_numpy(df):
-    if (
-        pyarrow_dtypes_enabled
-        and pyarrow_dtype is not None
-        and any(True for x in df.dtypes if isinstance(x, pyarrow_dtype))
-    ):
-        return convert_arrow_to_numpy_backend(df)
-    return df
-
-
-def convert_and_equal(df1, df2, **kwargs):
-    df1 = convert_to_numpy(df1)
-    df2 = convert_to_numpy(df2)
-    import_pandas().testing.assert_frame_equal(df1, df2, **kwargs)
-
-
-class ArrowMockTesting:
-    def __init__(self) -> None:
-        self.testing = import_pandas().testing
-        self.assert_frame_equal = convert_and_equal
-
-    def __getattr__(self, name: str) -> Any:  # noqa: ANN401
-        return getattr(self.testing, name)
-
-
-# This converts dataframes constructed with 'DataFrame(...)' to pyarrow backed dataframes
-# Assert equal does the opposite, turning all pyarrow backed dataframes into numpy backed ones
-# this is done because we don't produce pyarrow backed dataframes yet
-class ArrowPandas:
-    def __init__(self) -> None:
-        self.pandas = import_pandas()
-        if pandas_2_or_higher() and pyarrow_dtypes_enabled:
-            self.backend = "pyarrow"
-            self.DataFrame = arrow_pandas_df
-        else:
-            # For backwards compatible reasons, just mock regular pandas
-            self.backend = "numpy_nullable"
-            self.DataFrame = self.pandas.DataFrame
-        self.testing = ArrowMockTesting()
-
-    def __getattr__(self, name: str) -> Any:  # noqa: ANN401
-        return getattr(self.pandas, name)
-
-
 @pytest.fixture
 def require():
     def _require(extension_name, db_name="") -> Union[duckdb.DuckDBPyConnection, None]:

tests/coverage/test_pandas_categorical_coverage.py

@@ -1,5 +1,4 @@
-import pytest
-from conftest import NumpyPandas
+import pandas as pd
 
 import duckdb
 
@@ -9,23 +8,23 @@ def check_result_list(res):
         assert res_item[0] == res_item[1]
 
 
-def check_create_table(category, pandas):
+def check_create_table(category):
     conn = duckdb.connect()
 
     conn.execute("PRAGMA enable_verification")
-    df_in = pandas.DataFrame(
+    df_in = pd.DataFrame(
         {
-            "x": pandas.Categorical(category, ordered=True),
-            "y": pandas.Categorical(category, ordered=True),
+            "x": pd.Categorical(category, ordered=True),
+            "y": pd.Categorical(category, ordered=True),
             "z": category,
         }
     )
 
     category.append("bla")
 
-    df_in_diff = pandas.DataFrame(  # noqa: F841
+    df_in_diff = pd.DataFrame(  # noqa: F841
         {
-            "k": pandas.Categorical(category, ordered=True),
+            "k": pd.Categorical(category, ordered=True),
         }
     )
 
@@ -68,14 +67,11 @@ def check_create_table(category, pandas):
     conn.execute("DROP TABLE t1")
 
 
-# TODO: extend tests with ArrowPandas  # noqa: TD002, TD003
 class TestCategory:
-    @pytest.mark.parametrize("pandas", [NumpyPandas()])
-    def test_category_string_uint16(self, duckdb_cursor, pandas):
+    def test_category_string_uint16(self, duckdb_cursor):
         category = [str(i) for i in range(300)]
-        check_create_table(category, pandas)
+        check_create_table(category)
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas()])
-    def test_category_string_uint32(self, duckdb_cursor, pandas):
+    def test_category_string_uint32(self, duckdb_cursor):
         category = [str(i) for i in range(70000)]
-        check_create_table(category, pandas)
+        check_create_table(category)

tests/extensions/test_httpfs.py

@@ -1,8 +1,8 @@
 import datetime
 import os
 
+import pandas as pd
 import pytest
-from conftest import ArrowPandas, NumpyPandas
 
 import duckdb
 
@@ -34,8 +34,7 @@ def test_s3fs(self, require):
         res = rel.fetchone()
         assert res == (1, 0, datetime.date(1965, 2, 28), 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 6, 0, 0, 0, 0)
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_httpfs(self, require, pandas):
+    def test_httpfs(self, require):
         connection = require("httpfs")
         try:
             connection.execute("""
@@ -51,14 +50,14 @@ def test_httpfs(self, require, pandas):
                 raise
 
         result_df = connection.fetchdf()
-        exp_result = pandas.DataFrame(
+        exp_result = pd.DataFrame(
             {
-                "id": pandas.Series([1, 2, 3], dtype="int32"),
+                "id": pd.Series([1, 2, 3], dtype="int32"),
                 "first_name": ["Amanda", "Albert", "Evelyn"],
                 "last_name": ["Jordan", "Freeman", "Morgan"],
             }
         )
-        pandas.testing.assert_frame_equal(result_df, exp_result)
+        pd.testing.assert_frame_equal(result_df, exp_result, check_dtype=False)
 
     def test_http_exception(self, require):
         connection = require("httpfs")

tests/fast/api/test_3654.py

@@ -1,4 +1,4 @@
-import pytest
+import pandas as pd
 
 import duckdb
 
@@ -8,13 +8,11 @@
     can_run = True
 except Exception:
     can_run = False
-from conftest import ArrowPandas, NumpyPandas
 
 
 class Test3654:
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_3654_pandas(self, duckdb_cursor, pandas):
-        df1 = pandas.DataFrame(
+    def test_3654_pandas(self, duckdb_cursor):
+        df1 = pd.DataFrame(
             {
                 "id": [1, 1, 2],
             }
@@ -25,12 +23,11 @@ def test_3654_pandas(self, duckdb_cursor, pandas):
         print(rel.execute().fetchall())
         assert rel.execute().fetchall() == [(1,), (1,), (2,)]
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_3654_arrow(self, duckdb_cursor, pandas):
+    def test_3654_arrow(self, duckdb_cursor):
         if not can_run:
             return
 
-        df1 = pandas.DataFrame(
+        df1 = pd.DataFrame(
             {
                 "id": [1, 1, 2],
             }

tests/fast/api/test_config.py

@@ -2,37 +2,32 @@
 import os
 import re
 
-import pytest
-from conftest import ArrowPandas, NumpyPandas
+import pandas as pd
 
 import duckdb
 
 
 class TestDBConfig:
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_default_order(self, duckdb_cursor, pandas):
-        df = pandas.DataFrame({"a": [1, 2, 3]})
+    def test_default_order(self, duckdb_cursor):
+        df = pd.DataFrame({"a": [1, 2, 3]})
         con = duckdb.connect(":memory:", config={"default_order": "desc"})
         result = con.execute("select * from df order by a").fetchall()
         assert result == [(3,), (2,), (1,)]
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_null_order(self, duckdb_cursor, pandas):
-        df = pandas.DataFrame({"a": [1, 2, 3, None]})
+    def test_null_order(self, duckdb_cursor):
+        df = pd.DataFrame({"a": [1, 2, 3, None]})
         con = duckdb.connect(":memory:", config={"default_null_order": "nulls_last"})
         result = con.execute("select * from df order by a").fetchall()
         assert result == [(1,), (2,), (3,), (None,)]
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_multiple_options(self, duckdb_cursor, pandas):
-        df = pandas.DataFrame({"a": [1, 2, 3, None]})
+    def test_multiple_options(self, duckdb_cursor):
+        df = pd.DataFrame({"a": [1, 2, 3, None]})
         con = duckdb.connect(":memory:", config={"default_null_order": "nulls_last", "default_order": "desc"})
         result = con.execute("select * from df order by a").fetchall()
         assert result == [(3,), (2,), (1,), (None,)]
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_external_access(self, duckdb_cursor, pandas):
-        df = pandas.DataFrame({"a": [1, 2, 3]})
+    def test_external_access(self, duckdb_cursor):
+        df = pd.DataFrame({"a": [1, 2, 3]})
         # this works (replacement scan)
         con_regular = duckdb.connect(":memory:", config={})
         con_regular.execute("select * from df")

tests/fast/api/test_dbapi00.py

@@ -1,8 +1,8 @@
 # simple DB API testcase
 
 import numpy
+import pandas as pd
 import pytest
-from conftest import ArrowPandas, NumpyPandas
 
 
 def assert_result_equal(result):
@@ -83,30 +83,29 @@ def test_numpy_selection(self, duckdb_cursor, integers, timestamps):
         arr.mask = [False, False, True]
         numpy.testing.assert_array_equal(result["t"], arr, "Incorrect result returned")
 
-    @pytest.mark.parametrize("pandas", [NumpyPandas(), ArrowPandas()])
-    def test_pandas_selection(self, duckdb_cursor, pandas, integers, timestamps):
+    def test_pandas_selection(self, duckdb_cursor, integers, timestamps):
         import datetime
 
         from packaging.version import Version
 
         # I don't know when this exactly changed, but 2.0.3 does not support this, recent versions do
-        if Version(pandas.__version__) <= Version("2.0.3"):
+        if Version(pd.__version__) <= Version("2.0.3"):
             pytest.skip("The resulting dtype is 'object' when given a Series with dtype Int32DType")
 
         duckdb_cursor.execute("SELECT * FROM integers")
         result = duckdb_cursor.fetchdf()
         array = numpy.ma.masked_array(numpy.arange(11))
         array.mask = [False] * 10 + [True]
-        arr = {"i": pandas.Series(array.data, dtype=pandas.Int32Dtype)}
-        arr["i"][array.mask] = pandas.NA
-        arr = pandas.DataFrame(arr)
-        pandas.testing.assert_frame_equal(result, arr)
+        arr = {"i": pd.Series(array.data, dtype=pd.Int32Dtype)}
+        arr["i"][array.mask] = pd.NA
+        arr = pd.DataFrame(arr)
+        pd.testing.assert_frame_equal(result, arr)
 
         duckdb_cursor.execute("SELECT * FROM timestamps")
         result = duckdb_cursor.fetchdf()
-        df = pandas.DataFrame(
+        df = pd.DataFrame(
             {
-                "t": pandas.Series(
+                "t": pd.Series(
                     data=[
                         datetime.datetime(year=1992, month=10, day=3, hour=18, minute=34, second=45),
                         datetime.datetime(year=2010, month=1, day=1, hour=0, minute=0, second=1),
@@ -116,7 +115,7 @@ def test_pandas_selection(self, duckdb_cursor, pandas, integers, timestamps):
                 )
             }
         )
-        pandas.testing.assert_frame_equal(result, df)
+        pd.testing.assert_frame_equal(result, df)
 
     # def test_numpy_creation(self, duckdb_cursor):
     #     # numpyarray = {'i': numpy.arange(10), 'v': numpy.random.randint(100, size=(1, 10))}  # segfaults

tests/fast/api/test_dbapi08.py

@@ -1,21 +1,19 @@
 # test fetchdf with various types
-import pytest
-from conftest import NumpyPandas
+import pandas as pd
 
 import duckdb
 
 
 class TestType:
-    @pytest.mark.parametrize("pandas", [NumpyPandas()])
-    def test_fetchdf(self, pandas):
+    def test_fetchdf(self):
         con = duckdb.connect()
         con.execute("CREATE TABLE items(item VARCHAR)")
         con.execute("INSERT INTO items VALUES ('jeans'), (''), (NULL)")
         res = con.execute("SELECT item FROM items").fetchdf()
-        assert isinstance(res, pandas.core.frame.DataFrame)
+        assert isinstance(res, pd.core.frame.DataFrame)
 
-        df = pandas.DataFrame({"item": ["jeans", "", None]})
+        df = pd.DataFrame({"item": ["jeans", "", None]})
 
         print(res)
         print(df)
-        pandas.testing.assert_frame_equal(res, df)
+        pd.testing.assert_frame_equal(res, df, check_dtype=False)