Skip to content

[SPARK-53441][PS] Bools |/&/^ None should fail under ANSI #52182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

[SPARK-53441][PS] Bools |/&/^ None should fail under ANSI #52182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0372baf
raise
xinrong-meng Aug 29, 2025
4368cf9
existing
xinrong-meng Aug 29, 2025
20cbce4
change ansi only
xinrong-meng Aug 29, 2025
1ad2833
test
xinrong-meng Aug 29, 2025
8f7fd9b
test fix
xinrong-meng Aug 29, 2025
cdcc45c
cond
xinrong-meng Sep 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions python/pyspark/pandas/data_type_ops/boolean_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,12 @@ def rmod(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:

def __and__(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
_sanitize_list_like(right)
if (
is_ansi_mode_enabled(left._internal.spark_frame.sparkSession)
and self.dtype == bool
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to check self.dtype == bool because

>>> pd.Series([True, False, np.nan]) | None
0    False
1    False
2    False

>>> ps.Series([True, False, np.nan]).dtype
dtype('O')
>>> ps.Series([True, False, np.nan]).spark.data_type
BooleanType()

BooleanType() will direct the code path here, but we want to support (instead of TypeError) this case, so we check dtype explicitly

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tested here https://github.com/apache/spark/blob/master/python/pyspark/pandas/tests/series/test_series.py#L262

and right is None
):
raise TypeError("AND can not be applied to given types.")
if isinstance(right, IndexOpsMixin) and isinstance(right.dtype, extension_dtypes):
return right.__and__(left)
else:
Expand All @@ -256,6 +262,12 @@ def and_func(left: PySparkColumn, right: Any) -> PySparkColumn:

def xor(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
_sanitize_list_like(right)
if (
is_ansi_mode_enabled(left._internal.spark_frame.sparkSession)
and self.dtype == bool
and right is None
):
raise TypeError("XOR can not be applied to given types.")
if isinstance(right, IndexOpsMixin) and isinstance(right.dtype, extension_dtypes):
return right ^ left
elif _is_valid_for_logical_operator(right):
Expand All @@ -277,6 +289,12 @@ def xor_func(left: PySparkColumn, right: Any) -> PySparkColumn:

def __or__(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
_sanitize_list_like(right)
if (
is_ansi_mode_enabled(left._internal.spark_frame.sparkSession)
and self.dtype == bool
and right is None
):
raise TypeError("OR can not be applied to given types.")
if isinstance(right, IndexOpsMixin) and isinstance(right.dtype, extension_dtypes):
return right.__or__(left)
else:
Expand Down
19 changes: 15 additions & 4 deletions python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

from pyspark import pandas as ps
from pyspark.pandas import option_context
from pyspark.testing.utils import is_ansi_mode_test
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
from pyspark.pandas.typedef.typehints import (
Expand Down Expand Up @@ -255,11 +256,15 @@ def test_and(self):
self.assert_eq(pser & pser, psser & psser)
self.assert_eq(pser & other_pser, psser & other_psser)
self.assert_eq(other_pser & pser, other_psser & psser)
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: psser & None)

def test_rand(self):
pser, psser = self.pdf["bool"], self.psdf["bool"]
self.assert_eq(True & pser, True & psser)
self.assert_eq(False & pser, False & psser)
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: None & psser)

def test_or(self):
pdf, psdf = self.bool_pdf, self.bool_psdf
Expand All @@ -269,16 +274,18 @@ def test_or(self):
self.assert_eq(pser | True, psser | True)
self.assert_eq(pser | False, psser | False)
self.assert_eq(pser | pser, psser | psser)
self.assert_eq(True | pser, True | psser)
self.assert_eq(False | pser, False | psser)

self.assert_eq(pser | other_pser, psser | other_psser)
self.assert_eq(other_pser | pser, other_psser | psser)
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: psser | None)

def test_ror(self):
pser, psser = self.pdf["bool"], self.psdf["bool"]
self.assert_eq(True | pser, True | psser)
self.assert_eq(False | pser, False | psser)
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: None | psser)

def test_xor(self):
pdf, psdf = self.bool_pdf, self.bool_psdf
Expand All @@ -293,6 +300,8 @@ def test_xor(self):

with self.assertRaisesRegex(TypeError, "XOR can not be applied to given types."):
psser ^ "a"
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: psser ^ None)

with option_context("compute.ops_on_diff_frames", True):
pser, other_pser = self.pdf["bool"], self.integral_pdf["this"]
Expand All @@ -305,6 +314,8 @@ def test_rxor(self):
self.assert_eq(True ^ pser, True ^ psser)
self.assert_eq(False ^ pser, False ^ psser)
self.assert_eq(1 ^ pser, 1 ^ psser)
if is_ansi_mode_test:
self.assertRaises(TypeError, lambda: None ^ psser)

def test_isnull(self):
self.assert_eq(self.pdf["bool"].isnull(), self.psdf["bool"].isnull())
Expand Down Expand Up @@ -721,8 +732,8 @@ def test_xor(self):

def test_rxor(self):
pser, psser = self.boolean_pdf["this"], self.boolean_psdf["this"]
self.check_extension(True | pser, True | psser)
self.check_extension(False | pser, False | psser)
self.check_extension(True ^ pser, True ^ psser)
self.check_extension(False ^ pser, False ^ psser)
with self.assertRaisesRegex(TypeError, "XOR can not be applied to given types."):
1 ^ psser

Expand Down