diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.csv b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.csv new file mode 100644 index 000000000000..11f2831ca359 --- /dev/null +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.csv @@ -0,0 +1,126 @@ +test case pyarrow array types_mapper=pd.ArrowDtype, zero_copy_only=False types_mapper=pd.ArrowDtype, zero_copy_only=True verified zero-copy +int8:standard [0, 1, -1, 127, -128]@int8 [0, 1, -1, 127, -128]@Series[int8[pyarrow]] [0, 1, -1, 127, -128]@Series[int8[pyarrow]] zero-copy +int8:nullable [0, 1, None]@int8 [0, 1, ]@Series[int8[pyarrow]] [0, 1, ]@Series[int8[pyarrow]] zero-copy +int8:empty []@int8 []@Series[int8[pyarrow]] []@Series[int8[pyarrow]] zero-copy +int16:standard [0, 1, -1, 32767, -32768]@int16 [0, 1, -1, 32767, -32768]@Series[int16[pyarrow]] [0, 1, -1, 32767, -32768]@Series[int16[pyarrow]] zero-copy +int16:nullable [0, 1, None]@int16 [0, 1, ]@Series[int16[pyarrow]] [0, 1, ]@Series[int16[pyarrow]] zero-copy +int16:empty []@int16 []@Series[int16[pyarrow]] []@Series[int16[pyarrow]] zero-copy +int32:standard [0, 1, -1, 2147483647, -2147483648]@int32 [0, 1, -1, 2147483647, -2147483648]@Series[int32[pyarrow]] [0, 1, -1, 2147483647, -2147483648]@Series[int32[pyarrow]] zero-copy +int32:nullable [0, 1, None]@int32 [0, 1, ]@Series[int32[pyarrow]] [0, 1, ]@Series[int32[pyarrow]] zero-copy +int32:empty []@int32 []@Series[int32[pyarrow]] []@Series[int32[pyarrow]] zero-copy +int64:standard [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64[pyarrow]] [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64[pyarrow]] zero-copy +int64:nullable [0, 1, None]@int64 [0, 1, ]@Series[int64[pyarrow]] [0, 1, ]@Series[int64[pyarrow]] zero-copy +int64:empty []@int64 []@Series[int64[pyarrow]] []@Series[int64[pyarrow]] zero-copy +uint8:standard [0, 1, 255]@uint8 [0, 1, 255]@Series[uint8[pyarrow]] [0, 1, 255]@Series[uint8[pyarrow]] zero-copy +uint8:nullable [0, 1, None]@uint8 [0, 1, ]@Series[uint8[pyarrow]] [0, 1, ]@Series[uint8[pyarrow]] zero-copy +uint8:empty []@uint8 []@Series[uint8[pyarrow]] []@Series[uint8[pyarrow]] zero-copy +uint16:standard [0, 1, 65535]@uint16 [0, 1, 65535]@Series[uint16[pyarrow]] [0, 1, 65535]@Series[uint16[pyarrow]] zero-copy +uint16:nullable [0, 1, None]@uint16 [0, 1, ]@Series[uint16[pyarrow]] [0, 1, ]@Series[uint16[pyarrow]] zero-copy +uint16:empty []@uint16 []@Series[uint16[pyarrow]] []@Series[uint16[pyarrow]] zero-copy +uint32:standard [0, 1, 4294967295]@uint32 [0, 1, 4294967295]@Series[uint32[pyarrow]] [0, 1, 4294967295]@Series[uint32[pyarrow]] zero-copy +uint32:nullable [0, 1, None]@uint32 [0, 1, ]@Series[uint32[pyarrow]] [0, 1, ]@Series[uint32[pyarrow]] zero-copy +uint32:empty []@uint32 []@Series[uint32[pyarrow]] []@Series[uint32[pyarrow]] zero-copy +uint64:standard [0, 1, 18446744073709551615]@uint64 [0, 1, 18446744073709551615]@Series[uint64[pyarrow]] [0, 1, 18446744073709551615]@Series[uint64[pyarrow]] zero-copy +uint64:nullable [0, 1, None]@uint64 [0, 1, ]@Series[uint64[pyarrow]] [0, 1, ]@Series[uint64[pyarrow]] zero-copy +uint64:empty []@uint64 []@Series[uint64[pyarrow]] []@Series[uint64[pyarrow]] zero-copy +float32:standard [0.0, 1.5, -1.5]@float32 [0.0, 1.5, -1.5]@Series[float[pyarrow]] [0.0, 1.5, -1.5]@Series[float[pyarrow]] zero-copy +float32:nullable [0.0, 1.5, None]@float32 [0.0, 1.5, ]@Series[float[pyarrow]] [0.0, 1.5, ]@Series[float[pyarrow]] zero-copy +float32:empty []@float32 []@Series[float[pyarrow]] []@Series[float[pyarrow]] zero-copy +float64:standard [0.0, 1.5, -1.5]@float64 [0.0, 1.5, -1.5]@Series[double[pyarrow]] [0.0, 1.5, -1.5]@Series[double[pyarrow]] zero-copy +float64:nullable [0.0, 1.5, None]@float64 [0.0, 1.5, ]@Series[double[pyarrow]] [0.0, 1.5, ]@Series[double[pyarrow]] zero-copy +float64:special [nan, inf, -inf]@float64 [nan, inf, -inf]@Series[double[pyarrow]] [nan, inf, -inf]@Series[double[pyarrow]] zero-copy +float64:empty []@float64 []@Series[double[pyarrow]] []@Series[double[pyarrow]] zero-copy +bool:standard [True, False, True]@bool [True, False, True]@Series[bool[pyarrow]] [True, False, True]@Series[bool[pyarrow]] zero-copy +bool:nullable [True, False, None]@bool [True, False, ]@Series[bool[pyarrow]] [True, False, ]@Series[bool[pyarrow]] zero-copy +bool:empty []@bool []@Series[bool[pyarrow]] []@Series[bool[pyarrow]] zero-copy +string:standard [hello, world, ]@string ['hello', 'world', '']@Series[string[pyarrow]] ['hello', 'world', '']@Series[string[pyarrow]] zero-copy +string:nullable [hello, None, world]@string ['hello', , 'world']@Series[string[pyarrow]] ['hello', , 'world']@Series[string[pyarrow]] zero-copy +string:empty []@string []@Series[string[pyarrow]] []@Series[string[pyarrow]] zero-copy +large_string:standard [hello, world]@large_string ['hello', 'world']@Series[large_string[pyarrow]] ['hello', 'world']@Series[large_string[pyarrow]] zero-copy +large_string:nullable [hello, None]@large_string ['hello', ]@Series[large_string[pyarrow]] ['hello', ]@Series[large_string[pyarrow]] zero-copy +large_string:empty []@large_string []@Series[large_string[pyarrow]] []@Series[large_string[pyarrow]] zero-copy +binary:standard [b'hello', b'world']@binary [b'hello', b'world']@Series[binary[pyarrow]] [b'hello', b'world']@Series[binary[pyarrow]] zero-copy +binary:nullable [b'hello', None]@binary [b'hello', ]@Series[binary[pyarrow]] [b'hello', ]@Series[binary[pyarrow]] zero-copy +binary:empty []@binary []@Series[binary[pyarrow]] []@Series[binary[pyarrow]] zero-copy +large_binary:standard [b'hello', b'world']@large_binary [b'hello', b'world']@Series[large_binary[pyarrow]] [b'hello', b'world']@Series[large_binary[pyarrow]] zero-copy +large_binary:nullable [b'hello', None]@large_binary [b'hello', ]@Series[large_binary[pyarrow]] [b'hello', ]@Series[large_binary[pyarrow]] zero-copy +large_binary:empty []@large_binary []@Series[large_binary[pyarrow]] []@Series[large_binary[pyarrow]] zero-copy +decimal128:standard [1.23, 4.56, -7.89]@decimal128(5, 2) [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[decimal128(5, 2)[pyarrow]] [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[decimal128(5, 2)[pyarrow]] zero-copy +decimal128:nullable [1.23, None, 4.56]@decimal128(5, 2) [Decimal('1.23'), , Decimal('4.56')]@Series[decimal128(5, 2)[pyarrow]] [Decimal('1.23'), , Decimal('4.56')]@Series[decimal128(5, 2)[pyarrow]] zero-copy +decimal128:empty []@decimal128(5, 2) []@Series[decimal128(5, 2)[pyarrow]] []@Series[decimal128(5, 2)[pyarrow]] zero-copy +date32:standard [2024-01-01, 2024-06-15]@date32[day] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date32[day][pyarrow]] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date32[day][pyarrow]] zero-copy +date32:nullable [2024-01-01, None]@date32[day] [datetime.date(2024, 1, 1), ]@Series[date32[day][pyarrow]] [datetime.date(2024, 1, 1), ]@Series[date32[day][pyarrow]] zero-copy +date32:empty []@date32[day] []@Series[date32[day][pyarrow]] []@Series[date32[day][pyarrow]] zero-copy +date64:standard [2024-01-01, 2024-06-15]@date64[ms] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date64[ms][pyarrow]] [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date64[ms][pyarrow]] zero-copy +date64:nullable [2024-01-01, None]@date64[ms] [datetime.date(2024, 1, 1), ]@Series[date64[ms][pyarrow]] [datetime.date(2024, 1, 1), ]@Series[date64[ms][pyarrow]] zero-copy +date64:empty []@date64[ms] []@Series[date64[ms][pyarrow]] []@Series[date64[ms][pyarrow]] zero-copy +timestamp[s]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[s] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[s][pyarrow]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[s][pyarrow]] zero-copy +timestamp[s]:nullable [2024-01-01 12:00:00, None]@timestamp[s] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[s][pyarrow]] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[s][pyarrow]] zero-copy +timestamp[s]:empty []@timestamp[s] []@Series[timestamp[s][pyarrow]] []@Series[timestamp[s][pyarrow]] zero-copy +timestamp[ms]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ms] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ms][pyarrow]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ms][pyarrow]] zero-copy +timestamp[ms]:nullable [2024-01-01 12:00:00, None]@timestamp[ms] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ms][pyarrow]] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ms][pyarrow]] zero-copy +timestamp[ms]:empty []@timestamp[ms] []@Series[timestamp[ms][pyarrow]] []@Series[timestamp[ms][pyarrow]] zero-copy +timestamp[us]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[us] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[us][pyarrow]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[us][pyarrow]] zero-copy +timestamp[us]:nullable [2024-01-01 12:00:00, None]@timestamp[us] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[us][pyarrow]] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[us][pyarrow]] zero-copy +timestamp[us]:empty []@timestamp[us] []@Series[timestamp[us][pyarrow]] []@Series[timestamp[us][pyarrow]] zero-copy +timestamp[ns]:standard [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ns] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ns][pyarrow]] [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ns][pyarrow]] zero-copy +timestamp[ns]:nullable [2024-01-01 12:00:00, None]@timestamp[ns] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ns][pyarrow]] [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ns][pyarrow]] zero-copy +timestamp[ns]:empty []@timestamp[ns] []@Series[timestamp[ns][pyarrow]] []@Series[timestamp[ns][pyarrow]] zero-copy +timestamp[us,tz=UTC]:standard [2024-01-01 12:00:00+00:00, 2024-06-15 18:30:00+00:00]@timestamp[us, tz=UTC] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[timestamp[us, tz=UTC][pyarrow]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[timestamp[us, tz=UTC][pyarrow]] zero-copy +timestamp[us,tz=UTC]:nullable [2024-01-01 12:00:00+00:00, None]@timestamp[us, tz=UTC] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), ]@Series[timestamp[us, tz=UTC][pyarrow]] [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), ]@Series[timestamp[us, tz=UTC][pyarrow]] zero-copy +timestamp[us,tz=UTC]:empty []@timestamp[us, tz=UTC] []@Series[timestamp[us, tz=UTC][pyarrow]] []@Series[timestamp[us, tz=UTC][pyarrow]] zero-copy +duration[s]:standard [1 day, 0:00:00, 2:30:00]@duration[s] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[s][pyarrow]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[s][pyarrow]] zero-copy +duration[s]:nullable [1 day, 0:00:00, None]@duration[s] [Timedelta('1 days 00:00:00'), ]@Series[duration[s][pyarrow]] [Timedelta('1 days 00:00:00'), ]@Series[duration[s][pyarrow]] zero-copy +duration[s]:empty []@duration[s] []@Series[duration[s][pyarrow]] []@Series[duration[s][pyarrow]] zero-copy +duration[ms]:standard [1 day, 0:00:00, 2:30:00]@duration[ms] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ms][pyarrow]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ms][pyarrow]] zero-copy +duration[ms]:nullable [1 day, 0:00:00, None]@duration[ms] [Timedelta('1 days 00:00:00'), ]@Series[duration[ms][pyarrow]] [Timedelta('1 days 00:00:00'), ]@Series[duration[ms][pyarrow]] zero-copy +duration[ms]:empty []@duration[ms] []@Series[duration[ms][pyarrow]] []@Series[duration[ms][pyarrow]] zero-copy +duration[us]:standard [1 day, 0:00:00, 2:30:00]@duration[us] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[us][pyarrow]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[us][pyarrow]] zero-copy +duration[us]:nullable [1 day, 0:00:00, None]@duration[us] [Timedelta('1 days 00:00:00'), ]@Series[duration[us][pyarrow]] [Timedelta('1 days 00:00:00'), ]@Series[duration[us][pyarrow]] zero-copy +duration[us]:empty []@duration[us] []@Series[duration[us][pyarrow]] []@Series[duration[us][pyarrow]] zero-copy +duration[ns]:standard [1 days 00:00:00, 0 days 02:30:00]@duration[ns] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ns][pyarrow]] [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ns][pyarrow]] zero-copy +duration[ns]:nullable [1 days 00:00:00, None]@duration[ns] [Timedelta('1 days 00:00:00'), ]@Series[duration[ns][pyarrow]] [Timedelta('1 days 00:00:00'), ]@Series[duration[ns][pyarrow]] zero-copy +duration[ns]:empty []@duration[ns] []@Series[duration[ns][pyarrow]] []@Series[duration[ns][pyarrow]] zero-copy +time32[s]:standard [12:30:00, 18:45:30]@time32[s] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[s][pyarrow]] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[s][pyarrow]] zero-copy +time32[s]:nullable [12:30:00, None]@time32[s] [datetime.time(12, 30), ]@Series[time32[s][pyarrow]] [datetime.time(12, 30), ]@Series[time32[s][pyarrow]] zero-copy +time32[s]:empty []@time32[s] []@Series[time32[s][pyarrow]] []@Series[time32[s][pyarrow]] zero-copy +time32[ms]:standard [12:30:00, 18:45:30]@time32[ms] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[ms][pyarrow]] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[ms][pyarrow]] zero-copy +time32[ms]:nullable [12:30:00, None]@time32[ms] [datetime.time(12, 30), ]@Series[time32[ms][pyarrow]] [datetime.time(12, 30), ]@Series[time32[ms][pyarrow]] zero-copy +time32[ms]:empty []@time32[ms] []@Series[time32[ms][pyarrow]] []@Series[time32[ms][pyarrow]] zero-copy +time64[us]:standard [12:30:00, 18:45:30]@time64[us] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[us][pyarrow]] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[us][pyarrow]] zero-copy +time64[us]:nullable [12:30:00, None]@time64[us] [datetime.time(12, 30), ]@Series[time64[us][pyarrow]] [datetime.time(12, 30), ]@Series[time64[us][pyarrow]] zero-copy +time64[us]:empty []@time64[us] []@Series[time64[us][pyarrow]] []@Series[time64[us][pyarrow]] zero-copy +time64[ns]:standard [12:30:00, 18:45:30]@time64[ns] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[ns][pyarrow]] [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[ns][pyarrow]] zero-copy +time64[ns]:nullable [12:30:00, None]@time64[ns] [datetime.time(12, 30), ]@Series[time64[ns][pyarrow]] [datetime.time(12, 30), ]@Series[time64[ns][pyarrow]] zero-copy +time64[ns]:empty []@time64[ns] []@Series[time64[ns][pyarrow]] []@Series[time64[ns][pyarrow]] zero-copy +null:standard [None, None, None]@null [, , ]@Series[null[pyarrow]] [, , ]@Series[null[pyarrow]] zero-copy +null:empty []@null []@Series[null[pyarrow]] []@Series[null[pyarrow]] zero-copy +list:standard [[1, 2], [3, 4, 5]]@list [[1, 2], [3, 4, 5]]@Series[list[pyarrow]] [[1, 2], [3, 4, 5]]@Series[list[pyarrow]] zero-copy +list:nullable [[1, 2], None, [3]]@list [[1, 2], , [3]]@Series[list[pyarrow]] [[1, 2], , [3]]@Series[list[pyarrow]] zero-copy +list:empty []@list []@Series[list[pyarrow]] []@Series[list[pyarrow]] zero-copy +list:standard [['a', 'b'], ['c']]@list [['a', 'b'], ['c']]@Series[list[pyarrow]] [['a', 'b'], ['c']]@Series[list[pyarrow]] zero-copy +large_list:standard [[1, 2], [3, 4]]@large_list [[1, 2], [3, 4]]@Series[large_list[pyarrow]] [[1, 2], [3, 4]]@Series[large_list[pyarrow]] zero-copy +large_list:empty []@large_list []@Series[large_list[pyarrow]] []@Series[large_list[pyarrow]] zero-copy +fixed_size_list[3]:standard [[1, 2, 3], [4, 5, 6]]@fixed_size_list[3] [[1, 2, 3], [4, 5, 6]]@Series[fixed_size_list[3][pyarrow]] [[1, 2, 3], [4, 5, 6]]@Series[fixed_size_list[3][pyarrow]] zero-copy +fixed_size_list[3]:empty []@fixed_size_list[3] []@Series[fixed_size_list[3][pyarrow]] []@Series[fixed_size_list[3][pyarrow]] zero-copy +struct:standard [[('x', 1), ('y', 'a')], [('x', 2), ('y', 'b')]]@struct [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[struct[pyarrow]] [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[struct[pyarrow]] zero-copy +struct:nullable [[('x', 1), ('y', 'a')], None]@struct [{'x': 1, 'y': 'a'}, ]@Series[struct[pyarrow]] [{'x': 1, 'y': 'a'}, ]@Series[struct[pyarrow]] zero-copy +struct:empty []@struct []@Series[struct[pyarrow]] []@Series[struct[pyarrow]] zero-copy +map:standard [[('a', 1), ('b', 2)], [('c', 3)]]@map [[('a', 1), ('b', 2)], [('c', 3)]]@Series[map[pyarrow]] [[('a', 1), ('b', 2)], [('c', 3)]]@Series[map[pyarrow]] zero-copy +map:empty []@map []@Series[map[pyarrow]] []@Series[map[pyarrow]] zero-copy +list>:standard [[[1, 2], [3]], [[4, 5, 6]]]@list> [[[1, 2], [3]], [[4, 5, 6]]]@Series[list>[pyarrow]] [[[1, 2], [3]], [[4, 5, 6]]]@Series[list>[pyarrow]] zero-copy +list:standard [[{'x': 1}, {'x': 2}], [{'x': 3}]]@list> [[{'x': 1}, {'x': 2}], [{'x': 3}]]@Series[list>[pyarrow]] [[{'x': 1}, {'x': 2}], [{'x': 3}]]@Series[list>[pyarrow]] zero-copy +list>:standard [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@list> [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@Series[list>[pyarrow]] [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@Series[list>[pyarrow]] zero-copy +struct:standard [[('outer', {'inner': 1})], [('outer', {'inner': 2})]]@struct> [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[struct>[pyarrow]] [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[struct>[pyarrow]] zero-copy +struct>:standard [[('items', [1, 2, 3])], [('items', [4, 5])]]@struct> [{'items': [1, 2, 3]}, {'items': [4, 5]}]@Series[struct>[pyarrow]] [{'items': [1, 2, 3]}, {'items': [4, 5]}]@Series[struct>[pyarrow]] zero-copy +struct>:standard [[('mapping', [('a', 1)])], [('mapping', [('b', 2)])]]@struct> [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[struct>[pyarrow]] [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[struct>[pyarrow]] zero-copy +map>:standard [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@map> [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@Series[map>[pyarrow]] [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@Series[map>[pyarrow]] zero-copy +map:standard [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@map> [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[map>[pyarrow]] [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[map>[pyarrow]] zero-copy +map>:standard [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@map> [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[map>[pyarrow]] [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[map>[pyarrow]] zero-copy +dictionary:standard [a, b, a, b]@dictionary ['a', 'b', 'a', 'b']@Series[dictionary[pyarrow]] ['a', 'b', 'a', 'b']@Series[dictionary[pyarrow]] zero-copy +dictionary:nullable [a, b, None, a]@dictionary ['a', 'b', , 'a']@Series[dictionary[pyarrow]] ['a', 'b', , 'a']@Series[dictionary[pyarrow]] zero-copy +dictionary:empty []@dictionary []@Series[dictionary[pyarrow]] []@Series[dictionary[pyarrow]] zero-copy +int64:sliced [2, 3, 4]@int64 [2, 3, 4]@Series[int64[pyarrow]] [2, 3, 4]@Series[int64[pyarrow]] zero-copy +int64:sliced-with-null [2, None, 4]@int64 [2, , 4]@Series[int64[pyarrow]] [2, , 4]@Series[int64[pyarrow]] zero-copy +int64:single-chunk [1, 2, 3]@int64 [1, 2, 3]@Series[int64[pyarrow]] [1, 2, 3]@Series[int64[pyarrow]] zero-copy +int64:multi-chunk [1, 2, 3, 4]@int64 [1, 2, 3, 4]@Series[int64[pyarrow]] [1, 2, 3, 4]@Series[int64[pyarrow]] zero-copy diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.md b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.md new file mode 100644 index 000000000000..ac177461a2ed --- /dev/null +++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed.md @@ -0,0 +1,127 @@ +| test case | pyarrow array | types_mapper=pd.ArrowDtype, zero_copy_only=False | types_mapper=pd.ArrowDtype, zero_copy_only=True | verified zero-copy | +|----------------------------------------|-----------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|----------------------| +| int8:standard | [0, 1, -1, 127, -128]@int8 | [0, 1, -1, 127, -128]@Series[int8[pyarrow]] | [0, 1, -1, 127, -128]@Series[int8[pyarrow]] | zero-copy | +| int8:nullable | [0, 1, None]@int8 | [0, 1, ]@Series[int8[pyarrow]] | [0, 1, ]@Series[int8[pyarrow]] | zero-copy | +| int8:empty | []@int8 | []@Series[int8[pyarrow]] | []@Series[int8[pyarrow]] | zero-copy | +| int16:standard | [0, 1, -1, 32767, -32768]@int16 | [0, 1, -1, 32767, -32768]@Series[int16[pyarrow]] | [0, 1, -1, 32767, -32768]@Series[int16[pyarrow]] | zero-copy | +| int16:nullable | [0, 1, None]@int16 | [0, 1, ]@Series[int16[pyarrow]] | [0, 1, ]@Series[int16[pyarrow]] | zero-copy | +| int16:empty | []@int16 | []@Series[int16[pyarrow]] | []@Series[int16[pyarrow]] | zero-copy | +| int32:standard | [0, 1, -1, 2147483647, -2147483648]@int32 | [0, 1, -1, 2147483647, -2147483648]@Series[int32[pyarrow]] | [0, 1, -1, 2147483647, -2147483648]@Series[int32[pyarrow]] | zero-copy | +| int32:nullable | [0, 1, None]@int32 | [0, 1, ]@Series[int32[pyarrow]] | [0, 1, ]@Series[int32[pyarrow]] | zero-copy | +| int32:empty | []@int32 | []@Series[int32[pyarrow]] | []@Series[int32[pyarrow]] | zero-copy | +| int64:standard | [0, 1, -1, 9223372036854775807, -9223372036854775808]@int64 | [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64[pyarrow]] | [0, 1, -1, 9223372036854775807, -9223372036854775808]@Series[int64[pyarrow]] | zero-copy | +| int64:nullable | [0, 1, None]@int64 | [0, 1, ]@Series[int64[pyarrow]] | [0, 1, ]@Series[int64[pyarrow]] | zero-copy | +| int64:empty | []@int64 | []@Series[int64[pyarrow]] | []@Series[int64[pyarrow]] | zero-copy | +| uint8:standard | [0, 1, 255]@uint8 | [0, 1, 255]@Series[uint8[pyarrow]] | [0, 1, 255]@Series[uint8[pyarrow]] | zero-copy | +| uint8:nullable | [0, 1, None]@uint8 | [0, 1, ]@Series[uint8[pyarrow]] | [0, 1, ]@Series[uint8[pyarrow]] | zero-copy | +| uint8:empty | []@uint8 | []@Series[uint8[pyarrow]] | []@Series[uint8[pyarrow]] | zero-copy | +| uint16:standard | [0, 1, 65535]@uint16 | [0, 1, 65535]@Series[uint16[pyarrow]] | [0, 1, 65535]@Series[uint16[pyarrow]] | zero-copy | +| uint16:nullable | [0, 1, None]@uint16 | [0, 1, ]@Series[uint16[pyarrow]] | [0, 1, ]@Series[uint16[pyarrow]] | zero-copy | +| uint16:empty | []@uint16 | []@Series[uint16[pyarrow]] | []@Series[uint16[pyarrow]] | zero-copy | +| uint32:standard | [0, 1, 4294967295]@uint32 | [0, 1, 4294967295]@Series[uint32[pyarrow]] | [0, 1, 4294967295]@Series[uint32[pyarrow]] | zero-copy | +| uint32:nullable | [0, 1, None]@uint32 | [0, 1, ]@Series[uint32[pyarrow]] | [0, 1, ]@Series[uint32[pyarrow]] | zero-copy | +| uint32:empty | []@uint32 | []@Series[uint32[pyarrow]] | []@Series[uint32[pyarrow]] | zero-copy | +| uint64:standard | [0, 1, 18446744073709551615]@uint64 | [0, 1, 18446744073709551615]@Series[uint64[pyarrow]] | [0, 1, 18446744073709551615]@Series[uint64[pyarrow]] | zero-copy | +| uint64:nullable | [0, 1, None]@uint64 | [0, 1, ]@Series[uint64[pyarrow]] | [0, 1, ]@Series[uint64[pyarrow]] | zero-copy | +| uint64:empty | []@uint64 | []@Series[uint64[pyarrow]] | []@Series[uint64[pyarrow]] | zero-copy | +| float32:standard | [0.0, 1.5, -1.5]@float32 | [0.0, 1.5, -1.5]@Series[float[pyarrow]] | [0.0, 1.5, -1.5]@Series[float[pyarrow]] | zero-copy | +| float32:nullable | [0.0, 1.5, None]@float32 | [0.0, 1.5, ]@Series[float[pyarrow]] | [0.0, 1.5, ]@Series[float[pyarrow]] | zero-copy | +| float32:empty | []@float32 | []@Series[float[pyarrow]] | []@Series[float[pyarrow]] | zero-copy | +| float64:standard | [0.0, 1.5, -1.5]@float64 | [0.0, 1.5, -1.5]@Series[double[pyarrow]] | [0.0, 1.5, -1.5]@Series[double[pyarrow]] | zero-copy | +| float64:nullable | [0.0, 1.5, None]@float64 | [0.0, 1.5, ]@Series[double[pyarrow]] | [0.0, 1.5, ]@Series[double[pyarrow]] | zero-copy | +| float64:special | [nan, inf, -inf]@float64 | [nan, inf, -inf]@Series[double[pyarrow]] | [nan, inf, -inf]@Series[double[pyarrow]] | zero-copy | +| float64:empty | []@float64 | []@Series[double[pyarrow]] | []@Series[double[pyarrow]] | zero-copy | +| bool:standard | [True, False, True]@bool | [True, False, True]@Series[bool[pyarrow]] | [True, False, True]@Series[bool[pyarrow]] | zero-copy | +| bool:nullable | [True, False, None]@bool | [True, False, ]@Series[bool[pyarrow]] | [True, False, ]@Series[bool[pyarrow]] | zero-copy | +| bool:empty | []@bool | []@Series[bool[pyarrow]] | []@Series[bool[pyarrow]] | zero-copy | +| string:standard | [hello, world, ]@string | ['hello', 'world', '']@Series[string[pyarrow]] | ['hello', 'world', '']@Series[string[pyarrow]] | zero-copy | +| string:nullable | [hello, None, world]@string | ['hello', , 'world']@Series[string[pyarrow]] | ['hello', , 'world']@Series[string[pyarrow]] | zero-copy | +| string:empty | []@string | []@Series[string[pyarrow]] | []@Series[string[pyarrow]] | zero-copy | +| large_string:standard | [hello, world]@large_string | ['hello', 'world']@Series[large_string[pyarrow]] | ['hello', 'world']@Series[large_string[pyarrow]] | zero-copy | +| large_string:nullable | [hello, None]@large_string | ['hello', ]@Series[large_string[pyarrow]] | ['hello', ]@Series[large_string[pyarrow]] | zero-copy | +| large_string:empty | []@large_string | []@Series[large_string[pyarrow]] | []@Series[large_string[pyarrow]] | zero-copy | +| binary:standard | [b'hello', b'world']@binary | [b'hello', b'world']@Series[binary[pyarrow]] | [b'hello', b'world']@Series[binary[pyarrow]] | zero-copy | +| binary:nullable | [b'hello', None]@binary | [b'hello', ]@Series[binary[pyarrow]] | [b'hello', ]@Series[binary[pyarrow]] | zero-copy | +| binary:empty | []@binary | []@Series[binary[pyarrow]] | []@Series[binary[pyarrow]] | zero-copy | +| large_binary:standard | [b'hello', b'world']@large_binary | [b'hello', b'world']@Series[large_binary[pyarrow]] | [b'hello', b'world']@Series[large_binary[pyarrow]] | zero-copy | +| large_binary:nullable | [b'hello', None]@large_binary | [b'hello', ]@Series[large_binary[pyarrow]] | [b'hello', ]@Series[large_binary[pyarrow]] | zero-copy | +| large_binary:empty | []@large_binary | []@Series[large_binary[pyarrow]] | []@Series[large_binary[pyarrow]] | zero-copy | +| decimal128:standard | [1.23, 4.56, -7.89]@decimal128(5, 2) | [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[decimal128(5, 2)[pyarrow]] | [Decimal('1.23'), Decimal('4.56'), Decimal('-7.89')]@Series[decimal128(5, 2)[pyarrow]] | zero-copy | +| decimal128:nullable | [1.23, None, 4.56]@decimal128(5, 2) | [Decimal('1.23'), , Decimal('4.56')]@Series[decimal128(5, 2)[pyarrow]] | [Decimal('1.23'), , Decimal('4.56')]@Series[decimal128(5, 2)[pyarrow]] | zero-copy | +| decimal128:empty | []@decimal128(5, 2) | []@Series[decimal128(5, 2)[pyarrow]] | []@Series[decimal128(5, 2)[pyarrow]] | zero-copy | +| date32:standard | [2024-01-01, 2024-06-15]@date32[day] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date32[day][pyarrow]] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date32[day][pyarrow]] | zero-copy | +| date32:nullable | [2024-01-01, None]@date32[day] | [datetime.date(2024, 1, 1), ]@Series[date32[day][pyarrow]] | [datetime.date(2024, 1, 1), ]@Series[date32[day][pyarrow]] | zero-copy | +| date32:empty | []@date32[day] | []@Series[date32[day][pyarrow]] | []@Series[date32[day][pyarrow]] | zero-copy | +| date64:standard | [2024-01-01, 2024-06-15]@date64[ms] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date64[ms][pyarrow]] | [datetime.date(2024, 1, 1), datetime.date(2024, 6, 15)]@Series[date64[ms][pyarrow]] | zero-copy | +| date64:nullable | [2024-01-01, None]@date64[ms] | [datetime.date(2024, 1, 1), ]@Series[date64[ms][pyarrow]] | [datetime.date(2024, 1, 1), ]@Series[date64[ms][pyarrow]] | zero-copy | +| date64:empty | []@date64[ms] | []@Series[date64[ms][pyarrow]] | []@Series[date64[ms][pyarrow]] | zero-copy | +| timestamp[s]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[s] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[s][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[s][pyarrow]] | zero-copy | +| timestamp[s]:nullable | [2024-01-01 12:00:00, None]@timestamp[s] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[s][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[s][pyarrow]] | zero-copy | +| timestamp[s]:empty | []@timestamp[s] | []@Series[timestamp[s][pyarrow]] | []@Series[timestamp[s][pyarrow]] | zero-copy | +| timestamp[ms]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ms] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ms][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ms][pyarrow]] | zero-copy | +| timestamp[ms]:nullable | [2024-01-01 12:00:00, None]@timestamp[ms] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ms][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ms][pyarrow]] | zero-copy | +| timestamp[ms]:empty | []@timestamp[ms] | []@Series[timestamp[ms][pyarrow]] | []@Series[timestamp[ms][pyarrow]] | zero-copy | +| timestamp[us]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[us] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[us][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[us][pyarrow]] | zero-copy | +| timestamp[us]:nullable | [2024-01-01 12:00:00, None]@timestamp[us] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[us][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[us][pyarrow]] | zero-copy | +| timestamp[us]:empty | []@timestamp[us] | []@Series[timestamp[us][pyarrow]] | []@Series[timestamp[us][pyarrow]] | zero-copy | +| timestamp[ns]:standard | [2024-01-01 12:00:00, 2024-06-15 18:30:00]@timestamp[ns] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ns][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), Timestamp('2024-06-15 18:30:00')]@Series[timestamp[ns][pyarrow]] | zero-copy | +| timestamp[ns]:nullable | [2024-01-01 12:00:00, None]@timestamp[ns] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ns][pyarrow]] | [Timestamp('2024-01-01 12:00:00'), ]@Series[timestamp[ns][pyarrow]] | zero-copy | +| timestamp[ns]:empty | []@timestamp[ns] | []@Series[timestamp[ns][pyarrow]] | []@Series[timestamp[ns][pyarrow]] | zero-copy | +| timestamp[us,tz=UTC]:standard | [2024-01-01 12:00:00+00:00, 2024-06-15 18:30:00+00:00]@timestamp[us, tz=UTC] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[timestamp[us, tz=UTC][pyarrow]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), Timestamp('2024-06-15 18:30:00+0000', tz='UTC')]@Series[timestamp[us, tz=UTC][pyarrow]] | zero-copy | +| timestamp[us,tz=UTC]:nullable | [2024-01-01 12:00:00+00:00, None]@timestamp[us, tz=UTC] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), ]@Series[timestamp[us, tz=UTC][pyarrow]] | [Timestamp('2024-01-01 12:00:00+0000', tz='UTC'), ]@Series[timestamp[us, tz=UTC][pyarrow]] | zero-copy | +| timestamp[us,tz=UTC]:empty | []@timestamp[us, tz=UTC] | []@Series[timestamp[us, tz=UTC][pyarrow]] | []@Series[timestamp[us, tz=UTC][pyarrow]] | zero-copy | +| duration[s]:standard | [1 day, 0:00:00, 2:30:00]@duration[s] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[s][pyarrow]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[s][pyarrow]] | zero-copy | +| duration[s]:nullable | [1 day, 0:00:00, None]@duration[s] | [Timedelta('1 days 00:00:00'), ]@Series[duration[s][pyarrow]] | [Timedelta('1 days 00:00:00'), ]@Series[duration[s][pyarrow]] | zero-copy | +| duration[s]:empty | []@duration[s] | []@Series[duration[s][pyarrow]] | []@Series[duration[s][pyarrow]] | zero-copy | +| duration[ms]:standard | [1 day, 0:00:00, 2:30:00]@duration[ms] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ms][pyarrow]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ms][pyarrow]] | zero-copy | +| duration[ms]:nullable | [1 day, 0:00:00, None]@duration[ms] | [Timedelta('1 days 00:00:00'), ]@Series[duration[ms][pyarrow]] | [Timedelta('1 days 00:00:00'), ]@Series[duration[ms][pyarrow]] | zero-copy | +| duration[ms]:empty | []@duration[ms] | []@Series[duration[ms][pyarrow]] | []@Series[duration[ms][pyarrow]] | zero-copy | +| duration[us]:standard | [1 day, 0:00:00, 2:30:00]@duration[us] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[us][pyarrow]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[us][pyarrow]] | zero-copy | +| duration[us]:nullable | [1 day, 0:00:00, None]@duration[us] | [Timedelta('1 days 00:00:00'), ]@Series[duration[us][pyarrow]] | [Timedelta('1 days 00:00:00'), ]@Series[duration[us][pyarrow]] | zero-copy | +| duration[us]:empty | []@duration[us] | []@Series[duration[us][pyarrow]] | []@Series[duration[us][pyarrow]] | zero-copy | +| duration[ns]:standard | [1 days 00:00:00, 0 days 02:30:00]@duration[ns] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ns][pyarrow]] | [Timedelta('1 days 00:00:00'), Timedelta('0 days 02:30:00')]@Series[duration[ns][pyarrow]] | zero-copy | +| duration[ns]:nullable | [1 days 00:00:00, None]@duration[ns] | [Timedelta('1 days 00:00:00'), ]@Series[duration[ns][pyarrow]] | [Timedelta('1 days 00:00:00'), ]@Series[duration[ns][pyarrow]] | zero-copy | +| duration[ns]:empty | []@duration[ns] | []@Series[duration[ns][pyarrow]] | []@Series[duration[ns][pyarrow]] | zero-copy | +| time32[s]:standard | [12:30:00, 18:45:30]@time32[s] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[s][pyarrow]] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[s][pyarrow]] | zero-copy | +| time32[s]:nullable | [12:30:00, None]@time32[s] | [datetime.time(12, 30), ]@Series[time32[s][pyarrow]] | [datetime.time(12, 30), ]@Series[time32[s][pyarrow]] | zero-copy | +| time32[s]:empty | []@time32[s] | []@Series[time32[s][pyarrow]] | []@Series[time32[s][pyarrow]] | zero-copy | +| time32[ms]:standard | [12:30:00, 18:45:30]@time32[ms] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[ms][pyarrow]] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time32[ms][pyarrow]] | zero-copy | +| time32[ms]:nullable | [12:30:00, None]@time32[ms] | [datetime.time(12, 30), ]@Series[time32[ms][pyarrow]] | [datetime.time(12, 30), ]@Series[time32[ms][pyarrow]] | zero-copy | +| time32[ms]:empty | []@time32[ms] | []@Series[time32[ms][pyarrow]] | []@Series[time32[ms][pyarrow]] | zero-copy | +| time64[us]:standard | [12:30:00, 18:45:30]@time64[us] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[us][pyarrow]] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[us][pyarrow]] | zero-copy | +| time64[us]:nullable | [12:30:00, None]@time64[us] | [datetime.time(12, 30), ]@Series[time64[us][pyarrow]] | [datetime.time(12, 30), ]@Series[time64[us][pyarrow]] | zero-copy | +| time64[us]:empty | []@time64[us] | []@Series[time64[us][pyarrow]] | []@Series[time64[us][pyarrow]] | zero-copy | +| time64[ns]:standard | [12:30:00, 18:45:30]@time64[ns] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[ns][pyarrow]] | [datetime.time(12, 30), datetime.time(18, 45, 30)]@Series[time64[ns][pyarrow]] | zero-copy | +| time64[ns]:nullable | [12:30:00, None]@time64[ns] | [datetime.time(12, 30), ]@Series[time64[ns][pyarrow]] | [datetime.time(12, 30), ]@Series[time64[ns][pyarrow]] | zero-copy | +| time64[ns]:empty | []@time64[ns] | []@Series[time64[ns][pyarrow]] | []@Series[time64[ns][pyarrow]] | zero-copy | +| null:standard | [None, None, None]@null | [, , ]@Series[null[pyarrow]] | [, , ]@Series[null[pyarrow]] | zero-copy | +| null:empty | []@null | []@Series[null[pyarrow]] | []@Series[null[pyarrow]] | zero-copy | +| list:standard | [[1, 2], [3, 4, 5]]@list | [[1, 2], [3, 4, 5]]@Series[list[pyarrow]] | [[1, 2], [3, 4, 5]]@Series[list[pyarrow]] | zero-copy | +| list:nullable | [[1, 2], None, [3]]@list | [[1, 2], , [3]]@Series[list[pyarrow]] | [[1, 2], , [3]]@Series[list[pyarrow]] | zero-copy | +| list:empty | []@list | []@Series[list[pyarrow]] | []@Series[list[pyarrow]] | zero-copy | +| list:standard | [['a', 'b'], ['c']]@list | [['a', 'b'], ['c']]@Series[list[pyarrow]] | [['a', 'b'], ['c']]@Series[list[pyarrow]] | zero-copy | +| large_list:standard | [[1, 2], [3, 4]]@large_list | [[1, 2], [3, 4]]@Series[large_list[pyarrow]] | [[1, 2], [3, 4]]@Series[large_list[pyarrow]] | zero-copy | +| large_list:empty | []@large_list | []@Series[large_list[pyarrow]] | []@Series[large_list[pyarrow]] | zero-copy | +| fixed_size_list[3]:standard | [[1, 2, 3], [4, 5, 6]]@fixed_size_list[3] | [[1, 2, 3], [4, 5, 6]]@Series[fixed_size_list[3][pyarrow]] | [[1, 2, 3], [4, 5, 6]]@Series[fixed_size_list[3][pyarrow]] | zero-copy | +| fixed_size_list[3]:empty | []@fixed_size_list[3] | []@Series[fixed_size_list[3][pyarrow]] | []@Series[fixed_size_list[3][pyarrow]] | zero-copy | +| struct:standard | [[('x', 1), ('y', 'a')], [('x', 2), ('y', 'b')]]@struct | [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[struct[pyarrow]] | [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]@Series[struct[pyarrow]] | zero-copy | +| struct:nullable | [[('x', 1), ('y', 'a')], None]@struct | [{'x': 1, 'y': 'a'}, ]@Series[struct[pyarrow]] | [{'x': 1, 'y': 'a'}, ]@Series[struct[pyarrow]] | zero-copy | +| struct:empty | []@struct | []@Series[struct[pyarrow]] | []@Series[struct[pyarrow]] | zero-copy | +| map:standard | [[('a', 1), ('b', 2)], [('c', 3)]]@map | [[('a', 1), ('b', 2)], [('c', 3)]]@Series[map[pyarrow]] | [[('a', 1), ('b', 2)], [('c', 3)]]@Series[map[pyarrow]] | zero-copy | +| map:empty | []@map | []@Series[map[pyarrow]] | []@Series[map[pyarrow]] | zero-copy | +| list>:standard | [[[1, 2], [3]], [[4, 5, 6]]]@list> | [[[1, 2], [3]], [[4, 5, 6]]]@Series[list>[pyarrow]] | [[[1, 2], [3]], [[4, 5, 6]]]@Series[list>[pyarrow]] | zero-copy | +| list:standard | [[{'x': 1}, {'x': 2}], [{'x': 3}]]@list> | [[{'x': 1}, {'x': 2}], [{'x': 3}]]@Series[list>[pyarrow]] | [[{'x': 1}, {'x': 2}], [{'x': 3}]]@Series[list>[pyarrow]] | zero-copy | +| list>:standard | [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@list> | [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@Series[list>[pyarrow]] | [[[('a', 1)], [('b', 2)]], [[('c', 3)]]]@Series[list>[pyarrow]] | zero-copy | +| struct:standard | [[('outer', {'inner': 1})], [('outer', {'inner': 2})]]@struct> | [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[struct>[pyarrow]] | [{'outer': {'inner': 1}}, {'outer': {'inner': 2}}]@Series[struct>[pyarrow]] | zero-copy | +| struct>:standard | [[('items', [1, 2, 3])], [('items', [4, 5])]]@struct> | [{'items': [1, 2, 3]}, {'items': [4, 5]}]@Series[struct>[pyarrow]] | [{'items': [1, 2, 3]}, {'items': [4, 5]}]@Series[struct>[pyarrow]] | zero-copy | +| struct>:standard | [[('mapping', [('a', 1)])], [('mapping', [('b', 2)])]]@struct> | [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[struct>[pyarrow]] | [{'mapping': [('a', 1)]}, {'mapping': [('b', 2)]}]@Series[struct>[pyarrow]] | zero-copy | +| map>:standard | [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@map> | [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@Series[map>[pyarrow]] | [[('a', [1, 2]), ('b', [3])], [('c', [4, 5, 6])]]@Series[map>[pyarrow]] | zero-copy | +| map:standard | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@map> | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[map>[pyarrow]] | [[('a', {'v': 1}), ('b', {'v': 2})], [('c', {'v': 3})]]@Series[map>[pyarrow]] | zero-copy | +| map>:standard | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@map> | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[map>[pyarrow]] | [[('a', [('x', 1)]), ('b', [('y', 2)])], [('c', [('z', 3)])]]@Series[map>[pyarrow]] | zero-copy | +| dictionary:standard | [a, b, a, b]@dictionary | ['a', 'b', 'a', 'b']@Series[dictionary[pyarrow]] | ['a', 'b', 'a', 'b']@Series[dictionary[pyarrow]] | zero-copy | +| dictionary:nullable | [a, b, None, a]@dictionary | ['a', 'b', , 'a']@Series[dictionary[pyarrow]] | ['a', 'b', , 'a']@Series[dictionary[pyarrow]] | zero-copy | +| dictionary:empty | []@dictionary | []@Series[dictionary[pyarrow]] | []@Series[dictionary[pyarrow]] | zero-copy | +| int64:sliced | [2, 3, 4]@int64 | [2, 3, 4]@Series[int64[pyarrow]] | [2, 3, 4]@Series[int64[pyarrow]] | zero-copy | +| int64:sliced-with-null | [2, None, 4]@int64 | [2, , 4]@Series[int64[pyarrow]] | [2, , 4]@Series[int64[pyarrow]] | zero-copy | +| int64:single-chunk | [1, 2, 3]@int64 | [1, 2, 3]@Series[int64[pyarrow]] | [1, 2, 3]@Series[int64[pyarrow]] | zero-copy | +| int64:multi-chunk | [1, 2, 3, 4]@int64 | [1, 2, 3, 4]@Series[int64[pyarrow]] | [1, 2, 3, 4]@Series[int64[pyarrow]] | zero-copy | \ No newline at end of file diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py index ed660f163aab..fcd9adce73e5 100644 --- a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py +++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py @@ -384,6 +384,13 @@ class PyArrowArrayToPandasZeroCopyTests(_PyArrowToPandasTestBase): so both golden files pin the same types, and appends the layout variants that only matter for zero-copy: sliced (offset) arrays and single- vs multi-chunk ChunkedArrays. + + A second test method repeats those rows with ``types_mapper=pd.ArrowDtype``, which + asks pandas to keep pointing at the Arrow buffers instead of materializing them + into NumPy -- avoiding the copy is the point of that backend. Its own golden file + records the result, so the two can be read side by side. PySpark takes this path + in ``ArrowArrayToPandasConversion.convert_numpy`` + (``python/pyspark/sql/conversion.py``). """ def _build_source_arrays(self): @@ -492,6 +499,49 @@ def compute_cell(row_name, col_name): overrides=overrides, ) + # Output columns for the Arrow-backed conversion. Both zero_copy_only states are + # recorded even though the flag is expected to make no difference here + COL_ARROW_ZERO_COPY_OFF = "types_mapper=pd.ArrowDtype, zero_copy_only=False" + COL_ARROW_ZERO_COPY_ON = "types_mapper=pd.ArrowDtype, zero_copy_only=True" + COL_ARROW_VERIFIED = "verified zero-copy" + + def test_to_pandas_zero_copy_only_arrow_backed(self): + """Test pa.Array.to_pandas(types_mapper=pd.ArrowDtype) against golden file.""" + sources = self._build_source_arrays() + row_names = list(sources.keys()) + col_names = [ + "pyarrow array", + self.COL_ARROW_ZERO_COPY_OFF, + self.COL_ARROW_ZERO_COPY_ON, + self.COL_ARROW_VERIFIED, + ] + + # Version-specific expected values go here, keyed by (row, col), when a + # newer pandas/PyArrow/NumPy legitimately changes a cell's output. + overrides: dict[tuple[str, str], str] = {} + + def compute_cell(row_name, col_name): + arr = sources[row_name] + if col_name == "pyarrow array": + return self.repr_value(arr, max_len=0) + elif col_name == self.COL_ARROW_ZERO_COPY_OFF: + return self._to_pandas_cell(arr, types_mapper=pd.ArrowDtype, zero_copy_only=False) + elif col_name == self.COL_ARROW_ZERO_COPY_ON: + return self._to_pandas_cell(arr, types_mapper=pd.ArrowDtype, zero_copy_only=True) + elif col_name == self.COL_ARROW_VERIFIED: + return self._verify_zero_copy(arr, types_mapper=pd.ArrowDtype) + else: + raise ValueError(f"unknown column: {col_name}") + + self.compare_or_generate_golden_matrix( + row_names=row_names, + col_names=col_names, + compute_cell=compute_cell, + golden_file_prefix="golden_pyarrow_arrow_to_pandas_zero_copy_arrow_backed", + index_name="test case", + overrides=overrides, + ) + if __name__ == "__main__": from pyspark.testing import main