Merge v1.4-andium into v1.5-variegata with breaking change fixes

.github/workflows/code_quality.yml

@@ -32,7 +32,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.9
+          python-version: "3.12"
 
       - name: pre-commit (cache)
         uses: actions/cache@v4

.github/workflows/coverage.yml

@@ -71,7 +71,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.9
+          python-version: 3.12
           enable-cache: true
           cache-suffix: -${{ github.workflow }}
 

.github/workflows/packaging_sdist.yml

@@ -59,7 +59,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Build sdist
         run: uv build --sdist

.github/workflows/packaging_wheels.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
+        python: [ cp310, cp311, cp312, cp313, cp314 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: windows-11-arm,   arch: ARM64,      cibw_system: win } # cibw requires ARM64 to be uppercase
@@ -47,7 +47,6 @@ jobs:
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
-          - { python: cp39,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
           - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
     runs-on: ${{ matrix.platform.os }}
     env:

.github/workflows/targeted_test.yml

@@ -19,7 +19,6 @@ on:
         required: true
         type: choice
         options:
-          - '3.9'
           - '3.10'
           - '3.11'
           - '3.12'

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018-2025 Stichting DuckDB Foundation
+Copyright 2018-2026 Stichting DuckDB Foundation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

adbc_driver_duckdb/__init__.py

@@ -20,7 +20,6 @@
 import enum
 import functools
 import importlib.util
-import typing
 
 import adbc_driver_manager
 
@@ -32,7 +31,7 @@ class StatementOptions(enum.Enum):
     BATCH_ROWS = "adbc.duckdb.query.batch_rows"
 
 
-def connect(path: typing.Optional[str] = None) -> adbc_driver_manager.AdbcDatabase:
+def connect(path: str | None = None) -> adbc_driver_manager.AdbcDatabase:
     """Create a low level ADBC connection to DuckDB."""
     if path is None:
         return adbc_driver_manager.AdbcDatabase(driver=driver_path(), entrypoint="duckdb_adbc_init")

adbc_driver_duckdb/dbapi.py

@@ -17,8 +17,6 @@
 
 """DBAPI 2.0-compatible facade for the ADBC DuckDB driver."""
 
-import typing
-
 import adbc_driver_manager
 import adbc_driver_manager.dbapi
 
@@ -91,7 +89,7 @@
 # Functions
 
 
-def connect(path: typing.Optional[str] = None, **kwargs) -> "Connection":
+def connect(path: str | None = None, **kwargs) -> "Connection":
     """Connect to DuckDB via ADBC."""
     db = None
     conn = None

duckdb/bytes_io_wrapper.py

@@ -34,7 +34,7 @@
 """
 
 from io import StringIO, TextIOBase
-from typing import Any, Union
+from typing import Any
 
 
 class BytesIOWrapper:
@@ -43,7 +43,7 @@ class BytesIOWrapper:
     Created for compat with pyarrow read_csv.
     """
 
-    def __init__(self, buffer: Union[StringIO, TextIOBase], encoding: str = "utf-8") -> None:  # noqa: D107
+    def __init__(self, buffer: StringIO | TextIOBase, encoding: str = "utf-8") -> None:  # noqa: D107
         self.buffer = buffer
         self.encoding = encoding
         # Because a character can be represented by more than 1 byte,
@@ -55,7 +55,7 @@ def __init__(self, buffer: Union[StringIO, TextIOBase], encoding: str = "utf-8")
     def __getattr__(self, attr: str) -> Any:  # noqa: D105, ANN401
         return getattr(self.buffer, attr)
 
-    def read(self, n: Union[int, None] = -1) -> bytes:  # noqa: D102
+    def read(self, n: int | None = -1) -> bytes:  # noqa: D102
         assert self.buffer is not None
         bytestring = self.buffer.read(n).encode(self.encoding)
         # When n=-1/n greater than remaining bytes: Read entire file/rest of file

duckdb/experimental/spark/_globals.py

@@ -33,6 +33,7 @@ def foo(arg=pyducdkb.spark._NoValue):
 
 __ALL__ = ["_NoValue"]
 
+from typing_extensions import Self
 
 # Disallow reloading this module so as to preserve the identities of the
 # classes defined here.
@@ -54,7 +55,7 @@ class _NoValueType:
 
     __instance = None
 
-    def __new__(cls) -> "_NoValueType":
+    def __new__(cls) -> Self:
         # ensure that only one instance exists
         if not cls.__instance:
             cls.__instance = super().__new__(cls)

duckdb/experimental/spark/_typing.py

@@ -16,16 +16,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from collections.abc import Iterable, Sized
-from typing import Callable, TypeVar, Union
+from collections.abc import Callable, Iterable, Sized
+from typing import Literal, TypeVar
 
 from numpy import float32, float64, int32, int64, ndarray
-from typing_extensions import Literal, Protocol, Self
+from typing_extensions import Protocol, Self
 
 F = TypeVar("F", bound=Callable)
 T_co = TypeVar("T_co", covariant=True)
 
-PrimitiveType = Union[bool, float, int, str]
+PrimitiveType = bool | float | int | str
 
 NonUDFType = Literal[0]
 

duckdb/experimental/spark/conf.py

@@ -1,5 +1,3 @@
-from typing import Optional  # noqa: D100
-
 from duckdb.experimental.spark.exception import ContributionsAcceptedError
 
 
@@ -10,7 +8,7 @@ def __init__(self) -> None:  # noqa: D107
     def contains(self, key: str) -> bool:  # noqa: D102
         raise ContributionsAcceptedError
 
-    def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:  # noqa: D102
+    def get(self, key: str, defaultValue: str | None = None) -> str | None:  # noqa: D102
         raise ContributionsAcceptedError
 
     def getAll(self) -> list[tuple[str, str]]:  # noqa: D102
@@ -26,7 +24,7 @@ def setAppName(self, value: str) -> "SparkConf":  # noqa: D102
         raise ContributionsAcceptedError
 
     def setExecutorEnv(  # noqa: D102
-        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[list[tuple[str, str]]] = None
+        self, key: str | None = None, value: str | None = None, pairs: list[tuple[str, str]] | None = None
     ) -> "SparkConf":
         raise ContributionsAcceptedError
 

duckdb/experimental/spark/context.py

@@ -1,5 +1,3 @@
-from typing import Optional  # noqa: D100
-
 import duckdb
 from duckdb import DuckDBPyConnection
 from duckdb.experimental.spark.conf import SparkConf
@@ -20,7 +18,7 @@ def stop(self) -> None:  # noqa: D102
         self._connection.close()
 
     @classmethod
-    def getOrCreate(cls, conf: Optional[SparkConf] = None) -> "SparkContext":  # noqa: D102
+    def getOrCreate(cls, conf: SparkConf | None = None) -> "SparkContext":  # noqa: D102
         raise ContributionsAcceptedError
 
     @classmethod
@@ -93,13 +91,13 @@ def dump_profiles(self, path: str) -> None:  # noqa: D102
     # def emptyRDD(self) -> duckdb.experimental.spark.rdd.RDD[typing.Any]:
     # 	pass
 
-    def getCheckpointDir(self) -> Optional[str]:  # noqa: D102
+    def getCheckpointDir(self) -> str | None:  # noqa: D102
         raise ContributionsAcceptedError
 
     def getConf(self) -> SparkConf:  # noqa: D102
         raise ContributionsAcceptedError
 
-    def getLocalProperty(self, key: str) -> Optional[str]:  # noqa: D102
+    def getLocalProperty(self, key: str) -> str | None:  # noqa: D102
         raise ContributionsAcceptedError
 
     # def hadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str,

duckdb/experimental/spark/errors/error_classes.py

@@ -1,4 +1,4 @@
-# ruff: noqa: D100, E501
+# ruff: noqa: E501
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.

duckdb/experimental/spark/errors/exceptions/base.py

@@ -1,4 +1,4 @@
-from typing import Optional, cast  # noqa: D100
+from typing import cast
 
 from ..utils import ErrorClassesReader
 
@@ -8,11 +8,11 @@ class PySparkException(Exception):
 
     def __init__(  # noqa: D107
         self,
-        message: Optional[str] = None,
+        message: str | None = None,
         # The error class, decides the message format, must be one of the valid options listed in 'error_classes.py'
-        error_class: Optional[str] = None,
+        error_class: str | None = None,
         # The dictionary listing the arguments specified in the message (or the error_class)
-        message_parameters: Optional[dict[str, str]] = None,
+        message_parameters: dict[str, str] | None = None,
     ) -> None:
         # `message` vs `error_class` & `message_parameters` are mutually exclusive.
         assert (message is not None and (error_class is None and message_parameters is None)) or (
@@ -31,7 +31,7 @@ def __init__(  # noqa: D107
         self.error_class = error_class
         self.message_parameters = message_parameters
 
-    def getErrorClass(self) -> Optional[str]:
+    def getErrorClass(self) -> str | None:
         """Returns an error class as a string.
 
         .. versionadded:: 3.4.0
@@ -43,7 +43,7 @@ def getErrorClass(self) -> Optional[str]:
         """
         return self.error_class
 
-    def getMessageParameters(self) -> Optional[dict[str, str]]:
+    def getMessageParameters(self) -> dict[str, str] | None:
         """Returns a message parameters as a dictionary.
 
         .. versionadded:: 3.4.0

duckdb/experimental/spark/errors/utils.py

@@ -1,4 +1,4 @@
-#  # noqa: D100
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.

duckdb/experimental/spark/exception.py

@@ -1,14 +1,10 @@
-# ruff: noqa: D100
-from typing import Optional
-
-
 class ContributionsAcceptedError(NotImplementedError):
     """This method is not planned to be implemented, if you would like to implement this method
     or show your interest in this method to other members of the community,
     feel free to open up a PR or a Discussion over on https://github.com/duckdb/duckdb.
     """  # noqa: D205
 
-    def __init__(self, message: Optional[str] = None) -> None:  # noqa: D107
+    def __init__(self, message: str | None = None) -> None:  # noqa: D107
         doc = self.__class__.__doc__
         if message:
             doc = message + "\n" + doc

duckdb/experimental/spark/sql/_typing.py

@@ -16,18 +16,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections.abc import Callable
 from typing import (
     Any,
-    Callable,
-    Optional,
     TypeVar,
-    Union,
 )
 
 try:
     from typing import Literal, Protocol
 except ImportError:
-    from typing_extensions import Literal, Protocol
+    from typing import Literal
+
+    from typing_extensions import Protocol
 
 import datetime
 import decimal
@@ -36,14 +36,14 @@
 from . import types
 from .column import Column
 
-ColumnOrName = Union[Column, str]
+ColumnOrName = Column | str
 ColumnOrName_ = TypeVar("ColumnOrName_", bound=ColumnOrName)
 DecimalLiteral = decimal.Decimal
-DateTimeLiteral = Union[datetime.datetime, datetime.date]
+DateTimeLiteral = datetime.datetime | datetime.date
 LiteralType = PrimitiveType
-AtomicDataTypeOrString = Union[types.AtomicType, str]
-DataTypeOrString = Union[types.DataType, str]
-OptionalPrimitiveType = Optional[PrimitiveType]
+AtomicDataTypeOrString = types.AtomicType | str
+DataTypeOrString = types.DataType | str
+OptionalPrimitiveType = PrimitiveType | None
 
 AtomicValue = TypeVar(
     "AtomicValue",

duckdb/experimental/spark/sql/catalog.py

@@ -1,25 +1,25 @@
-from typing import NamedTuple, Optional, Union  # noqa: D100
+from typing import NamedTuple
 
 from .session import SparkSession
 
 
 class Database(NamedTuple):  # noqa: D101
     name: str
-    description: Optional[str]
+    description: str | None
     locationUri: str
 
 
 class Table(NamedTuple):  # noqa: D101
     name: str
-    database: Optional[str]
-    description: Optional[str]
+    database: str | None
+    description: str | None
     tableType: str
     isTemporary: bool
 
 
 class Column(NamedTuple):  # noqa: D101
     name: str
-    description: Optional[str]
+    description: str | None
     dataType: str
     nullable: bool
     isPartition: bool
@@ -28,7 +28,7 @@ class Column(NamedTuple):  # noqa: D101
 
 class Function(NamedTuple):  # noqa: D101
     name: str
-    description: Optional[str]
+    description: str | None
     className: str
     isTemporary: bool
 
@@ -55,21 +55,21 @@ def transform_to_table(x: list[str]) -> Table:
         tables = [transform_to_table(x) for x in res]
         return tables
 
-    def listColumns(self, tableName: str, dbName: Optional[str] = None) -> list[Column]:  # noqa: D102
+    def listColumns(self, tableName: str, dbName: str | None = None) -> list[Column]:  # noqa: D102
         query = f"""
 			select column_name, data_type, is_nullable from duckdb_columns() where table_name = '{tableName}'
 		"""
         if dbName:
             query += f" and database_name = '{dbName}'"
         res = self._session.conn.sql(query).fetchall()
 
-        def transform_to_column(x: list[Union[str, bool]]) -> Column:
+        def transform_to_column(x: list[str | bool]) -> Column:
             return Column(name=x[0], description=None, dataType=x[1], nullable=x[2], isPartition=False, isBucket=False)
 
         columns = [transform_to_column(x) for x in res]
         return columns
 
-    def listFunctions(self, dbName: Optional[str] = None) -> list[Function]:  # noqa: D102
+    def listFunctions(self, dbName: str | None = None) -> list[Function]:  # noqa: D102
         raise NotImplementedError
 
     def setCurrentDatabase(self, dbName: str) -> None:  # noqa: D102