fix: add compat with pandas 3

.github/workflows/ci.yml

@@ -60,8 +60,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv tool install --with='click!=8.3.0' hatch
+          echo "::group::Install hatch"
+          uv tool install hatch
+          echo "::endgroup::"
+          echo "::group::Create environment"
           hatch -v env create ${{ matrix.env.name }}
+          echo "::endgroup::"
+          hatch run ${{ matrix.env.name }}:session-info scanpy anndata
 
       - name: Run tests
         if: matrix.env.test-type == null

benchmarks/asv.conf.json

@@ -71,19 +71,7 @@
     // followed by the pip installed packages).
     //
     "matrix": {
-        "numpy": [""],
-        "scipy": [""],
-        "h5py": [""],
-        "natsort": [""],
-        "pandas": [""],
-        "memory_profiler": [""],
-        "zarr": [""],
-        "pytest": [""],
-        "pip+igraph": [""], // https://github.com/airspeed-velocity/asv/issues/1554
-        // "psutil": [""]
         "pooch": [""],
-        "scikit-image": [""], // https://github.com/conda-forge/scikit-misc-feedstock/pull/29
-        // "scikit-misc": [""],
     },
 
     // Combinations of libraries/python versions can be excluded/included

docs/release-notes/3929.fix.md

@@ -0,0 +1 @@
+Fix compatibility with pandas 3.0 {smaller}`P Angerer`

hatch.toml

@@ -35,6 +35,7 @@ overrides.matrix.deps.python = [
 ]
 overrides.matrix.deps.extra-dependencies = [
     { if = [ "pre" ], value = "anndata @ git+https://github.com/scverse/anndata.git" },
+    { if = [ "pre" ], value = "pandas>=3rc0" },
 ]
 overrides.matrix.deps.dependency-groups = [
     { if = [ "stable", "pre", "low-vers" ], value = "test" },

pyproject.toml

@@ -54,7 +54,7 @@ dependencies = [
     "numpy>=2",
     "fast-array-utils[accel,sparse]>=1.2.1",
     "matplotlib>=3.9",
-    "pandas >=2.2.2, <3.0.0rc0",
+    "pandas >=2.2.2",
     "scipy>=1.13",
     "seaborn>=0.13.2",
     "h5py>=3.11",

src/scanpy/_utils/__init__.py

@@ -29,6 +29,7 @@
 
 import h5py
 import numpy as np
+import pandas as pd
 from anndata._core.sparse_dataset import BaseCompressedSparseDataset
 from packaging.version import Version
 
@@ -44,6 +45,7 @@
     from anndata import AnnData
     from igraph import Graph
     from numpy.typing import ArrayLike, NDArray
+    from pandas._typing import Dtype as PdDtype
 
     from .._compat import CSRBase
     from ..neighbors import NeighborsParams, RPForestDict
@@ -79,6 +81,7 @@
     "sanitize_anndata",
     "select_groups",
     "update_params",
+    "with_cat_dtype",
 ]
 
 
@@ -494,6 +497,23 @@ def moving_average(a: np.ndarray, n: int):
     return ret[n - 1 :] / n
 
 
+@singledispatch
+def with_cat_dtype[X: pd.Series | pd.CategoricalIndex | pd.Categorical](
+    x: X, dtype: PdDtype
+) -> X:
+    raise NotImplementedError
+
+
+@with_cat_dtype.register(pd.Series)
+def _(x: pd.Series, dtype: PdDtype) -> pd.Series:
+    return x.cat.set_categories(x.cat.categories.astype(dtype))
+
+
+@with_cat_dtype.register(pd.Categorical | pd.CategoricalIndex)
+def _[X: pd.Categorical | pd.CategoricalIndex](x: X, dtype: PdDtype) -> X:
+    return x.set_categories(x.categories.astype(dtype))
+
+
 # --------------------------------------------------------------------------------
 # Deal with tool parameters
 # --------------------------------------------------------------------------------

src/scanpy/external/exporting.py

@@ -219,7 +219,7 @@ def spring_project(  # noqa: PLR0912, PLR0915
     np.save(subplot_dir / "cell_filter.npy", np.arange(x.shape[0]))
 
     # Write 2-D coordinates, after adjusting to roughly match SPRING's default d3js force layout parameters
-    coords = coords - coords.min(0)[None, :]
+    coords = coords - coords.min(axis=0)[None, :]
     coords = (
         coords * (np.array([1000, 1000]) / coords.ptp(0))[None, :]
         + np.array([200, -200])[None, :]
@@ -342,8 +342,8 @@ def _get_color_stats_genes(color_stats, x, gene_list):
     means, variances = mean_var(x, axis=0, correction=1)
     stdevs = np.zeros(variances.shape, dtype=float)
     stdevs[variances > 0] = np.sqrt(variances[variances > 0])
-    mins = x.min(0).todense().A1
-    maxes = x.max(0).todense().A1
+    mins = x.min(axis=0).todense().A1
+    maxes = x.max(axis=0).todense().A1
 
     pctl = 99.6
     pctl_n = (100 - pctl) / 100.0 * x.shape[0]

src/scanpy/get/get.py

@@ -259,8 +259,8 @@ def obs_df(
     >>> plotdf = sc.get.obs_df(
     ...     pbmc, keys=["CD8B", "n_genes"], obsm_keys=[("X_umap", 0), ("X_umap", 1)]
     ... )
-    >>> plotdf.columns
-    Index(['CD8B', 'n_genes', 'X_umap-0', 'X_umap-1'], dtype='object')
+    >>> plotdf.columns.astype("string")
+    Index(['CD8B', 'n_genes', 'X_umap-0', 'X_umap-1'], dtype='string')
     >>> plotdf.plot.scatter("X_umap-0", "X_umap-1", c="CD8B")  # doctest: +SKIP
     <Axes: xlabel='X_umap-0', ylabel='X_umap-1'>
 

src/scanpy/plotting/_anndata.py

@@ -749,7 +749,7 @@ def violin(  # noqa: PLR0912, PLR0913, PLR0915
     layer: str | None = None,
     density_norm: DensityNorm = "width",
     order: Sequence[str] | None = None,
-    multi_panel: bool | None = None,
+    multi_panel: bool = False,
     xlabel: str = "",
     ylabel: str | Sequence[str] | None = None,
     rotation: float | None = None,
@@ -1202,11 +1202,11 @@ def heatmap(  # noqa: PLR0912, PLR0913, PLR0915
     ).issubset(categories)
 
     if standard_scale == "obs":
-        obs_tidy = obs_tidy.sub(obs_tidy.min(1), axis=0)
-        obs_tidy = obs_tidy.div(obs_tidy.max(1), axis=0).fillna(0)
+        obs_tidy = obs_tidy.sub(obs_tidy.min(axis=1), axis=0)
+        obs_tidy = obs_tidy.div(obs_tidy.max(axis=1), axis=0).fillna(0)
     elif standard_scale == "var":
-        obs_tidy -= obs_tidy.min(0)
-        obs_tidy = (obs_tidy / obs_tidy.max(0)).fillna(0)
+        obs_tidy -= obs_tidy.min(axis=0)
+        obs_tidy = (obs_tidy / obs_tidy.max(axis=0)).fillna(0)
     elif standard_scale is None:
         pass
     else:

src/scanpy/plotting/_dotplot.py

@@ -213,11 +213,13 @@ def __init__(  # noqa: PLR0913
                 dot_color_df = self.obs_tidy.groupby(level=0, observed=True).mean()
 
             if standard_scale == "group":
-                dot_color_df = dot_color_df.sub(dot_color_df.min(1), axis=0)
-                dot_color_df = dot_color_df.div(dot_color_df.max(1), axis=0).fillna(0)
+                dot_color_df = dot_color_df.sub(dot_color_df.min(axis=1), axis=0)
+                dot_color_df = dot_color_df.div(
+                    dot_color_df.max(axis=1), axis=0
+                ).fillna(0)
             elif standard_scale == "var":
-                dot_color_df -= dot_color_df.min(0)
-                dot_color_df = (dot_color_df / dot_color_df.max(0)).fillna(0)
+                dot_color_df -= dot_color_df.min(axis=0)
+                dot_color_df = (dot_color_df / dot_color_df.max(axis=0)).fillna(0)
             elif standard_scale is None:
                 pass
             else:
@@ -696,10 +698,10 @@ def _dotplot(  # noqa: PLR0912, PLR0913, PLR0915
                 group_axis = 1
         if standard_scale is not None:
             dot_color = dot_color.sub(
-                dot_color.min((group_axis + 1) % 2), axis=group_axis
+                dot_color.min(axis=1 - group_axis), axis=group_axis
             )
             dot_color = dot_color.div(
-                dot_color.max((group_axis + 1) % 2), axis=group_axis
+                dot_color.max(axis=1 - group_axis), axis=group_axis
             ).fillna(0)
         # make scatter plot in which
         # x = var_names

src/scanpy/plotting/_matrixplot.py

@@ -180,11 +180,11 @@ def __init__(  # noqa: PLR0913
             )
 
             if standard_scale == "group":
-                values_df = values_df.sub(values_df.min(1), axis=0)
-                values_df = values_df.div(values_df.max(1), axis=0).fillna(0)
+                values_df = values_df.sub(values_df.min(axis=1), axis=0)
+                values_df = values_df.div(values_df.max(axis=1), axis=0).fillna(0)
             elif standard_scale == "var":
-                values_df -= values_df.min(0)
-                values_df = (values_df / values_df.max(0)).fillna(0)
+                values_df -= values_df.min(axis=0)
+                values_df = (values_df / values_df.max(axis=0)).fillna(0)
             elif standard_scale is None:
                 pass
             else:

src/scanpy/plotting/_scrublet.py

@@ -78,7 +78,7 @@ def scrublet_score_distribution(
 
     if "batched_by" in adata.uns["scrublet"]:
         batched_by = adata.uns["scrublet"]["batched_by"]
-        batches = adata.obs[batched_by].astype("category", copy=False)
+        batches = adata.obs[batched_by].astype("category")
         n_batches = len(batches.cat.categories)
         figsize = (figsize[0], figsize[1] * n_batches)
     else:

src/scanpy/plotting/_stacked_violin.py

@@ -229,11 +229,13 @@ def __init__(  # noqa: PLR0913
             msg = "`standard_scale='obs'` is deprecated, use `standard_scale='group'` instead"
             warn(msg, FutureWarning)
         if standard_scale == "group":
-            self.obs_tidy = self.obs_tidy.sub(self.obs_tidy.min(1), axis=0)
-            self.obs_tidy = self.obs_tidy.div(self.obs_tidy.max(1), axis=0).fillna(0)
+            self.obs_tidy = self.obs_tidy.sub(self.obs_tidy.min(axis=1), axis=0)
+            self.obs_tidy = self.obs_tidy.div(self.obs_tidy.max(axis=1), axis=0).fillna(
+                0
+            )
         elif standard_scale == "var":
-            self.obs_tidy -= self.obs_tidy.min(0)
-            self.obs_tidy = (self.obs_tidy / self.obs_tidy.max(0)).fillna(0)
+            self.obs_tidy -= self.obs_tidy.min(axis=0)
+            self.obs_tidy = (self.obs_tidy / self.obs_tidy.max(axis=0)).fillna(0)
         elif standard_scale is None:
             pass
         else:
@@ -556,7 +558,7 @@ def _make_rows_of_violinplots(
                 x=x,
                 y="values",
                 data=_df,
-                orient="vertical",
+                orient="v",
                 ax=row_ax,
                 # use a single `color`` if row_colors[idx] is defined
                 # else use the palette

src/scanpy/plotting/_tools/__init__.py

@@ -16,7 +16,7 @@
 from ... import logging as logg
 from ..._compat import old_positionals
 from ..._settings import settings
-from ..._utils import _doc_params, _empty, sanitize_anndata
+from ..._utils import _doc_params, _empty, sanitize_anndata, with_cat_dtype
 from ...get import rank_genes_groups_df
 from .._anndata import ranking
 from .._docs import (
@@ -1295,12 +1295,13 @@ def rank_genes_groups_violin(  # noqa: PLR0913
             _gene_names = _gene_names.tolist()
         df = obs_df(adata, _gene_names, use_raw=use_raw, gene_symbols=gene_symbols)
         new_gene_names = df.columns
-        df["hue"] = adata.obs[groups_key].astype(str).values
+        df["hue"] = adata.obs[groups_key].astype(str).array
         if reference == "rest":
             df.loc[df["hue"] != group_name, "hue"] = "rest"
         else:
             df.loc[~df["hue"].isin([group_name, reference]), "hue"] = np.nan
-        df["hue"] = df["hue"].astype("category")
+        # Convert categories to object because of https://github.com/mwaskom/seaborn/issues/3893
+        df["hue"] = with_cat_dtype(df["hue"].astype("category"), object)
         df_tidy = pd.melt(df, id_vars="hue", value_vars=new_gene_names)
         x = "variable"
         y = "value"
@@ -1316,7 +1317,7 @@ def rank_genes_groups_violin(  # noqa: PLR0913
             hue="hue",
             split=split,
             density_norm=density_norm,
-            orient="vertical",
+            orient="v",
             ax=ax,
         )
         if strip:

src/scanpy/preprocessing/_highly_variable_genes.py

@@ -814,9 +814,7 @@ def highly_variable_genes(  # noqa: PLR0913
     adata.var["highly_variable"] = df["highly_variable"]
     adata.var["means"] = df["means"]
     adata.var["dispersions"] = df["dispersions"]
-    adata.var["dispersions_norm"] = df["dispersions_norm"].astype(
-        np.float32, copy=False
-    )
+    adata.var["dispersions_norm"] = df["dispersions_norm"].astype(np.float32)
 
     if batch_key is not None:
         adata.var["highly_variable_nbatches"] = df["highly_variable_nbatches"]

tests/conftest.py

@@ -2,6 +2,7 @@
 
 import shutil
 import sys
+from contextlib import ExitStack
 from pathlib import Path
 from textwrap import dedent
 from typing import TYPE_CHECKING, TypedDict, cast
@@ -148,3 +149,9 @@ def plt():
     from matplotlib import pyplot as plt
 
     return plt
+
+
+@pytest.fixture
+def exit_stack() -> Generator[ExitStack]:
+    with ExitStack() as stack:
+        yield stack

tests/external/test_hashsolo.py

@@ -39,6 +39,7 @@ def test_cell_demultiplexing():
     expected = pd.array(doublets + classes + negatives, dtype="string")
     classification = test_data.obs["Classification"].array.astype("string")
     # This is a bit flaky, so allow some mismatches:
-    if (expected != classification).sum() > 3:
+    # (Series() because of https://github.com/pandas-dev/pandas/issues/63458)
+    if pd.Series(expected != classification).sum() > 3:
         # Compare lists for better error message
         assert classification.tolist() == expected.tolist()