Fix PyDESeq2 layer handling in fit()

pertpy/tools/_differential_gene_expression/_pydeseq2.py

@@ -42,12 +42,18 @@ def fit(self, **kwargs) -> pd.DataFrame:
         try:
             usable_cpus = len(os.sched_getaffinity(0))  # type: ignore # os.sched_getaffinity is not available on Windows and macOS
         except AttributeError:
-            usable_cpus = os.cpu_count()
+            usable_cpus = os.cpu_count() or 1
 
         inference = DefaultInference(n_cpus=kwargs.pop("n_cpus", usable_cpus))
 
+        adata_for_dds = self.adata
+        if self.layer is not None:
+            # pydeseq2 always uses `adata.X` as the count matrix, so ensure that `X`
+            # reflects the chosen layer without mutating the user-provided `.X`.
+            adata_for_dds = AnnData(X=self.data, obs=self.adata.obs, var=self.adata.var)
+
         dds = DeseqDataSet(
-            adata=self.adata,
+            adata=adata_for_dds,
             design=self.design,  # initialize using design matrix, not formula
             refit_cooks=True,
             inference=inference,

tests/tools/_differential_gene_expression/test_pydeseq2.py

@@ -1,5 +1,6 @@
 from importlib.util import find_spec
 
+import numpy as np
 import numpy.testing as npt
 import pytest
 
@@ -77,3 +78,23 @@ def test_pydeseq2_formula(test_adata):
     res_2 = model2.test_contrasts(model2.contrast("condition", "A", "B"))
 
     npt.assert_almost_equal(res_2.log_fc.values, res_1.log_fc.values)
+
+
+def test_pydeseq2_uses_layer_counts_for_fit(test_adata):
+    """Regression test: when `layer` is provided, `.fit()` must use that layer as counts, not `.X`."""
+    from pertpy.tools._differential_gene_expression import PyDESeq2
+
+    test_adata.layers["sum"] = test_adata.X.copy()
+
+    # Make X explicitly non-integer to reproduce the failure mode if `.fit()` ignores `layer=`.
+    X_layer_before = np.array(test_adata.layers["sum"], copy=True)
+    test_adata.X = np.array(test_adata.X, dtype=float) / 2.0
+    X_before_fit = np.array(test_adata.X, copy=True)
+
+    model = PyDESeq2(adata=test_adata, layer="sum", design="~condition")
+    model.fit(n_cpus=1)
+    res_df = model.test_contrasts(model.contrast("condition", "A", "B"))
+
+    assert len(res_df) == test_adata.n_vars
+    npt.assert_array_equal(X_layer_before, test_adata.layers["sum"])
+    npt.assert_array_equal(X_before_fit, np.array(test_adata.X))