Skip to content

base

Factory for making (regularized) covariance estimators.

A Gaussian copula estimates one covariance per group of cells, these may have fewer cells than genes if appropriately regularized. Different covariance estimators can be substituted as estimator classes, leading to different copula types .

Estimation streams. We make a single pass through the loader, using :meth:CovarianceEstimator.update on each (batch, group) combination. This is the basic recipe.

estimator = prototype.clone()      # one per group
estimator.start(n_features)
estimator.update(z)                # once per batch
estimator.finalize()               # sets covariance_
estimator.release()                # drop accumulators before pickling

Examples:

>>> import numpy as np
>>> from scdesigner.covariance import LedoitWolf
>>>
>>> rng = np.random.default_rng(0)
>>> z = rng.standard_normal((40, 8))
>>> estimator = LedoitWolf()
>>> estimator.start(8)
>>> for start in range(0, 40, 16):
...     estimator.update(z[start:start + 16])
>>> _ = estimator.finalize()
>>> estimator.covariance_.shape
(8, 8)
>>> 0.0 <= estimator.intensity_ <= 1.0
True

CovarianceEstimator

Bases: ABC

Streaming covariance estimator for a copula group.

Attributes:

Name Type Description
regularized bool

Class attribute. True when :attr:covariance_ is positive definite by construction; the copula then treats a failed factorization as a numerical error. An estimator that can return a singular estimate must leave this False.

n_features_ int

Number of features, set by :meth:start.

n_samples_ int

Number of observations seen so far, modified by :meth:update.

covariance_ ndarray or None

Estimated covariance of shape (n_features_, n_features_), set by :meth:finalize.

Source code in scdesigner/src/scdesigner/covariance/base.py
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
class CovarianceEstimator(ABC):
    """
    Streaming covariance estimator for a copula group.

    Attributes
    ----------
    regularized : bool
        Class attribute. ``True`` when :attr:`covariance_` is positive definite
        by construction; the copula then treats a failed factorization as a
        numerical error. An estimator that can return a singular estimate must
        leave this ``False``.
    n_features_ : int
        Number of features, set by :meth:`start`.
    n_samples_ : int
        Number of observations seen so far, modified by :meth:`update`.
    covariance_ : np.ndarray or None
        Estimated covariance of shape ``(n_features_, n_features_)``, set by
        :meth:`finalize`.
    """

    regularized = False

    # ------------------------------------------------------------------
    # construction
    # ------------------------------------------------------------------
    @classmethod
    def _param_names(cls):
        """
        Constructor parameter names, used by :meth:`get_params` and :meth:`clone`.
        """
        if cls.__init__ is object.__init__:
            return []

        parameters = inspect.signature(cls.__init__).parameters
        variadic = [
            name
            for name, p in parameters.items()
            if p.kind in (p.VAR_KEYWORD, p.VAR_POSITIONAL)
        ]
        if variadic:
            raise RuntimeError(
                f"{cls.__name__}.__init__ expects arguments {', '.join(variadic)}."
            )
        return [name for name in parameters if name != "self"]

    def get_params(self) -> Dict:
        """Return the constructor arguments of this estimator."""
        return {name: getattr(self, name) for name in self._param_names()}

    def clone(self) -> "CovarianceEstimator":
        """
        Create a new copy of the estimator with the same hyperparameters.
        """
        return type(self)(**self.get_params())

    def __repr__(self):
        arguments = ", ".join(f"{k}={v!r}" for k, v in self.get_params().items())
        return f"{type(self).__name__}({arguments})"

    # ------------------------------------------------------------------
    # Stream across samples
    # ------------------------------------------------------------------
    def start(self, n_features: int) -> None:
        """
        Prepare to stream covariance over ``n_features`` features.

        Parameters
        ----------
        n_features : int
            Number of genes the copula models for this group.
        """
        self.n_features_ = int(n_features)
        self.n_samples_ = 0
        self.covariance_ = None
        self._start(self.n_features_)

    def update(self, z: np.ndarray) -> None:
        """
        Substitute a new block of z-scores into the streaming estimate

        Parameters
        ----------
        z : np.ndarray
            Array of shape ``(n_batch, n_features_)`` holding the latent normal
            scores for the cells of this group within one batch.
        """
        self.n_samples_ += z.shape[0]
        self._update(z)

    def finalize(self) -> "CovarianceEstimator":
        """
        Compute :attr:`covariance_` from the accumulators.

        Returns
        -------
        CovarianceEstimator
            ``self``, so calls can be chained.
        """
        if self.n_samples_ > 0:
            self.covariance_ = self._finalize()
        return self

    # ------------------------------------------------------------------
    # required steps for streaming
    # ------------------------------------------------------------------
    @abstractmethod
    def _start(self, n_features: int) -> None:
        """Allocate accumulators for ``n_features`` features."""

    @abstractmethod
    def _update(self, z: np.ndarray) -> None:
        """Fold one ``(n_batch, n_features)`` block into the accumulators."""

    @abstractmethod
    def _finalize(self) -> np.ndarray:
        """Return the estimated ``(n_features, n_features)`` covariance."""

    # ------------------------------------------------------------------
    # optional helpers
    # ------------------------------------------------------------------
    @property
    def second_moment_(self) -> Optional[np.ndarray]:
        """
        Uncentered second moment ``Z.T @ Z``, or ``None`` if not retained.
        """
        return None

    def num_params(self, n_features: int) -> int:
        """
        Free parameters in the estimate, used for AIC and BIC.

        The default counts every off-diagonal entry. This is actually an upper
        bound for regularized estimators, so those should implement their own
        parameter functions.
        """
        return n_features * (n_features - 1) // 2

    def diagnostics(self) -> Dict:
        """Any extra, method-specific diagnostics"""
        return {}

    def release(self) -> None:
        """
        Drop accumulators no longer needed after :meth:`finalize`.

        The copula calls this once it has consumed :attr:`second_moment_`.  This
        allows us to free up memory for large class definitions.
        """

second_moment_ property

Uncentered second moment Z.T @ Z, or None if not retained.

clone()

Create a new copy of the estimator with the same hyperparameters.

Source code in scdesigner/src/scdesigner/covariance/base.py
92
93
94
95
96
def clone(self) -> "CovarianceEstimator":
    """
    Create a new copy of the estimator with the same hyperparameters.
    """
    return type(self)(**self.get_params())

diagnostics()

Any extra, method-specific diagnostics

Source code in scdesigner/src/scdesigner/covariance/base.py
180
181
182
def diagnostics(self) -> Dict:
    """Any extra, method-specific diagnostics"""
    return {}

finalize()

Compute :attr:covariance_ from the accumulators.

Returns:

Type Description
CovarianceEstimator

self, so calls can be chained.

Source code in scdesigner/src/scdesigner/covariance/base.py
132
133
134
135
136
137
138
139
140
141
142
143
def finalize(self) -> "CovarianceEstimator":
    """
    Compute :attr:`covariance_` from the accumulators.

    Returns
    -------
    CovarianceEstimator
        ``self``, so calls can be chained.
    """
    if self.n_samples_ > 0:
        self.covariance_ = self._finalize()
    return self

get_params()

Return the constructor arguments of this estimator.

Source code in scdesigner/src/scdesigner/covariance/base.py
88
89
90
def get_params(self) -> Dict:
    """Return the constructor arguments of this estimator."""
    return {name: getattr(self, name) for name in self._param_names()}

num_params(n_features)

Free parameters in the estimate, used for AIC and BIC.

The default counts every off-diagonal entry. This is actually an upper bound for regularized estimators, so those should implement their own parameter functions.

Source code in scdesigner/src/scdesigner/covariance/base.py
170
171
172
173
174
175
176
177
178
def num_params(self, n_features: int) -> int:
    """
    Free parameters in the estimate, used for AIC and BIC.

    The default counts every off-diagonal entry. This is actually an upper
    bound for regularized estimators, so those should implement their own
    parameter functions.
    """
    return n_features * (n_features - 1) // 2

release()

Drop accumulators no longer needed after :meth:finalize.

The copula calls this once it has consumed :attr:second_moment_. This allows us to free up memory for large class definitions.

Source code in scdesigner/src/scdesigner/covariance/base.py
184
185
186
187
188
189
190
def release(self) -> None:
    """
    Drop accumulators no longer needed after :meth:`finalize`.

    The copula calls this once it has consumed :attr:`second_moment_`.  This
    allows us to free up memory for large class definitions.
    """

start(n_features)

Prepare to stream covariance over n_features features.

Parameters:

Name Type Description Default
n_features int

Number of genes the copula models for this group.

required
Source code in scdesigner/src/scdesigner/covariance/base.py
105
106
107
108
109
110
111
112
113
114
115
116
117
def start(self, n_features: int) -> None:
    """
    Prepare to stream covariance over ``n_features`` features.

    Parameters
    ----------
    n_features : int
        Number of genes the copula models for this group.
    """
    self.n_features_ = int(n_features)
    self.n_samples_ = 0
    self.covariance_ = None
    self._start(self.n_features_)

update(z)

Substitute a new block of z-scores into the streaming estimate

Parameters:

Name Type Description Default
z ndarray

Array of shape (n_batch, n_features_) holding the latent normal scores for the cells of this group within one batch.

required
Source code in scdesigner/src/scdesigner/covariance/base.py
119
120
121
122
123
124
125
126
127
128
129
130
def update(self, z: np.ndarray) -> None:
    """
    Substitute a new block of z-scores into the streaming estimate

    Parameters
    ----------
    z : np.ndarray
        Array of shape ``(n_batch, n_features_)`` holding the latent normal
        scores for the cells of this group within one batch.
    """
    self.n_samples_ += z.shape[0]
    self._update(z)

as_covariance_estimator(covariance)

Convert a covariance string to an estimator object

Parameters:

Name Type Description Default
covariance CovarianceEstimator, str, or None

This can be an existing estimator, a registered name e.g."ledoit_wolf"`, orNone`` for the unregularized sample covariance.

required

Returns:

Type Description
CovarianceEstimator

The estimator to use as a prototype.

Raises:

Type Description
ValueError

If the name is not registered.

Examples:

>>> from scdesigner.covariance import as_covariance_estimator
>>> as_covariance_estimator("oas")
OAS()
>>> as_covariance_estimator(None)
SampleCovariance()
Source code in scdesigner/src/scdesigner/covariance/base.py
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
def as_covariance_estimator(covariance) -> CovarianceEstimator:
    """
    Convert a covariance string to an estimator object

    Parameters
    ----------
    covariance : CovarianceEstimator, str, or None
        This can be an existing estimator, a registered name
        e.g.``"ledoit_wolf"`, or ``None`` for the unregularized sample
        covariance.

    Returns
    -------
    CovarianceEstimator
        The estimator to use as a prototype.

    Raises
    ------
    ValueError
        If the name is not registered.

    Examples
    --------
    >>> from scdesigner.covariance import as_covariance_estimator
    >>> as_covariance_estimator("oas")
    OAS()
    >>> as_covariance_estimator(None)
    SampleCovariance()
    """
    if isinstance(covariance, CovarianceEstimator):
        return covariance
    if covariance is None:
        return _ESTIMATORS["sample"]()
    if isinstance(covariance, str):
        name = covariance.lower()
        if name not in _ESTIMATORS:
            raise ValueError(
                f"Unknown covariance estimator {covariance!r}. Expected one of "
                f"{sorted(_ESTIMATORS)}, a CovarianceEstimator instance, or None."
            )
        return _ESTIMATORS[name]()
    raise ValueError(
        f"Unsupported covariance value {covariance!r}. Expected one of "
        f"{sorted(_ESTIMATORS)}, a CovarianceEstimator instance, or None."
    )

register_estimator(name, estimator_class)

Make an estimator searchable by name.

This allows us to search for estimators in a YAML config file.

Parameters:

Name Type Description Default
name str

Lowercase name, as accepted by :func:as_covariance_estimator.

required
estimator_class type

A :class:CovarianceEstimator subclass constructible with no arguments.

required
Source code in scdesigner/src/scdesigner/covariance/base.py
196
197
198
199
200
201
202
203
204
205
206
207
208
209
def register_estimator(name: str, estimator_class: type) -> None:
    """
    Make an estimator searchable by name.

    This allows us to search for estimators in a YAML config file.

    Parameters
    ----------
    name : str
        Lowercase name, as accepted by :func:`as_covariance_estimator`.
    estimator_class : type
        A :class:`CovarianceEstimator` subclass constructible with no arguments.
    """
    _ESTIMATORS[name] = estimator_class