From 518f9da8bbf076ce490b3a5ddafb3a45c3f6dddb Mon Sep 17 00:00:00 2001 From: sichao Date: Thu, 16 Nov 2023 16:21:07 -0500 Subject: [PATCH 01/14] deprecate dynamo_bk and dynamo_fitting --- dynamo/tools/deprecated.py | 339 +++++++++++++++++++++++++++++++++ dynamo/tools/dynamo_bk.py | 80 -------- dynamo/tools/dynamo_fitting.py | 161 ---------------- 3 files changed, 339 insertions(+), 241 deletions(-) create mode 100644 dynamo/tools/deprecated.py delete mode 100755 dynamo/tools/dynamo_bk.py delete mode 100755 dynamo/tools/dynamo_fitting.py diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py new file mode 100644 index 000000000..f29788523 --- /dev/null +++ b/dynamo/tools/deprecated.py @@ -0,0 +1,339 @@ +import warnings +from typing import Callable, Iterable, List, Optional, Tuple, Union + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + +import functools + +import numpy as np +from scipy.integrate import odeint +from scipy.optimize import least_squares + + +def deprecated(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"{func.__name__} is deprecated and will be removed in a future release. " + f"Please update your code to use the new replacement function.", + category=DeprecationWarning, + stacklevel=2, + ) + return func(*args, **kwargs) + + return wrapper + + +# --------------------------------------------------------------------------------------------------- +# deprecated dynamo_bk.py +@deprecated +def sol_u(*args, **kwargs): + _sol_u_legacy(*args, **kwargs) + + +def _sol_u_legacy(t, u0, alpha, beta): + return u0 * np.exp(-beta * t) + alpha / beta * (1 - np.exp(-beta * t)) + + +@deprecated +def sol_s_dynamo_bk(*args, **kwargs): + _sol_s_dynamo_bk_legacy(*args, **kwargs) + + +def _sol_s_dynamo_bk_legacy(t, s0, u0, alpha, beta, gamma): + exp_gt = np.exp(-gamma * t) + return ( + s0 * exp_gt + alpha / gamma * (1 - exp_gt) + (alpha + u0 * beta) / (gamma - beta) * (exp_gt - np.exp(-beta * t)) + ) + + +@deprecated +def fit_gamma_labelling_dynamo_bk(*args, **kwargs): + _fit_gamma_labelling_dynamo_bk_legacy(*args, **kwargs) + + +def _fit_gamma_labelling_dynamo_bk_legacy(t, l, mode=None, lbound=None): + n = l.size + tau = t - np.min(t) + tm = np.mean(tau) + + # prepare y + if lbound is not None: + l[l < lbound] = lbound + y = np.log(l) + ym = np.mean(y) + + # calculate slope + var_t = np.mean(tau**2) - tm**2 + cov = np.sum(y.dot(tau)) / n - ym * tm + k = cov / var_t + + # calculate intercept + b = np.exp(ym - k * tm) if mode != "fast" else None + + return -k, b + + +@deprecated +def fit_alpha_dynamo_bk_labelling(*args, **kwargs): + _fit_alpha_labelling_dynamo_bk_legacy(*args, **kwargs) + + +def _fit_alpha_labelling_dynamo_bk_legacy(t, u, gamma, mode=None): + n = u.size + tau = t - np.min(t) + expt = np.exp(gamma * tau) + + # prepare x + x = expt - 1 + xm = np.mean(x) + + # prepare y + y = u * expt + ym = np.mean(y) + + # calculate slope + var_x = np.mean(x**2) - xm**2 + cov = np.sum(y.dot(x)) / n - ym * xm + k = cov / var_x + + # calculate intercept + b = ym - k * xm if mode != "fast" else None + + return k * gamma, b + + +@deprecated +def fit_gamma_splicing_dynamo_bk(*args, **kwargs): + _fit_gamma_splicing_dynamo_bk_legacy(*args, **kwargs) + + +def _fit_gamma_splicing_dynamo_bk_legacy(t, s, beta, u0, bounds=(0, np.inf)): + tau = t - np.min(t) + s0 = np.mean(s[tau == 0]) + g0 = beta * u0 / s0 + + f_lsq = lambda g: _sol_s_dynamo_bk_legacy(tau, u0, s0, 0, beta, g) - s + ret = least_squares(f_lsq, g0, bounds=bounds) + return ret.x, s0 + + +@deprecated +def fit_gamma_dynamo_bk(*args, **kwargs): + _fit_gamma_dynamo_bk_legacy(*args, **kwargs) + + +def _fit_gamma_dynamo_bk_legacy(u, s): + cov = u.dot(s) / len(u) - np.mean(u) * np.mean(s) + var_s = s.dot(s) / len(s) - np.mean(s) ** 2 + gamma = cov / var_s + return gamma + + +# --------------------------------------------------------------------------------------------------- +# deprecated dynamo_fitting.py +@deprecated +def sol_s_dynamo_fitting(*args, **kwargs): + _sol_s_dynamo_fitting_legacy(*args, **kwargs) + + +def _sol_s_dynamo_fitting_legacy(t, s0, u0, alpha, beta, gamma): + exp_gt = np.exp(-gamma * t) + if beta == gamma: + s = s0 * exp_gt + (beta * u0 - alpha) * t * exp_gt + alpha / gamma * (1 - exp_gt) + else: + s = ( + s0 * exp_gt + + alpha / gamma * (1 - exp_gt) + + (alpha - u0 * beta) / (gamma - beta) * (exp_gt - np.exp(-beta * t)) + ) + return s + + +@deprecated +def sol_p_dynamo_fitting(*args, **kwargs): + _sol_p_dynamo_fitting_legacy(*args, **kwargs) + + +def _sol_p_dynamo_fitting_legacy(t, p0, s0, u0, alpha, beta, gamma, eta, gamma_p): + u = _sol_u_legacy(t, u0, alpha, beta) + s = _sol_s_dynamo_fitting_legacy(t, s0, u0, alpha, beta, gamma) + exp_gt = np.exp(-gamma_p * t) + p = p0 * exp_gt + eta / (gamma_p - gamma) * ( + s - s0 * exp_gt - beta / (gamma_p - beta) * (u - u0 * exp_gt - alpha / gamma_p * (1 - exp_gt)) + ) + return p, s, u + + +@deprecated +def sol_ode_dynamo_fitting(*args, **kwargs): + _sol_ode_dynamo_fitting_legacy(*args, **kwargs) + + +def _sol_ode_dynamo_fitting_legacy(x, t, alpha, beta, gamma, eta, gamma_p): + dx = np.zeros(x.shape) + dx[0] = alpha - beta * x[0] + dx[1] = beta * x[0] - gamma * x[1] + dx[2] = eta * x[1] - gamma_p * x[2] + return dx + + +@deprecated +def sol_num_dynamo_fitting(args, kwargs): + _sol_num_dynamo_fitting_legacy(*args, **kwargs) + + +def _sol_num_dynamo_fitting_legacy(t, p0, s0, u0, alpha, beta, gamma, eta, gamma_p): + sol = odeint( + lambda x, t: _sol_ode_dynamo_fitting_legacy(x, t, alpha, beta, gamma, eta, gamma_p), + np.array([u0, s0, p0]), + t, + ) + return sol + + +@deprecated +def fit_gamma_labelling_dynamo_fitting(*args, **kwargs): + _fit_gamma_labelling_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_gamma_labelling_dynamo_fitting_legacy(t, l, mode=None, lbound=None): + t = np.array(t, dtype=float) + l = np.array(l, dtype=float) + if l.ndim == 1: + # l is a vector + n_rep = 1 + else: + n_rep = l.shape[0] + t = np.tile(t, n_rep) + l = l.flatten() + + # remove low counts based on lbound + if lbound is not None: + t[l < lbound] = np.nan + l[l < lbound] = np.nan + + n = np.sum(~np.isnan(t)) + tau = t - np.nanmin(t) + tm = np.nanmean(tau) + + # prepare y + y = np.log(l) + ym = np.nanmean(y) + + # calculate slope + var_t = np.nanmean(tau**2) - tm**2 + cov = np.nansum(y * tau) / n - ym * tm + k = cov / var_t + + # calculate intercept + b = np.exp(ym - k * tm) if mode != "fast" else None + + gamma = -k + u0 = b + + return gamma, u0 + + +@deprecated +def fit_beta_lsq_dynamo_fitting(*args, **kwargs): + _fit_beta_lsq_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_beta_lsq_dynamo_fitting_legacy(t, l, bounds=(0, np.inf), fix_l0=False, beta_0=None): + tau = t - np.min(t) + l0 = np.mean(l[:, tau == 0]) + if beta_0 is None: + beta_0 = 1 + + if fix_l0: + f_lsq = lambda b: (_sol_u_legacy(tau, l0, 0, b) - l).flatten() + ret = least_squares(f_lsq, beta_0, bounds=bounds) + beta = ret.x + else: + f_lsq = lambda p: (_sol_u_legacy(tau, p[1], 0, p[0]) - l).flatten() + ret = least_squares(f_lsq, np.array([beta_0, l0]), bounds=bounds) + beta = ret.x[0] + l0 = ret.x[1] + return beta, l0 + + +@deprecated +def fit_alpha_labelling_dynamo_fitting(*args, **kwargs): + _fit_alpha_labelling_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_alpha_labelling_dynamo_fitting_legacy(t, u, gamma, mode=None): + n = u.size + tau = t - np.min(t) + expt = np.exp(gamma * tau) + + # prepare x + x = expt - 1 + xm = np.mean(x) + + # prepare y + y = u * expt + ym = np.mean(y) + + # calculate slope + var_x = np.mean(x**2) - xm**2 + cov = np.sum(y.dot(x)) / n - ym * xm + k = cov / var_x + + # calculate intercept + b = ym - k * xm if mode != "fast" else None + + return k * gamma, b + + +@deprecated +def fit_alpha_synthesis_dynamo_fitting(*args, **kwargs): + _fit_alpha_synthesis_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_alpha_synthesis_dynamo_fitting_legacy(t, u, beta, mode=None): + tau = t - np.min(t) + expt = np.exp(-beta * tau) + + # prepare x + x = 1 - expt + + return beta * np.mean(u) / np.mean(x) + + +@deprecated +def fit_gamma_splicing_dynamo_fitting(*args, **kwargs): + _fit_gamma_splicing_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_gamma_splicing_dynamo_fitting_legacy(t, s, beta, u0, bounds=(0, np.inf), fix_s0=False): + tau = t - np.min(t) + s0 = np.mean(s[:, tau == 0]) + g0 = beta * u0 / s0 + + if fix_s0: + f_lsq = lambda g: (_sol_s_dynamo_fitting_legacy(tau, s0, u0, 0, beta, g) - s).flatten() + ret = least_squares(f_lsq, g0, bounds=bounds) + gamma = ret.x + else: + f_lsq = lambda p: (_sol_s_dynamo_fitting_legacy(tau, p[1], u0, 0, beta, p[0]) - s).flatten() + ret = least_squares(f_lsq, np.array([g0, s0]), bounds=bounds) + gamma = ret.x[0] + s0 = ret.x[1] + return gamma, s0 + + +@deprecated +def fit_gamma_dynamo_fitting(*args, **kwargs): + _fit_gamma_dynamo_fitting_legacy(*args, **kwargs) + + +def _fit_gamma_dynamo_fitting_legacy(u, s): + cov = u.dot(s) / len(u) - np.mean(u) * np.mean(s) + var_s = s.dot(s) / len(s) - np.mean(s) ** 2 + gamma = cov / var_s + return gamma diff --git a/dynamo/tools/dynamo_bk.py b/dynamo/tools/dynamo_bk.py deleted file mode 100755 index 94e8c8934..000000000 --- a/dynamo/tools/dynamo_bk.py +++ /dev/null @@ -1,80 +0,0 @@ -import numpy as np -from scipy.optimize import least_squares -from scipy.stats import norm as normal -from sklearn.decomposition import TruncatedSVD -from sklearn.manifold import TSNE -from sklearn.neighbors import NearestNeighbors - - -def sol_u(t, u0, alpha, beta): - return u0 * np.exp(-beta * t) + alpha / beta * (1 - np.exp(-beta * t)) - - -def sol_s(t, s0, u0, alpha, beta, gamma): - exp_gt = np.exp(-gamma * t) - return ( - s0 * exp_gt + alpha / gamma * (1 - exp_gt) + (alpha + u0 * beta) / (gamma - beta) * (exp_gt - np.exp(-beta * t)) - ) - - -def fit_gamma_labelling(t, l, mode=None, lbound=None): - n = l.size - tau = t - np.min(t) - tm = np.mean(tau) - - # prepare y - if lbound is not None: - l[l < lbound] = lbound - y = np.log(l) - ym = np.mean(y) - - # calculate slope - var_t = np.mean(tau**2) - tm**2 - cov = np.sum(y.dot(tau)) / n - ym * tm - k = cov / var_t - - # calculate intercept - b = np.exp(ym - k * tm) if mode != "fast" else None - - return -k, b - - -def fit_alpha_labelling(t, u, gamma, mode=None): - n = u.size - tau = t - np.min(t) - expt = np.exp(gamma * tau) - - # prepare x - x = expt - 1 - xm = np.mean(x) - - # prepare y - y = u * expt - ym = np.mean(y) - - # calculate slope - var_x = np.mean(x**2) - xm**2 - cov = np.sum(y.dot(x)) / n - ym * xm - k = cov / var_x - - # calculate intercept - b = ym - k * xm if mode != "fast" else None - - return k * gamma, b - - -def fit_gamma_splicing(t, s, beta, u0, bounds=(0, np.inf)): - tau = t - np.min(t) - s0 = np.mean(s[tau == 0]) - g0 = beta * u0 / s0 - - f_lsq = lambda g: sol_s(tau, u0, s0, 0, beta, g) - s - ret = least_squares(f_lsq, g0, bounds=bounds) - return ret.x, s0 - - -def fit_gamma(u, s): - cov = u.dot(s) / len(u) - np.mean(u) * np.mean(s) - var_s = s.dot(s) / len(s) - np.mean(s) ** 2 - gamma = cov / var_s - return gamma diff --git a/dynamo/tools/dynamo_fitting.py b/dynamo/tools/dynamo_fitting.py deleted file mode 100755 index c95ec43ca..000000000 --- a/dynamo/tools/dynamo_fitting.py +++ /dev/null @@ -1,161 +0,0 @@ -import numpy as np -from scipy.integrate import odeint -from scipy.optimize import least_squares - - -def sol_u(t, u0, alpha, beta): - return u0 * np.exp(-beta * t) + alpha / beta * (1 - np.exp(-beta * t)) - - -def sol_s(t, s0, u0, alpha, beta, gamma): - exp_gt = np.exp(-gamma * t) - if beta == gamma: - s = s0 * exp_gt + (beta * u0 - alpha) * t * exp_gt + alpha / gamma * (1 - exp_gt) - else: - s = ( - s0 * exp_gt - + alpha / gamma * (1 - exp_gt) - + (alpha - u0 * beta) / (gamma - beta) * (exp_gt - np.exp(-beta * t)) - ) - return s - - -def sol_p(t, p0, s0, u0, alpha, beta, gamma, eta, gamma_p): - u = sol_u(t, u0, alpha, beta) - s = sol_s(t, s0, u0, alpha, beta, gamma) - exp_gt = np.exp(-gamma_p * t) - p = p0 * exp_gt + eta / (gamma_p - gamma) * ( - s - s0 * exp_gt - beta / (gamma_p - beta) * (u - u0 * exp_gt - alpha / gamma_p * (1 - exp_gt)) - ) - return p, s, u - - -def sol_ode(x, t, alpha, beta, gamma, eta, gamma_p): - dx = np.zeros(x.shape) - dx[0] = alpha - beta * x[0] - dx[1] = beta * x[0] - gamma * x[1] - dx[2] = eta * x[1] - gamma_p * x[2] - return dx - - -def sol_num(t, p0, s0, u0, alpha, beta, gamma, eta, gamma_p): - sol = odeint( - lambda x, t: sol_ode(x, t, alpha, beta, gamma, eta, gamma_p), - np.array([u0, s0, p0]), - t, - ) - return sol - - -def fit_gamma_labelling(t, l, mode=None, lbound=None): - t = np.array(t, dtype=float) - l = np.array(l, dtype=float) - if l.ndim == 1: - # l is a vector - n_rep = 1 - else: - n_rep = l.shape[0] - t = np.tile(t, n_rep) - l = l.flatten() - - # remove low counts based on lbound - if lbound is not None: - t[l < lbound] = np.nan - l[l < lbound] = np.nan - - n = np.sum(~np.isnan(t)) - tau = t - np.nanmin(t) - tm = np.nanmean(tau) - - # prepare y - y = np.log(l) - ym = np.nanmean(y) - - # calculate slope - var_t = np.nanmean(tau**2) - tm**2 - cov = np.nansum(y * tau) / n - ym * tm - k = cov / var_t - - # calculate intercept - b = np.exp(ym - k * tm) if mode != "fast" else None - - gamma = -k - u0 = b - - return gamma, u0 - - -def fit_beta_lsq(t, l, bounds=(0, np.inf), fix_l0=False, beta_0=None): - tau = t - np.min(t) - l0 = np.mean(l[:, tau == 0]) - if beta_0 is None: - beta_0 = 1 - - if fix_l0: - f_lsq = lambda b: (sol_u(tau, l0, 0, b) - l).flatten() - ret = least_squares(f_lsq, beta_0, bounds=bounds) - beta = ret.x - else: - f_lsq = lambda p: (sol_u(tau, p[1], 0, p[0]) - l).flatten() - ret = least_squares(f_lsq, np.array([beta_0, l0]), bounds=bounds) - beta = ret.x[0] - l0 = ret.x[1] - return beta, l0 - - -def fit_alpha_labelling(t, u, gamma, mode=None): - n = u.size - tau = t - np.min(t) - expt = np.exp(gamma * tau) - - # prepare x - x = expt - 1 - xm = np.mean(x) - - # prepare y - y = u * expt - ym = np.mean(y) - - # calculate slope - var_x = np.mean(x**2) - xm**2 - cov = np.sum(y.dot(x)) / n - ym * xm - k = cov / var_x - - # calculate intercept - b = ym - k * xm if mode != "fast" else None - - return k * gamma, b - - -def fit_alpha_synthesis(t, u, beta, mode=None): - tau = t - np.min(t) - expt = np.exp(-beta * tau) - - # prepare x - x = 1 - expt - - return beta * np.mean(u) / np.mean(x) - - -def fit_gamma_splicing(t, s, beta, u0, bounds=(0, np.inf), fix_s0=False): - tau = t - np.min(t) - s0 = np.mean(s[:, tau == 0]) - g0 = beta * u0 / s0 - - if fix_s0: - f_lsq = lambda g: (sol_s(tau, s0, u0, 0, beta, g) - s).flatten() - ret = least_squares(f_lsq, g0, bounds=bounds) - gamma = ret.x - else: - f_lsq = lambda p: (sol_s(tau, p[1], u0, 0, beta, p[0]) - s).flatten() - ret = least_squares(f_lsq, np.array([g0, s0]), bounds=bounds) - gamma = ret.x[0] - s0 = ret.x[1] - return gamma, s0 - - -def fit_gamma(u, s): - cov = u.dot(s) / len(u) - np.mean(u) * np.mean(s) - var_s = s.dot(s) / len(s) - np.mean(s) ** 2 - gamma = cov / var_s - return gamma From a55ef45d8639e538dbcde15ff288da1e74a0fff5 Mon Sep 17 00:00:00 2001 From: sichao Date: Thu, 16 Nov 2023 17:04:19 -0500 Subject: [PATCH 02/14] deprecate _dynamics_deprecated utils_moments_deprecated --- dynamo/tools/_dynamics_deprecated.py | 338 -------- dynamo/tools/deprecated.py | 951 ++++++++++++++++++++++- dynamo/tools/utils_moments_deprecated.py | 618 --------------- 3 files changed, 950 insertions(+), 957 deletions(-) delete mode 100755 dynamo/tools/_dynamics_deprecated.py delete mode 100755 dynamo/tools/utils_moments_deprecated.py diff --git a/dynamo/tools/_dynamics_deprecated.py b/dynamo/tools/_dynamics_deprecated.py deleted file mode 100755 index 4da44cf85..000000000 --- a/dynamo/tools/_dynamics_deprecated.py +++ /dev/null @@ -1,338 +0,0 @@ -import warnings - -import numpy as np - -from ..dynamo_logger import main_info, main_warning -from .moments import moment_model -from .utils import ( - get_data_for_kin_params_estimation, - get_mapper, - get_U_S_for_velocity_estimation, - get_valid_bools, - set_param_kinetic, - set_param_ss, - set_velocity, -) -from .utils_moments import moments -from .velocity import ss_estimation, velocity - - -# incorporate the model selection code soon -def _dynamics( - adata, - tkey=None, - filter_gene_mode="final", - mode="moment", - use_smoothed=True, - group=None, - protein_names=None, - experiment_type=None, - assumption_mRNA=None, - assumption_protein="ss", - NTR_vel=True, - concat_data=False, - log_unnormalized=True, - one_shot_method="combined", -): - """Inclusive model of expression dynamics considers splicing, metabolic labeling and protein translation. It supports - learning high-dimensional velocity vector samples for droplet based (10x, inDrop, drop-seq, etc), scSLAM-seq, NASC-seq - sci-fate, scNT-seq or cite-seq datasets. - - Parameters - ---------- - adata: :class:`~anndata.AnnData` - AnnData object. - tkey: `str` or None (default: None) - The column key for the time label of cells in .obs. Used for either "steady_state" or non-"steady_state" mode or `moment` - mode with labeled data. - filter_gene_mode: `str` (default: `final`) - The string for indicating which mode (one of, {'final', 'basic', 'no'}) of gene filter will be used. - mode: `str` (default: `deterministic`) - String indicates which estimation mode will be used. This parameter should be used in conjunction with assumption_mRNA. - * Available options when the `assumption_mRNA` is 'ss' include: - (1) 'linear_regression': The canonical method from the seminar RNA velocity paper based on deterministic ordinary - differential equations; - (2) 'gmm': The new generalized methods of moments from us that is based on master equations, similar to the - "moment" mode in the excellent scvelo package; - (3) 'negbin': The new method from us that models steady state RNA expression as a negative binomial distribution, - also built upons on master equations. - Note that all those methods require using extreme data points (except negbin) for the estimation. Extreme data points - are defined as the data from cells where the expression of unspliced / spliced or new / total RNA, etc. are in the - top or bottom, 5%, for example. `linear_regression` only considers the mean of RNA species (based on the deterministic - ordinary different equations) while moment based methods (`gmm`, `negbin`) considers both first moment (mean) and - second moment (uncentered variance) of RNA species (based on the stochastic master equations). - * Available options when the `assumption_mRNA` is 'kinetic' include: - (1) 'deterministic': The method based on deterministic ordinary differential equations; - (2) 'stochastic' or `moment`: The new method from us that is based on master equations; - Note that `kinetic` model implicitly assumes the `experiment_type` is not `conventional`. Thus `deterministic`, - `stochastic` (equivalent to `moment`) models are only possible for the labeling experiments. - A "model_selection" mode will be supported soon in which alpha, beta and gamma will be modeled as a function of time. - use_smoothed: `bool` (default: `True`) - Whether to use the smoothed data when calculating velocity for each gene. `use_smoothed` is only relevant when - mode is `linear_regression` (and experiment_type and assumption_mRNA correspond to `conventional` and `ss` implicitly). - group: `str` or None (default: `None`) - The column key/name that identifies the grouping information (for example, clusters that correspond to different cell types) - of cells. This will be used to estimate group-specific (i.e cell-type specific) kinetic parameters. - protein_names: `List` - A list of gene names corresponds to the rows of the measured proteins in the `X_protein` of the `obsm` attribute. - The names have to be included in the adata.var.index. - experiment_type: `str` - single cell RNA-seq experiment type. Available options are: - (1) 'conventional': conventional single-cell RNA-seq experiment; - (2) 'deg': chase/degradation experiment; - (3) 'kin': pulse/synthesis/kinetics experiment; - (4) 'one-shot': one-shot kinetic experiment. - assumption_mRNA: `str` - Parameter estimation assumption for mRNA. Available options are: - (1) 'ss': pseudo steady state; - (2) 'kinetic' or None: degradation and kinetic data without steady state assumption. - If no labelling data exists, assumption_mRNA will automatically set to be 'ss'. For one-shot experiment, assumption_mRNA - is set to be None. However we will use steady state assumption to estimate parameters alpha and gamma either by a deterministic - linear regression or the first order decay approach in line of the sci-fate paper. - assumption_protein: `str` - Parameter estimation assumption for protein. Available options are: - (1) 'ss': pseudo steady state; - NTR_vel: `bool` (default: `True`) - Whether to use NTR (new/total ratio) velocity for labeling datasets. - concat_data: `bool` (default: `False`) - Whether to concatenate data before estimation. If your data is a list of matrices for each time point, this need to be set as True. - log_unnormalized: `bool` (default: `True`) - Whether to log transform the unnormalized data. - - Returns - ------- - adata: :class:`~anndata.AnnData` - An updated AnnData object with estimated kinetic parameters and inferred velocity included. - """ - - if "use_for_dynamics" not in adata.var.columns and "pass_basic_filter" not in adata.var.columns: - filter_gene_mode = "no" - - valid_ind = get_valid_bools(adata, filter_gene_mode) - - if mode == "moment" or (use_smoothed and len([i for i in adata.layers.keys() if i.startswith("M_")]) < 2): - if experiment_type == "kin": - use_smoothed = False - else: - moments(adata) - - valid_adata = adata[:, valid_ind].copy() - if group is not None and group in adata.obs[group]: - _group = adata.obs[group].unique() - else: - _group = ["_all_cells"] - - for cur_grp in _group: - if cur_grp == "_all_cells": - kin_param_pre = "" - cur_cells_bools = np.ones(valid_adata.shape[0], dtype=bool) - subset_adata = valid_adata[cur_cells_bools] - else: - kin_param_pre = group + "_" + cur_grp + "_" - cur_cells_bools = (valid_adata.obs[group] == cur_grp).values - subset_adata = valid_adata[cur_cells_bools] - - ( - U, - Ul, - S, - Sl, - P, - US, - S2, - t, - normalized, - has_splicing, - has_labeling, - has_protein, - ind_for_proteins, - assumption_mRNA, - exp_type, - ) = get_data_for_kin_params_estimation( - subset_adata, - mode, - use_smoothed, - tkey, - protein_names, - experiment_type, - log_unnormalized, - NTR_vel, - ) - - if exp_type is not None: - if experiment_type != exp_type: - main_warning( - "dynamo detects the experiment type of your data as {}, but your input experiment_type " - "is {}".format(exp_type, experiment_type) - ) - - experiment_type = exp_type - assumption_mRNA = "ss" if exp_type == "conventional" and mode == "deterministic" else None - NTR_vel = False - - if mode == "moment" and experiment_type not in ["conventional", "kin"]: - """ - # temporially convert to deterministic mode as moment mode for one-shot, - degradation and other types of labeling experiment is ongoing.""" - - mode = "deterministic" - - if mode == "deterministic" or (experiment_type != "kin" and mode == "moment"): - est = ss_estimation( - U=U, - Ul=Ul, - S=S, - Sl=Sl, - P=P, - US=US, - S2=S2, - t=t, - ind_for_proteins=ind_for_proteins, - experiment_type=experiment_type, - assumption_mRNA=assumption_mRNA, - assumption_protein=assumption_protein, - concat_data=concat_data, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - - if experiment_type in ["one-shot", "one_shot"]: - est.train(one_shot_method=one_shot_method) - else: - est.train() - - alpha, beta, gamma, eta, delta = est.parameters.values() - - U, S = get_U_S_for_velocity_estimation( - subset_adata, - use_smoothed, - has_splicing, - has_labeling, - log_unnormalized, - NTR_vel, - ) - vel = velocity(estimation=est) - vel_U = vel.vel_u(U) - vel_S = vel.vel_s(U, S) - vel_P = vel.vel_p(S, P) - - adata = set_velocity( - adata, - vel_U, - vel_S, - vel_P, - _group, - cur_grp, - cur_cells_bools, - valid_ind, - ind_for_proteins, - ) - - adata = set_param_ss( - adata, - est, - alpha, - beta, - gamma, - eta, - delta, - experiment_type, - _group, - cur_grp, - kin_param_pre, - valid_ind, - ind_for_proteins, - ) - - elif mode == "moment": - adata, Est, t_ind = moment_model(adata, subset_adata, _group, cur_grp, log_unnormalized, tkey) - t_ind += 1 - - params, costs = Est.train() - a, b, alpha_a, alpha_i, beta, gamma = ( - params[:, 0], - params[:, 1], - params[:, 2], - params[:, 3], - params[:, 4], - params[:, 5], - ) - - def fbar(x_a, x_i, a, b): - return b / (a + b) * x_a + a / (a + b) * x_i - - alpha = fbar(alpha_a, alpha_i, a, b)[:, None] - - params = {"alpha": alpha, "beta": beta, "gamma": gamma, "t": t} - vel = velocity(**params) - - U, S = get_U_S_for_velocity_estimation( - subset_adata, - use_smoothed, - has_splicing, - has_labeling, - log_unnormalized, - NTR_vel, - ) - vel_U = vel.vel_u(U) - vel_S = vel.vel_s(U, S) - vel_P = vel.vel_p(S, P) - - adata = set_velocity( - adata, - vel_U, - vel_S, - vel_P, - _group, - cur_grp, - cur_cells_bools, - valid_ind, - ind_for_proteins, - ) - - adata = set_param_kinetic( - adata, - alpha, - a, - b, - alpha_a, - alpha_i, - beta, - gamma, - kin_param_pre, - _group, - cur_grp, - valid_ind, - ) - # add protein related parameters in the moment model below: - elif mode == "model_selection": - main_warning("Not implemented yet.") - - if group is not None and group in adata.obs[group]: - uns_key = group + "_dynamics" - else: - uns_key = "dynamics" - - if has_splicing and has_labeling: - adata.layers["X_U"], adata.layers["X_S"] = ( - adata.layers["X_uu"] + adata.layers["X_ul"], - adata.layers["X_su"] + adata.layers["X_sl"], - ) - - adata.uns[uns_key] = { - "t": t, - "group": group, - "asspt_mRNA": assumption_mRNA, - "experiment_type": experiment_type, - "normalized": normalized, - "mode": mode, - "has_splicing": has_splicing, - "has_labeling": has_labeling, - "has_protein": has_protein, - "use_smoothed": use_smoothed, - "NTR_vel": NTR_vel, - "log_unnormalized": log_unnormalized, - } - - return adata diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py index f29788523..51e399817 100644 --- a/dynamo/tools/deprecated.py +++ b/dynamo/tools/deprecated.py @@ -9,8 +9,23 @@ import functools import numpy as np +from numpy import * from scipy.integrate import odeint -from scipy.optimize import least_squares +from scipy.optimize import curve_fit, least_squares + +from ..dynamo_logger import main_info, main_warning +from .moments import moment_model +from .utils import ( + get_data_for_kin_params_estimation, + get_mapper, + get_U_S_for_velocity_estimation, + get_valid_bools, + set_param_kinetic, + set_param_ss, + set_velocity, +) +from .utils_moments import moments +from .velocity import ss_estimation, velocity def deprecated(func): @@ -27,6 +42,331 @@ def wrapper(*args, **kwargs): return wrapper +# --------------------------------------------------------------------------------------------------- +# deprecated _dynamics_deprecated.py +@deprecated +def _dynamics(*args, **kwargs): + _dynamics_legacy(*args, **kwargs) + + +def _dynamics_legacy( + adata, + tkey=None, + filter_gene_mode="final", + mode="moment", + use_smoothed=True, + group=None, + protein_names=None, + experiment_type=None, + assumption_mRNA=None, + assumption_protein="ss", + NTR_vel=True, + concat_data=False, + log_unnormalized=True, + one_shot_method="combined", +): + """Inclusive model of expression dynamics considers splicing, metabolic labeling and protein translation. It supports + learning high-dimensional velocity vector samples for droplet based (10x, inDrop, drop-seq, etc), scSLAM-seq, NASC-seq + sci-fate, scNT-seq or cite-seq datasets. + + Args: + adata: :class:`~anndata.AnnData` + AnnData object. + tkey: `str` or None (default: None) + The column key for the time label of cells in .obs. Used for either "steady_state" or non-"steady_state" mode or `moment` + mode with labeled data. + filter_gene_mode: `str` (default: `final`) + The string for indicating which mode (one of, {'final', 'basic', 'no'}) of gene filter will be used. + mode: `str` (default: `deterministic`) + String indicates which estimation mode will be used. This parameter should be used in conjunction with assumption_mRNA. + * Available options when the `assumption_mRNA` is 'ss' include: + (1) 'linear_regression': The canonical method from the seminar RNA velocity paper based on deterministic ordinary + differential equations; + (2) 'gmm': The new generalized methods of moments from us that is based on master equations, similar to the + "moment" mode in the excellent scvelo package; + (3) 'negbin': The new method from us that models steady state RNA expression as a negative binomial distribution, + also built upons on master equations. + Note that all those methods require using extreme data points (except negbin) for the estimation. Extreme data points + are defined as the data from cells where the expression of unspliced / spliced or new / total RNA, etc. are in the + top or bottom, 5%, for example. `linear_regression` only considers the mean of RNA species (based on the deterministic + ordinary different equations) while moment based methods (`gmm`, `negbin`) considers both first moment (mean) and + second moment (uncentered variance) of RNA species (based on the stochastic master equations). + * Available options when the `assumption_mRNA` is 'kinetic' include: + (1) 'deterministic': The method based on deterministic ordinary differential equations; + (2) 'stochastic' or `moment`: The new method from us that is based on master equations; + Note that `kinetic` model implicitly assumes the `experiment_type` is not `conventional`. Thus `deterministic`, + `stochastic` (equivalent to `moment`) models are only possible for the labeling experiments. + A "model_selection" mode will be supported soon in which alpha, beta and gamma will be modeled as a function of time. + use_smoothed: `bool` (default: `True`) + Whether to use the smoothed data when calculating velocity for each gene. `use_smoothed` is only relevant when + mode is `linear_regression` (and experiment_type and assumption_mRNA correspond to `conventional` and `ss` implicitly). + group: `str` or None (default: `None`) + The column key/name that identifies the grouping information (for example, clusters that correspond to different cell types) + of cells. This will be used to estimate group-specific (i.e cell-type specific) kinetic parameters. + protein_names: `List` + A list of gene names corresponds to the rows of the measured proteins in the `X_protein` of the `obsm` attribute. + The names have to be included in the adata.var.index. + experiment_type: `str` + single cell RNA-seq experiment type. Available options are: + (1) 'conventional': conventional single-cell RNA-seq experiment; + (2) 'deg': chase/degradation experiment; + (3) 'kin': pulse/synthesis/kinetics experiment; + (4) 'one-shot': one-shot kinetic experiment. + assumption_mRNA: `str` + Parameter estimation assumption for mRNA. Available options are: + (1) 'ss': pseudo steady state; + (2) 'kinetic' or None: degradation and kinetic data without steady state assumption. + If no labelling data exists, assumption_mRNA will automatically set to be 'ss'. For one-shot experiment, assumption_mRNA + is set to be None. However we will use steady state assumption to estimate parameters alpha and gamma either by a deterministic + linear regression or the first order decay approach in line of the sci-fate paper. + assumption_protein: `str` + Parameter estimation assumption for protein. Available options are: + (1) 'ss': pseudo steady state; + NTR_vel: `bool` (default: `True`) + Whether to use NTR (new/total ratio) velocity for labeling datasets. + concat_data: `bool` (default: `False`) + Whether to concatenate data before estimation. If your data is a list of matrices for each time point, this need to be set as True. + log_unnormalized: `bool` (default: `True`) + Whether to log transform the unnormalized data. + + Returns: + adata: :class:`~anndata.AnnData` + An updated AnnData object with estimated kinetic parameters and inferred velocity included. + """ + + if "use_for_dynamics" not in adata.var.columns and "pass_basic_filter" not in adata.var.columns: + filter_gene_mode = "no" + + valid_ind = get_valid_bools(adata, filter_gene_mode) + + if mode == "moment" or (use_smoothed and len([i for i in adata.layers.keys() if i.startswith("M_")]) < 2): + if experiment_type == "kin": + use_smoothed = False + else: + moments(adata) + + valid_adata = adata[:, valid_ind].copy() + if group is not None and group in adata.obs[group]: + _group = adata.obs[group].unique() + else: + _group = ["_all_cells"] + + for cur_grp in _group: + if cur_grp == "_all_cells": + kin_param_pre = "" + cur_cells_bools = np.ones(valid_adata.shape[0], dtype=bool) + subset_adata = valid_adata[cur_cells_bools] + else: + kin_param_pre = group + "_" + cur_grp + "_" + cur_cells_bools = (valid_adata.obs[group] == cur_grp).values + subset_adata = valid_adata[cur_cells_bools] + + ( + U, + Ul, + S, + Sl, + P, + US, + S2, + t, + normalized, + has_splicing, + has_labeling, + has_protein, + ind_for_proteins, + assumption_mRNA, + exp_type, + ) = get_data_for_kin_params_estimation( + subset_adata, + mode, + use_smoothed, + tkey, + protein_names, + experiment_type, + log_unnormalized, + NTR_vel, + ) + + if exp_type is not None: + if experiment_type != exp_type: + main_warning( + "dynamo detects the experiment type of your data as {}, but your input experiment_type " + "is {}".format(exp_type, experiment_type) + ) + + experiment_type = exp_type + assumption_mRNA = "ss" if exp_type == "conventional" and mode == "deterministic" else None + NTR_vel = False + + if mode == "moment" and experiment_type not in ["conventional", "kin"]: + """ + # temporially convert to deterministic mode as moment mode for one-shot, + degradation and other types of labeling experiment is ongoing.""" + + mode = "deterministic" + + if mode == "deterministic" or (experiment_type != "kin" and mode == "moment"): + est = ss_estimation( + U=U, + Ul=Ul, + S=S, + Sl=Sl, + P=P, + US=US, + S2=S2, + t=t, + ind_for_proteins=ind_for_proteins, + experiment_type=experiment_type, + assumption_mRNA=assumption_mRNA, + assumption_protein=assumption_protein, + concat_data=concat_data, + ) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + if experiment_type in ["one-shot", "one_shot"]: + est.train(one_shot_method=one_shot_method) + else: + est.train() + + alpha, beta, gamma, eta, delta = est.parameters.values() + + U, S = get_U_S_for_velocity_estimation( + subset_adata, + use_smoothed, + has_splicing, + has_labeling, + log_unnormalized, + NTR_vel, + ) + vel = velocity(estimation=est) + vel_U = vel.vel_u(U) + vel_S = vel.vel_s(U, S) + vel_P = vel.vel_p(S, P) + + adata = set_velocity( + adata, + vel_U, + vel_S, + vel_P, + _group, + cur_grp, + cur_cells_bools, + valid_ind, + ind_for_proteins, + ) + + adata = set_param_ss( + adata, + est, + alpha, + beta, + gamma, + eta, + delta, + experiment_type, + _group, + cur_grp, + kin_param_pre, + valid_ind, + ind_for_proteins, + ) + + elif mode == "moment": + adata, Est, t_ind = moment_model(adata, subset_adata, _group, cur_grp, log_unnormalized, tkey) + t_ind += 1 + + params, costs = Est.train() + a, b, alpha_a, alpha_i, beta, gamma = ( + params[:, 0], + params[:, 1], + params[:, 2], + params[:, 3], + params[:, 4], + params[:, 5], + ) + + def fbar(x_a, x_i, a, b): + return b / (a + b) * x_a + a / (a + b) * x_i + + alpha = fbar(alpha_a, alpha_i, a, b)[:, None] + + params = {"alpha": alpha, "beta": beta, "gamma": gamma, "t": t} + vel = velocity(**params) + + U, S = get_U_S_for_velocity_estimation( + subset_adata, + use_smoothed, + has_splicing, + has_labeling, + log_unnormalized, + NTR_vel, + ) + vel_U = vel.vel_u(U) + vel_S = vel.vel_s(U, S) + vel_P = vel.vel_p(S, P) + + adata = set_velocity( + adata, + vel_U, + vel_S, + vel_P, + _group, + cur_grp, + cur_cells_bools, + valid_ind, + ind_for_proteins, + ) + + adata = set_param_kinetic( + adata, + alpha, + a, + b, + alpha_a, + alpha_i, + beta, + gamma, + kin_param_pre, + _group, + cur_grp, + valid_ind, + ) + # add protein related parameters in the moment model below: + elif mode == "model_selection": + main_warning("Not implemented yet.") + + if group is not None and group in adata.obs[group]: + uns_key = group + "_dynamics" + else: + uns_key = "dynamics" + + if has_splicing and has_labeling: + adata.layers["X_U"], adata.layers["X_S"] = ( + adata.layers["X_uu"] + adata.layers["X_ul"], + adata.layers["X_su"] + adata.layers["X_sl"], + ) + + adata.uns[uns_key] = { + "t": t, + "group": group, + "asspt_mRNA": assumption_mRNA, + "experiment_type": experiment_type, + "normalized": normalized, + "mode": mode, + "has_splicing": has_splicing, + "has_labeling": has_labeling, + "has_protein": has_protein, + "use_smoothed": use_smoothed, + "NTR_vel": NTR_vel, + "log_unnormalized": log_unnormalized, + } + + return adata + + # --------------------------------------------------------------------------------------------------- # deprecated dynamo_bk.py @deprecated @@ -337,3 +677,612 @@ def _fit_gamma_dynamo_fitting_legacy(u, s): var_s = s.dot(s) / len(s) - np.mean(s) ** 2 gamma = cov / var_s return gamma + + +# --------------------------------------------------------------------------------------------------- +# deprecated utils_moments_deprecated.py +class moments: + def __init__( + self, + a=None, + b=None, + la=None, + alpha_a=None, + alpha_i=None, + sigma=None, + beta=None, + gamma=None, + ): + # species + self.ua = 0 + self.ui = 1 + self.wa = 2 + self.wi = 3 + self.xa = 4 + self.xi = 5 + self.ya = 6 + self.yi = 7 + self.uu = 8 + self.ww = 9 + self.xx = 10 + self.yy = 11 + self.uw = 12 + self.ux = 13 + self.uy = 14 + self.wy = 15 + + self.n_species = 16 + + # solution + self.t = None + self.x = None + self.x0 = zeros(self.n_species) + self.K = None + self.p = None + + # parameters + if not ( + a is None + or b is None + or la is None + or alpha_a is None + or alpha_i is None + or sigma is None + or beta is None + or gamma is None + ): + self.set_params(a, b, la, alpha_a, alpha_i, sigma, beta, gamma) + + def ode_moments(self, x, t): + dx = zeros(len(x)) + # parameters + a = self.a + b = self.b + la = self.la + aa = self.aa + ai = self.ai + si = self.si + be = self.be + ga = self.ga + + # first moments + dx[self.ua] = la * aa - be * x[self.ua] + a * (x[self.ui] - x[self.ua]) + dx[self.ui] = la * ai - be * x[self.ui] - b * (x[self.ui] - x[self.ua]) + dx[self.wa] = (1 - la) * aa - be * x[self.wa] + a * (x[self.wi] - x[self.wa]) + dx[self.wi] = (1 - la) * ai - be * x[self.wi] - b * (x[self.wi] - x[self.wa]) + dx[self.xa] = be * (1 - si) * x[self.ua] - ga * x[self.xa] + a * (x[self.xi] - x[self.xa]) + dx[self.xi] = be * (1 - si) * x[self.ui] - ga * x[self.xi] - b * (x[self.xi] - x[self.xa]) + dx[self.ya] = be * si * x[self.ua] + be * x[self.wa] - ga * x[self.ya] + a * (x[self.yi] - x[self.ya]) + dx[self.yi] = be * si * x[self.ui] + be * x[self.wi] - ga * x[self.yi] - b * (x[self.yi] - x[self.ya]) + + # second moments + dx[self.uu] = 2 * la * self.fbar(aa * x[self.ua], ai * x[self.ui]) - 2 * be * x[self.uu] + dx[self.ww] = 2 * (1 - la) * self.fbar(self.aa * x[self.wa], ai * x[self.wi]) - 2 * be * x[self.ww] + dx[self.xx] = 2 * be * (1 - si) * x[self.ux] - 2 * ga * x[self.xx] + dx[self.yy] = 2 * si * be * x[self.uy] + 2 * be * x[self.wy] - 2 * ga * x[self.yy] + dx[self.uw] = ( + la * self.fbar(aa * x[self.wa], ai * x[self.wi]) + + (1 - la) * self.fbar(aa * x[self.ua], ai * x[self.ui]) + - 2 * be * x[self.uw] + ) + dx[self.ux] = ( + la * self.fbar(aa * x[self.xa], ai * x[self.xi]) + be * (1 - si) * x[self.uu] - (be + ga) * x[self.ux] + ) + dx[self.uy] = ( + la * self.fbar(aa * x[self.ya], ai * x[self.yi]) + + si * be * x[self.uu] + + be * x[self.uw] + - (be + ga) * x[self.uy] + ) + dx[self.wy] = ( + (1 - la) * self.fbar(aa * x[self.ya], ai * x[self.yi]) + + si * be * x[self.uw] + + be * x[self.ww] + - (be + ga) * x[self.wy] + ) + + return dx + + def integrate(self, t, x0=None): + if x0 is None: + x0 = self.x0 + else: + self.x0 = x0 + sol = odeint(self.ode_moments, x0, t) + self.x = sol + self.t = t + return sol + + def fbar(self, x_a, x_i): + return self.b / (self.a + self.b) * x_a + self.a / (self.a + self.b) * x_i + + def set_params(self, a, b, la, alpha_a, alpha_i, sigma, beta, gamma): + self.a = a + self.b = b + self.la = la + self.aa = alpha_a + self.ai = alpha_i + self.si = sigma + self.be = beta + self.ga = gamma + + # reset solutions + self.t = None + self.x = None + self.K = None + self.p = None + + def get_all_central_moments(self): + ret = zeros((8, len(self.t))) + ret[0] = self.get_nu() + ret[1] = self.get_nw() + ret[2] = self.get_nx() + ret[3] = self.get_ny() + ret[4] = self.get_var_nu() + ret[5] = self.get_var_nw() + ret[6] = self.get_var_nx() + ret[7] = self.get_var_ny() + return ret + + def get_nosplice_central_moments(self): + ret = zeros((4, len(self.t))) + ret[0] = self.get_n_labeled() + ret[1] = self.get_n_unlabeled() + ret[2] = self.get_var_labeled() + ret[3] = self.get_var_unlabeled() + return ret + + def get_central_moments(self, keys=None): + if keys is None: + ret = self.get_all_centeral_moments() + else: + ret = zeros((len(keys) * 2, len(self.t))) + i = 0 + if "ul" in keys: + ret[i] = self.get_nu() + ret[i + 1] = self.get_var_nu() + i += 2 + if "uu" in keys: + ret[i] = self.get_nw() + ret[i + 1] = self.get_var_nw() + i += 2 + if "sl" in keys: + ret[i] = self.get_nx() + ret[i + 1] = self.get_var_nx() + i += 2 + if "su" in keys: + ret[i] = self.get_ny() + ret[i + 1] = self.get_var_ny() + i += 2 + return ret + + def get_nu(self): + return self.fbar(self.x[:, self.ua], self.x[:, self.ui]) + + def get_nw(self): + return self.fbar(self.x[:, self.wa], self.x[:, self.wi]) + + def get_nx(self): + return self.fbar(self.x[:, self.xa], self.x[:, self.xi]) + + def get_ny(self): + return self.fbar(self.x[:, self.ya], self.x[:, self.yi]) + + def get_n_labeled(self): + return self.get_nu() + self.get_nx() + + def get_n_unlabeled(self): + return self.get_nw() + self.get_ny() + + def get_var_nu(self): + c = self.get_nu() + return self.x[:, self.uu] + c - c**2 + + def get_var_nw(self): + c = self.get_nw() + return self.x[:, self.ww] + c - c**2 + + def get_var_nx(self): + c = self.get_nx() + return self.x[:, self.xx] + c - c**2 + + def get_var_ny(self): + c = self.get_ny() + return self.x[:, self.yy] + c - c**2 + + def get_cov_ux(self): + cu = self.get_nu() + cx = self.get_nx() + return self.x[:, self.ux] - cu * cx + + def get_cov_wy(self): + cw = self.get_nw() + cy = self.get_ny() + return self.x[:, self.wy] - cw * cy + + def get_var_labeled(self): + return self.get_var_nu() + self.get_var_nx() + 2 * self.get_cov_ux() + + def get_var_unlabeled(self): + return self.get_var_nw() + self.get_var_ny() + 2 * self.get_cov_wy() + + def computeKnp(self): + # parameters + a = self.a + b = self.b + la = self.la + aa = self.aa + ai = self.ai + si = self.si + be = self.be + ga = self.ga + + K = zeros((self.n_species, self.n_species)) + # E1 + K[self.ua, self.ua] = -be - a + K[self.ua, self.ui] = a + K[self.ui, self.ua] = b + K[self.ui, self.ui] = -be - b + K[self.wa, self.wa] = -be - a + K[self.wa, self.wi] = a + K[self.wi, self.wa] = b + K[self.wi, self.wi] = -be - b + + # E2 + K[self.xa, self.xa] = -ga - a + K[self.xa, self.xi] = a + K[self.xi, self.xa] = b + K[self.xi, self.xi] = -ga - b + K[self.ya, self.ya] = -ga - a + K[self.ya, self.yi] = a + K[self.yi, self.ya] = b + K[self.yi, self.yi] = -ga - b + + # E3 + K[self.uu, self.uu] = -2 * be + K[self.ww, self.ww] = -2 * be + K[self.xx, self.xx] = -2 * ga + K[self.yy, self.yy] = -2 * ga + + # E4 + K[self.uw, self.uw] = -2 * be + K[self.ux, self.ux] = -be - ga + K[self.uy, self.uy] = -be - ga + K[self.wy, self.wy] = -be - ga + K[self.uy, self.uw] = be + K[self.wy, self.uw] = si * be + + # F21 + K[self.xa, self.ua] = (1 - si) * be + K[self.xi, self.ui] = (1 - si) * be + K[self.ya, self.wa] = be + K[self.ya, self.ua] = si * be + K[self.yi, self.wi] = be + K[self.yi, self.ui] = si * be + + # F31 + K[self.uu, self.ua] = 2 * la * aa * b / (a + b) + K[self.uu, self.ui] = 2 * la * ai * a / (a + b) + K[self.ww, self.wa] = 2 * (1 - la) * aa * b / (a + b) + K[self.ww, self.wi] = 2 * (1 - la) * ai * a / (a + b) + + # F34 + K[self.xx, self.ux] = 2 * (1 - si) * be + K[self.yy, self.uy] = 2 * si * be + K[self.yy, self.wy] = 2 * be + + # F41 + K[self.uw, self.ua] = (1 - la) * aa * b / (a + b) + K[self.uw, self.ui] = (1 - la) * ai * a / (a + b) + K[self.uw, self.wa] = la * aa * b / (a + b) + K[self.uw, self.wi] = la * ai * a / (a + b) + + # F42 + K[self.ux, self.xa] = la * aa * b / (a + b) + K[self.ux, self.xi] = la * ai * a / (a + b) + K[self.uy, self.ya] = la * aa * b / (a + b) + K[self.uy, self.yi] = la * ai * a / (a + b) + K[self.wy, self.ya] = (1 - la) * aa * b / (a + b) + K[self.wy, self.yi] = (1 - la) * ai * a / (a + b) + + # F43 + K[self.ux, self.uu] = (1 - si) * be + K[self.uy, self.uu] = si * be + K[self.wy, self.ww] = be + + p = zeros(self.n_species) + p[self.ua] = la * aa + p[self.ui] = la * ai + p[self.wa] = (1 - la) * aa + p[self.wi] = (1 - la) * ai + + return K, p + + def solve(self, t, x0=None): + t0 = t[0] + if x0 is None: + x0 = self.x0 + else: + self.x0 = x0 + + if self.K is None or self.p is None: + K, p = self.computeKnp() + self.K = K + self.p = p + else: + K = self.K + p = self.p + x_ss = linalg.solve(K, p) + # x_ss = linalg.inv(K).dot(p) + y0 = x0 + x_ss + + D, U = linalg.eig(K) + V = linalg.inv(U) + D, U, V = map(real, (D, U, V)) + expD = exp(D) + x = zeros((len(t), self.n_species)) + x[0] = x0 + for i in range(1, len(t)): + x[i] = U.dot(diag(expD ** (t[i] - t0))).dot(V).dot(y0) - x_ss + self.x = x + self.t = t + return x + + +class moments_simple: + def __init__( + self, + a=None, + b=None, + la=None, + alpha_a=None, + alpha_i=None, + sigma=None, + beta=None, + gamma=None, + ): + # species + self._u = 0 + self._w = 1 + self._x = 2 + self._y = 3 + + self.n_species = 4 + + # solution + self.t = None + self.x = None + self.x0 = zeros(self.n_species) + self.K = None + self.p = None + + # parameters + if not ( + a is None + or b is None + or la is None + or alpha_a is None + or alpha_i is None + or sigma is None + or beta is None + or gamma is None + ): + self.set_params(a, b, la, alpha_a, alpha_i, sigma, beta, gamma) + + def set_initial_condition(self, nu0, nw0, nx0, ny0): + x = zeros(self.n_species) + x[self._u] = nu0 + x[self._w] = nw0 + x[self._x] = nx0 + x[self._y] = ny0 + + self.x0 = x + return x + + def get_x_velocity(self, nu0, nx0): + return self.be * (1 - self.si) * nu0 - self.ga * nx0 + + def get_y_velocity(self, nu0, nw0, ny0): + return self.be * self.si * nu0 + self.be * nw0 - self.ga * ny0 + + def fbar(self, x_a, x_i): + return self.b / (self.a + self.b) * x_a + self.a / (self.a + self.b) * x_i + + def set_params(self, a, b, la, alpha_a, alpha_i, sigma, beta, gamma): + self.a = a + self.b = b + self.la = la + self.aa = alpha_a + self.ai = alpha_i + self.si = sigma + self.be = beta + self.ga = gamma + + # reset solutions + self.t = None + self.x = None + self.K = None + self.p = None + + def get_total(self): + return sum(self.x, 1) + + def computeKnp(self): + # parameters + la = self.la + aa = self.aa + ai = self.ai + si = self.si + be = self.be + ga = self.ga + + K = zeros((self.n_species, self.n_species)) + + # Diagonal + K[self._u, self._u] = -be + K[self._w, self._w] = -be + K[self._x, self._x] = -ga + K[self._y, self._y] = -ga + + # off-diagonal + K[self._x, self._u] = be * (1 - si) + K[self._y, self._u] = si * be + K[self._y, self._w] = be + + p = zeros(self.n_species) + p[self._u] = la * self.fbar(aa, ai) + p[self._w] = (1 - la) * self.fbar(aa, ai) + + return K, p + + def solve(self, t, x0=None): + t0 = t[0] + if x0 is None: + x0 = self.x0 + else: + self.x0 = x0 + + if self.K is None or self.p is None: + K, p = self.computeKnp() + self.K = K + self.p = p + else: + K = self.K + p = self.p + x_ss = linalg.solve(K, p) + y0 = x0 + x_ss + + D, U = linalg.eig(K) + V = linalg.inv(U) + D, U, V = map(real, (D, U, V)) + expD = exp(D) + x = zeros((len(t), self.n_species)) + x[0] = x0 + for i in range(1, len(t)): + x[i] = U.dot(diag(expD ** (t[i] - t0))).dot(V).dot(y0) - x_ss + self.x = x + self.t = t + return x + + +class estimation: + def __init__(self, ranges, x0=None): + self.ranges = ranges + self.n_params = len(ranges) + self.simulator = moments() + if not x0 is None: + self.simulator.x0 = x0 + + def sample_p0(self, samples=1, method="lhs"): + ret = zeros((samples, self.n_params)) + if method == "lhs": + ret = self._lhsclassic(samples) + for i in range(self.n_params): + ret[:, i] = ret[:, i] * (self.ranges[i][1] - self.ranges[i][0]) + self.ranges[i][0] + else: + for n in range(samples): + for i in range(self.n_params): + r = random.rand() + ret[n, i] = r * (self.ranges[i][1] - self.ranges[i][0]) + self.ranges[i][0] + return ret + + def _lhsclassic(self, samples): + # From PyDOE + # Generate the intervals + cut = linspace(0, 1, samples + 1) + + # Fill points uniformly in each interval + u = random.rand(samples, self.n_params) + a = cut[:samples] + b = cut[1 : samples + 1] + rdpoints = zeros_like(u) + for j in range(self.n_params): + rdpoints[:, j] = u[:, j] * (b - a) + a + + # Make the random pairings + H = zeros_like(rdpoints) + for j in range(self.n_params): + order = random.permutation(range(samples)) + H[:, j] = rdpoints[order, j] + + return H + + def get_bound(self, index): + ret = zeros(self.n_params) + for i in range(self.n_params): + ret[i] = self.ranges[i][index] + return ret + + def normalize_data(self, X): + # ret = zeros(X.shape) + # for i in range(len(X)): + # x = X[i] + # #ret[i] = x / max(x) + # ret[i] = log10(x + 1) + return log10(X + 1) + + def f_curve_fit(self, t, *params): + self.simulator.set_params(*params) + self.simulator.integrate(t, self.simulator.x0) + ret = self.simulator.get_all_central_moments() + ret = self.normalize_data(ret) + return ret.flatten() + + def f_lsq(self, params, t, x_data_norm, method="analytical", experiment_type=None): + self.simulator.set_params(*params) + if method == "numerical": + self.simulator.integrate(t, self.simulator.x0) + elif method == "analytical": + self.simulator.solve(t, self.simulator.x0) + if experiment_type is None: + ret = self.simulator.get_all_central_moments() + elif experiment_type == "nosplice": + ret = self.simulator.get_nosplice_central_moments() + elif experiment_type == "label": + ret = self.simulator.get_central_moments(["ul", "sl"]) + ret = self.normalize_data(ret).flatten() + ret[isnan(ret)] = 0 + return ret - x_data_norm + + def fit(self, t, x_data, p0=None, bounds=None): + if p0 is None: + p0 = self.sample_p0() + x_data_norm = self.normalize_data(x_data) + if bounds is None: + bounds = (self.get_bound(0), self.get_bound(1)) + popt, pcov = curve_fit(self.f_curve_fit, t, x_data_norm.flatten(), p0=p0, bounds=bounds) + return popt, pcov + + def fit_lsq( + self, + t, + x_data, + p0=None, + n_p0=1, + bounds=None, + sample_method="lhs", + method="analytical", + experiment_type=None, + ): + if p0 is None: + p0 = self.sample_p0(n_p0, sample_method) + else: + if p0.ndim == 1: + p0 = [p0] + n_p0 = len(p0) + x_data_norm = self.normalize_data(x_data) + if bounds is None: + bounds = (self.get_bound(0), self.get_bound(1)) + + costs = zeros(n_p0) + X = [] + for i in range(n_p0): + ret = least_squares( + lambda p: self.f_lsq(p, t, x_data_norm.flatten(), method, experiment_type), + p0[i], + bounds=bounds, + ) + costs[i] = ret.cost + X.append(ret.x) + i_min = argmin(costs) + return X[i_min], costs[i_min] diff --git a/dynamo/tools/utils_moments_deprecated.py b/dynamo/tools/utils_moments_deprecated.py deleted file mode 100755 index b586a0eba..000000000 --- a/dynamo/tools/utils_moments_deprecated.py +++ /dev/null @@ -1,618 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Tue Mar 12 08:27:36 2019 - -@author: yaz -""" - -from numpy import * -from scipy.integrate import odeint -from scipy.optimize import curve_fit, least_squares - - -class moments: - def __init__( - self, - a=None, - b=None, - la=None, - alpha_a=None, - alpha_i=None, - sigma=None, - beta=None, - gamma=None, - ): - # species - self.ua = 0 - self.ui = 1 - self.wa = 2 - self.wi = 3 - self.xa = 4 - self.xi = 5 - self.ya = 6 - self.yi = 7 - self.uu = 8 - self.ww = 9 - self.xx = 10 - self.yy = 11 - self.uw = 12 - self.ux = 13 - self.uy = 14 - self.wy = 15 - - self.n_species = 16 - - # solution - self.t = None - self.x = None - self.x0 = zeros(self.n_species) - self.K = None - self.p = None - - # parameters - if not ( - a is None - or b is None - or la is None - or alpha_a is None - or alpha_i is None - or sigma is None - or beta is None - or gamma is None - ): - self.set_params(a, b, la, alpha_a, alpha_i, sigma, beta, gamma) - - def ode_moments(self, x, t): - dx = zeros(len(x)) - # parameters - a = self.a - b = self.b - la = self.la - aa = self.aa - ai = self.ai - si = self.si - be = self.be - ga = self.ga - - # first moments - dx[self.ua] = la * aa - be * x[self.ua] + a * (x[self.ui] - x[self.ua]) - dx[self.ui] = la * ai - be * x[self.ui] - b * (x[self.ui] - x[self.ua]) - dx[self.wa] = (1 - la) * aa - be * x[self.wa] + a * (x[self.wi] - x[self.wa]) - dx[self.wi] = (1 - la) * ai - be * x[self.wi] - b * (x[self.wi] - x[self.wa]) - dx[self.xa] = be * (1 - si) * x[self.ua] - ga * x[self.xa] + a * (x[self.xi] - x[self.xa]) - dx[self.xi] = be * (1 - si) * x[self.ui] - ga * x[self.xi] - b * (x[self.xi] - x[self.xa]) - dx[self.ya] = be * si * x[self.ua] + be * x[self.wa] - ga * x[self.ya] + a * (x[self.yi] - x[self.ya]) - dx[self.yi] = be * si * x[self.ui] + be * x[self.wi] - ga * x[self.yi] - b * (x[self.yi] - x[self.ya]) - - # second moments - dx[self.uu] = 2 * la * self.fbar(aa * x[self.ua], ai * x[self.ui]) - 2 * be * x[self.uu] - dx[self.ww] = 2 * (1 - la) * self.fbar(self.aa * x[self.wa], ai * x[self.wi]) - 2 * be * x[self.ww] - dx[self.xx] = 2 * be * (1 - si) * x[self.ux] - 2 * ga * x[self.xx] - dx[self.yy] = 2 * si * be * x[self.uy] + 2 * be * x[self.wy] - 2 * ga * x[self.yy] - dx[self.uw] = ( - la * self.fbar(aa * x[self.wa], ai * x[self.wi]) - + (1 - la) * self.fbar(aa * x[self.ua], ai * x[self.ui]) - - 2 * be * x[self.uw] - ) - dx[self.ux] = ( - la * self.fbar(aa * x[self.xa], ai * x[self.xi]) + be * (1 - si) * x[self.uu] - (be + ga) * x[self.ux] - ) - dx[self.uy] = ( - la * self.fbar(aa * x[self.ya], ai * x[self.yi]) - + si * be * x[self.uu] - + be * x[self.uw] - - (be + ga) * x[self.uy] - ) - dx[self.wy] = ( - (1 - la) * self.fbar(aa * x[self.ya], ai * x[self.yi]) - + si * be * x[self.uw] - + be * x[self.ww] - - (be + ga) * x[self.wy] - ) - - return dx - - def integrate(self, t, x0=None): - if x0 is None: - x0 = self.x0 - else: - self.x0 = x0 - sol = odeint(self.ode_moments, x0, t) - self.x = sol - self.t = t - return sol - - def fbar(self, x_a, x_i): - return self.b / (self.a + self.b) * x_a + self.a / (self.a + self.b) * x_i - - def set_params(self, a, b, la, alpha_a, alpha_i, sigma, beta, gamma): - self.a = a - self.b = b - self.la = la - self.aa = alpha_a - self.ai = alpha_i - self.si = sigma - self.be = beta - self.ga = gamma - - # reset solutions - self.t = None - self.x = None - self.K = None - self.p = None - - def get_all_central_moments(self): - ret = zeros((8, len(self.t))) - ret[0] = self.get_nu() - ret[1] = self.get_nw() - ret[2] = self.get_nx() - ret[3] = self.get_ny() - ret[4] = self.get_var_nu() - ret[5] = self.get_var_nw() - ret[6] = self.get_var_nx() - ret[7] = self.get_var_ny() - return ret - - def get_nosplice_central_moments(self): - ret = zeros((4, len(self.t))) - ret[0] = self.get_n_labeled() - ret[1] = self.get_n_unlabeled() - ret[2] = self.get_var_labeled() - ret[3] = self.get_var_unlabeled() - return ret - - def get_central_moments(self, keys=None): - if keys is None: - ret = self.get_all_centeral_moments() - else: - ret = zeros((len(keys) * 2, len(self.t))) - i = 0 - if "ul" in keys: - ret[i] = self.get_nu() - ret[i + 1] = self.get_var_nu() - i += 2 - if "uu" in keys: - ret[i] = self.get_nw() - ret[i + 1] = self.get_var_nw() - i += 2 - if "sl" in keys: - ret[i] = self.get_nx() - ret[i + 1] = self.get_var_nx() - i += 2 - if "su" in keys: - ret[i] = self.get_ny() - ret[i + 1] = self.get_var_ny() - i += 2 - return ret - - def get_nu(self): - return self.fbar(self.x[:, self.ua], self.x[:, self.ui]) - - def get_nw(self): - return self.fbar(self.x[:, self.wa], self.x[:, self.wi]) - - def get_nx(self): - return self.fbar(self.x[:, self.xa], self.x[:, self.xi]) - - def get_ny(self): - return self.fbar(self.x[:, self.ya], self.x[:, self.yi]) - - def get_n_labeled(self): - return self.get_nu() + self.get_nx() - - def get_n_unlabeled(self): - return self.get_nw() + self.get_ny() - - def get_var_nu(self): - c = self.get_nu() - return self.x[:, self.uu] + c - c**2 - - def get_var_nw(self): - c = self.get_nw() - return self.x[:, self.ww] + c - c**2 - - def get_var_nx(self): - c = self.get_nx() - return self.x[:, self.xx] + c - c**2 - - def get_var_ny(self): - c = self.get_ny() - return self.x[:, self.yy] + c - c**2 - - def get_cov_ux(self): - cu = self.get_nu() - cx = self.get_nx() - return self.x[:, self.ux] - cu * cx - - def get_cov_wy(self): - cw = self.get_nw() - cy = self.get_ny() - return self.x[:, self.wy] - cw * cy - - def get_var_labeled(self): - return self.get_var_nu() + self.get_var_nx() + 2 * self.get_cov_ux() - - def get_var_unlabeled(self): - return self.get_var_nw() + self.get_var_ny() + 2 * self.get_cov_wy() - - def computeKnp(self): - # parameters - a = self.a - b = self.b - la = self.la - aa = self.aa - ai = self.ai - si = self.si - be = self.be - ga = self.ga - - K = zeros((self.n_species, self.n_species)) - # E1 - K[self.ua, self.ua] = -be - a - K[self.ua, self.ui] = a - K[self.ui, self.ua] = b - K[self.ui, self.ui] = -be - b - K[self.wa, self.wa] = -be - a - K[self.wa, self.wi] = a - K[self.wi, self.wa] = b - K[self.wi, self.wi] = -be - b - - # E2 - K[self.xa, self.xa] = -ga - a - K[self.xa, self.xi] = a - K[self.xi, self.xa] = b - K[self.xi, self.xi] = -ga - b - K[self.ya, self.ya] = -ga - a - K[self.ya, self.yi] = a - K[self.yi, self.ya] = b - K[self.yi, self.yi] = -ga - b - - # E3 - K[self.uu, self.uu] = -2 * be - K[self.ww, self.ww] = -2 * be - K[self.xx, self.xx] = -2 * ga - K[self.yy, self.yy] = -2 * ga - - # E4 - K[self.uw, self.uw] = -2 * be - K[self.ux, self.ux] = -be - ga - K[self.uy, self.uy] = -be - ga - K[self.wy, self.wy] = -be - ga - K[self.uy, self.uw] = be - K[self.wy, self.uw] = si * be - - # F21 - K[self.xa, self.ua] = (1 - si) * be - K[self.xi, self.ui] = (1 - si) * be - K[self.ya, self.wa] = be - K[self.ya, self.ua] = si * be - K[self.yi, self.wi] = be - K[self.yi, self.ui] = si * be - - # F31 - K[self.uu, self.ua] = 2 * la * aa * b / (a + b) - K[self.uu, self.ui] = 2 * la * ai * a / (a + b) - K[self.ww, self.wa] = 2 * (1 - la) * aa * b / (a + b) - K[self.ww, self.wi] = 2 * (1 - la) * ai * a / (a + b) - - # F34 - K[self.xx, self.ux] = 2 * (1 - si) * be - K[self.yy, self.uy] = 2 * si * be - K[self.yy, self.wy] = 2 * be - - # F41 - K[self.uw, self.ua] = (1 - la) * aa * b / (a + b) - K[self.uw, self.ui] = (1 - la) * ai * a / (a + b) - K[self.uw, self.wa] = la * aa * b / (a + b) - K[self.uw, self.wi] = la * ai * a / (a + b) - - # F42 - K[self.ux, self.xa] = la * aa * b / (a + b) - K[self.ux, self.xi] = la * ai * a / (a + b) - K[self.uy, self.ya] = la * aa * b / (a + b) - K[self.uy, self.yi] = la * ai * a / (a + b) - K[self.wy, self.ya] = (1 - la) * aa * b / (a + b) - K[self.wy, self.yi] = (1 - la) * ai * a / (a + b) - - # F43 - K[self.ux, self.uu] = (1 - si) * be - K[self.uy, self.uu] = si * be - K[self.wy, self.ww] = be - - p = zeros(self.n_species) - p[self.ua] = la * aa - p[self.ui] = la * ai - p[self.wa] = (1 - la) * aa - p[self.wi] = (1 - la) * ai - - return K, p - - def solve(self, t, x0=None): - t0 = t[0] - if x0 is None: - x0 = self.x0 - else: - self.x0 = x0 - - if self.K is None or self.p is None: - K, p = self.computeKnp() - self.K = K - self.p = p - else: - K = self.K - p = self.p - x_ss = linalg.solve(K, p) - # x_ss = linalg.inv(K).dot(p) - y0 = x0 + x_ss - - D, U = linalg.eig(K) - V = linalg.inv(U) - D, U, V = map(real, (D, U, V)) - expD = exp(D) - x = zeros((len(t), self.n_species)) - x[0] = x0 - for i in range(1, len(t)): - x[i] = U.dot(diag(expD ** (t[i] - t0))).dot(V).dot(y0) - x_ss - self.x = x - self.t = t - return x - - -class moments_simple: - def __init__( - self, - a=None, - b=None, - la=None, - alpha_a=None, - alpha_i=None, - sigma=None, - beta=None, - gamma=None, - ): - # species - self._u = 0 - self._w = 1 - self._x = 2 - self._y = 3 - - self.n_species = 4 - - # solution - self.t = None - self.x = None - self.x0 = zeros(self.n_species) - self.K = None - self.p = None - - # parameters - if not ( - a is None - or b is None - or la is None - or alpha_a is None - or alpha_i is None - or sigma is None - or beta is None - or gamma is None - ): - self.set_params(a, b, la, alpha_a, alpha_i, sigma, beta, gamma) - - def set_initial_condition(self, nu0, nw0, nx0, ny0): - x = zeros(self.n_species) - x[self._u] = nu0 - x[self._w] = nw0 - x[self._x] = nx0 - x[self._y] = ny0 - - self.x0 = x - return x - - def get_x_velocity(self, nu0, nx0): - return self.be * (1 - self.si) * nu0 - self.ga * nx0 - - def get_y_velocity(self, nu0, nw0, ny0): - return self.be * self.si * nu0 + self.be * nw0 - self.ga * ny0 - - def fbar(self, x_a, x_i): - return self.b / (self.a + self.b) * x_a + self.a / (self.a + self.b) * x_i - - def set_params(self, a, b, la, alpha_a, alpha_i, sigma, beta, gamma): - self.a = a - self.b = b - self.la = la - self.aa = alpha_a - self.ai = alpha_i - self.si = sigma - self.be = beta - self.ga = gamma - - # reset solutions - self.t = None - self.x = None - self.K = None - self.p = None - - def get_total(self): - return sum(self.x, 1) - - def computeKnp(self): - # parameters - la = self.la - aa = self.aa - ai = self.ai - si = self.si - be = self.be - ga = self.ga - - K = zeros((self.n_species, self.n_species)) - - # Diagonal - K[self._u, self._u] = -be - K[self._w, self._w] = -be - K[self._x, self._x] = -ga - K[self._y, self._y] = -ga - - # off-diagonal - K[self._x, self._u] = be * (1 - si) - K[self._y, self._u] = si * be - K[self._y, self._w] = be - - p = zeros(self.n_species) - p[self._u] = la * self.fbar(aa, ai) - p[self._w] = (1 - la) * self.fbar(aa, ai) - - return K, p - - def solve(self, t, x0=None): - t0 = t[0] - if x0 is None: - x0 = self.x0 - else: - self.x0 = x0 - - if self.K is None or self.p is None: - K, p = self.computeKnp() - self.K = K - self.p = p - else: - K = self.K - p = self.p - x_ss = linalg.solve(K, p) - y0 = x0 + x_ss - - D, U = linalg.eig(K) - V = linalg.inv(U) - D, U, V = map(real, (D, U, V)) - expD = exp(D) - x = zeros((len(t), self.n_species)) - x[0] = x0 - for i in range(1, len(t)): - x[i] = U.dot(diag(expD ** (t[i] - t0))).dot(V).dot(y0) - x_ss - self.x = x - self.t = t - return x - - -class estimation: - def __init__(self, ranges, x0=None): - self.ranges = ranges - self.n_params = len(ranges) - self.simulator = moments() - if not x0 is None: - self.simulator.x0 = x0 - - def sample_p0(self, samples=1, method="lhs"): - ret = zeros((samples, self.n_params)) - if method == "lhs": - ret = self._lhsclassic(samples) - for i in range(self.n_params): - ret[:, i] = ret[:, i] * (self.ranges[i][1] - self.ranges[i][0]) + self.ranges[i][0] - else: - for n in range(samples): - for i in range(self.n_params): - r = random.rand() - ret[n, i] = r * (self.ranges[i][1] - self.ranges[i][0]) + self.ranges[i][0] - return ret - - def _lhsclassic(self, samples): - # From PyDOE - # Generate the intervals - cut = linspace(0, 1, samples + 1) - - # Fill points uniformly in each interval - u = random.rand(samples, self.n_params) - a = cut[:samples] - b = cut[1 : samples + 1] - rdpoints = zeros_like(u) - for j in range(self.n_params): - rdpoints[:, j] = u[:, j] * (b - a) + a - - # Make the random pairings - H = zeros_like(rdpoints) - for j in range(self.n_params): - order = random.permutation(range(samples)) - H[:, j] = rdpoints[order, j] - - return H - - def get_bound(self, index): - ret = zeros(self.n_params) - for i in range(self.n_params): - ret[i] = self.ranges[i][index] - return ret - - def normalize_data(self, X): - # ret = zeros(X.shape) - # for i in range(len(X)): - # x = X[i] - # #ret[i] = x / max(x) - # ret[i] = log10(x + 1) - return log10(X + 1) - - def f_curve_fit(self, t, *params): - self.simulator.set_params(*params) - self.simulator.integrate(t, self.simulator.x0) - ret = self.simulator.get_all_central_moments() - ret = self.normalize_data(ret) - return ret.flatten() - - def f_lsq(self, params, t, x_data_norm, method="analytical", experiment_type=None): - self.simulator.set_params(*params) - if method == "numerical": - self.simulator.integrate(t, self.simulator.x0) - elif method == "analytical": - self.simulator.solve(t, self.simulator.x0) - if experiment_type is None: - ret = self.simulator.get_all_central_moments() - elif experiment_type == "nosplice": - ret = self.simulator.get_nosplice_central_moments() - elif experiment_type == "label": - ret = self.simulator.get_central_moments(["ul", "sl"]) - ret = self.normalize_data(ret).flatten() - ret[isnan(ret)] = 0 - return ret - x_data_norm - - def fit(self, t, x_data, p0=None, bounds=None): - if p0 is None: - p0 = self.sample_p0() - x_data_norm = self.normalize_data(x_data) - if bounds is None: - bounds = (self.get_bound(0), self.get_bound(1)) - popt, pcov = curve_fit(self.f_curve_fit, t, x_data_norm.flatten(), p0=p0, bounds=bounds) - return popt, pcov - - def fit_lsq( - self, - t, - x_data, - p0=None, - n_p0=1, - bounds=None, - sample_method="lhs", - method="analytical", - experiment_type=None, - ): - if p0 is None: - p0 = self.sample_p0(n_p0, sample_method) - else: - if p0.ndim == 1: - p0 = [p0] - n_p0 = len(p0) - x_data_norm = self.normalize_data(x_data) - if bounds is None: - bounds = (self.get_bound(0), self.get_bound(1)) - - costs = zeros(n_p0) - X = [] - for i in range(n_p0): - ret = least_squares( - lambda p: self.f_lsq(p, t, x_data_norm.flatten(), method, experiment_type), - p0[i], - bounds=bounds, - ) - costs[i] = ret.cost - X.append(ret.x) - i_min = argmin(costs) - return X[i_min], costs[i_min] From f7210211f36e82332d776e25818437961e7b7262 Mon Sep 17 00:00:00 2001 From: sichao Date: Thu, 16 Nov 2023 18:58:03 -0500 Subject: [PATCH 03/14] reorganize utils_markers --- dynamo/external/utils.py | 2 +- dynamo/tools/markers.py | 43 +++++++++++++++++++++++-- dynamo/tools/utils.py | 22 ++++++++++++- dynamo/tools/utils_markers.py | 59 ----------------------------------- 4 files changed, 63 insertions(+), 63 deletions(-) delete mode 100644 dynamo/tools/utils_markers.py diff --git a/dynamo/external/utils.py b/dynamo/external/utils.py index 126640fae..cfb99be89 100644 --- a/dynamo/external/utils.py +++ b/dynamo/external/utils.py @@ -3,7 +3,7 @@ import scipy.stats as stats from scipy.sparse import issparse -from ..tools.utils_markers import fdr +from ..tools.utils import fdr def normalize_data(mm, szfactors, pseudo_expr: float = 0.1): diff --git a/dynamo/tools/markers.py b/dynamo/tools/markers.py index 24852abfc..b7063e248 100755 --- a/dynamo/tools/markers.py +++ b/dynamo/tools/markers.py @@ -31,8 +31,7 @@ ) from ..preprocessing.transform import _Freeman_Tukey from ..tools.connectivity import _gen_neighbor_keys, check_and_recompute_neighbors -from .utils import fetch_X_data -from .utils_markers import fdr, specificity +from .utils import fdr, fetch_X_data def moran_i( @@ -806,3 +805,43 @@ def lrt(full: GLMResultsWrapper, restr: GLMResultsWrapper) -> np.float64: lr_pvalue = stats.chi2.sf(lrstat, df=lrdf) return lr_pvalue + + +def specificity(percentage: np.ndarray, perfect_specificity: np.ndarray) -> float: + """Calculate specificity""" + + spec = 1 - JSdistVec(makeprobsvec(percentage), perfect_specificity) + + return spec + + +def makeprobsvec(p: np.ndarray) -> np.ndarray: + """Calculate the probability matrix for a relative abundance matrix""" + + phat = p / np.sum(p) + phat[np.isnan((phat))] = 0 + + return phat + + +def shannon_entropy(p: np.ndarray) -> float: + """Calculate the Shannon entropy based on the probability vector""" + + if np.min(p) < 0 or np.sum(p) <= 0: + return np.inf + p_norm = p[p > 0] / np.sum(p) + + return -np.sum(np.log(p_norm) * p_norm) + + +def JSdistVec(p: np.ndarray, q: np.ndarray) -> float: + """Calculate the Jessen-Shannon distance for two probability distribution""" + + Jsdiv = shannon_entropy((p + q) / 2) - (shannon_entropy(p) + shannon_entropy(q)) / 2 + if np.isinf(Jsdiv): + Jsdiv = 1 + if Jsdiv < 0: + Jsdiv = 0 + JSdist = np.sqrt(Jsdiv) + + return JSdist diff --git a/dynamo/tools/utils.py b/dynamo/tools/utils.py index d5871352e..16788e114 100755 --- a/dynamo/tools/utils.py +++ b/dynamo/tools/utils.py @@ -3336,4 +3336,24 @@ def density_corrected_transition_matrix(T: Union[npt.ArrayLike, sp.csr_matrix]) T_i -= T_i.mean() T[i, idx] = T_i - return T \ No newline at end of file + return T + + +# --------------------------------------------------------------------------------------------------- +# differential gene expression test related +def fdr(p_vals: np.ndarray) -> np.ndarray: + """Calculate False Discovery Rate using Benjamini–Hochberg (non-negative) method. + + Args: + p_vals: The p-values describes the likelihood of an observation based on a probability distribution. + + Returns: + The corrected False Discovery Rate. + """ + from scipy.stats import rankdata + + ranked_p_values = rankdata(p_vals) + fdr = p_vals * len(p_vals) / ranked_p_values + fdr[fdr > 1] = 1 + + return fdr diff --git a/dynamo/tools/utils_markers.py b/dynamo/tools/utils_markers.py deleted file mode 100644 index a964ad88d..000000000 --- a/dynamo/tools/utils_markers.py +++ /dev/null @@ -1,59 +0,0 @@ -import numpy as np - -# --------------------------------------------------------------------------------------------------- -# specificity related - - -def specificity(percentage: np.ndarray, perfect_specificity: np.ndarray) -> float: - """Calculate specificity""" - - spec = 1 - JSdistVec(makeprobsvec(percentage), perfect_specificity) - - return spec - - -def makeprobsvec(p: np.ndarray) -> np.ndarray: - """Calculate the probability matrix for a relative abundance matrix""" - - phat = p / np.sum(p) - phat[np.isnan((phat))] = 0 - - return phat - - -def shannon_entropy(p: np.ndarray) -> float: - """Calculate the Shannon entropy based on the probability vector""" - - if np.min(p) < 0 or np.sum(p) <= 0: - return np.inf - p_norm = p[p > 0] / np.sum(p) - - return -np.sum(np.log(p_norm) * p_norm) - - -def JSdistVec(p: np.ndarray, q: np.ndarray) -> float: - """Calculate the Jessen-Shannon distance for two probability distribution""" - - Jsdiv = shannon_entropy((p + q) / 2) - (shannon_entropy(p) + shannon_entropy(q)) / 2 - if np.isinf(Jsdiv): - Jsdiv = 1 - if Jsdiv < 0: - Jsdiv = 0 - JSdist = np.sqrt(Jsdiv) - - return JSdist - - -# --------------------------------------------------------------------------------------------------- -# differential gene expression test related - - -def fdr(p_vals: np.ndarray) -> np.ndarray: - """Calculate fdr_bh (Benjamini/Hochberg (non-negative))""" - from scipy.stats import rankdata - - ranked_p_values = rankdata(p_vals) - fdr = p_vals * len(p_vals) / ranked_p_values - fdr[fdr > 1] = 1 - - return fdr From 6a49474066be341e866f212d30007d23b7f5d5a7 Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 12:29:18 -0500 Subject: [PATCH 04/14] deprecated DDRTree based legacy functions --- dynamo/tools/__init__.py | 7 +- dynamo/tools/construct_velocity_tree.py | 229 ----------------- dynamo/tools/deprecated.py | 321 +++++++++++++++++++++++- dynamo/tools/pseudotime.py | 60 ----- 4 files changed, 323 insertions(+), 294 deletions(-) diff --git a/dynamo/tools/__init__.py b/dynamo/tools/__init__.py index cb5f8b781..14125b101 100755 --- a/dynamo/tools/__init__.py +++ b/dynamo/tools/__init__.py @@ -35,7 +35,7 @@ ) # Pseudotime related -from .construct_velocity_tree import construct_velocity_tree, construct_velocity_tree_py +from .construct_velocity_tree import construct_velocity_tree from .DDRTree_py import DDRTree, cal_ncenter from .pseudotime import order_cells from .time_series import directed_pg @@ -112,3 +112,8 @@ scv_dyn_convertor, vlm_to_adata, ) + +# deprecated functions +from .deprecated import ( + construct_velocity_tree_py, +) diff --git a/dynamo/tools/construct_velocity_tree.py b/dynamo/tools/construct_velocity_tree.py index 1f185dfad..fbe6b01a5 100755 --- a/dynamo/tools/construct_velocity_tree.py +++ b/dynamo/tools/construct_velocity_tree.py @@ -1,77 +1,13 @@ -import re from typing import Dict, List, Optional, Tuple, Union import matplotlib.pyplot as plt import numpy as np -import scipy from anndata import AnnData from scipy.sparse import issparse, csr_matrix -from scipy.sparse.csgraph import shortest_path - -from .DDRTree_py import DDRTree from ..dynamo_logger import main_info, main_info_insert_adata_uns -def remove_velocity_points(G: np.ndarray, n: int) -> np.ndarray: - """Modify a tree graph to remove the nodes themselves and recalculate the weights. - - Args: - G: A smooth tree graph embedded in the low dimension space. - n: The number of genes (column num of the original data) - - Returns: - The tree graph with a node itself removed and weight recalculated. - """ - for nodeid in range(n, 2 * n): - nb_ids = [] - for nb_id in range(len(G[0])): - if G[nodeid][nb_id] != 0: - nb_ids = nb_ids + [nb_id] - num_nbs = len(nb_ids) - - if num_nbs == 1: - G[nodeid][nb_ids[0]] = 0 - G[nb_ids[0]][nodeid] = 0 - else: - min_val = np.inf - for i in range(len(G[0])): - if G[nodeid][i] != 0: - if G[nodeid][i] < min_val: - min_val = G[nodeid][i] - min_ind = i - for i in nb_ids: - if i != min_ind: - new_weight = G[nodeid][i] + min_val - G[i][min_ind] = new_weight - G[min_ind][i] = new_weight - # print('Add ege %s, %s\n',G.Nodes.Name {nb_ids(i)}, G.Nodes.Name {nb_ids(min_ind)}); - G[nodeid][nb_ids[0]] = 0 - G[nb_ids[0]][nodeid] = 0 - - return G - - -def calculate_angle(o: np.ndarray, y: np.ndarray, x: np.ndarray) -> float: - """Calculate the angle between two vectors. - - Args: - o: Coordination of the origin. - y: End point of the first vector. - x: End point of the second vector. - - Returns: - The angle between the two vectors. - """ - - yo = y - o - norm_yo = yo / scipy.linalg.norm(yo) - xo = x - o - norm_xo = xo / scipy.linalg.norm(xo) - angle = np.arccos(norm_yo.T * norm_xo) - return angle - - def _compute_center_transition_matrix(transition_matrix: Union[csr_matrix, np.ndarray], R: np.ndarray) -> np.ndarray: """Calculate the transition matrix for DDRTree centers. @@ -258,168 +194,3 @@ def construct_velocity_tree(adata: AnnData, transition_matrix_key: str = "pearso adata.uns["directed_velocity_tree"] = directed_velocity_tree main_info_insert_adata_uns("directed_velocity_tree") return directed_velocity_tree - - -def construct_velocity_tree_py(X1: np.ndarray, X2: np.ndarray) -> None: - """Save a velocity tree graph with given data. - - Args: - X1: Expression matrix. - X2: Velocity matrix. - """ - if issparse(X1): - X1 = X1.toarray() - if issparse(X2): - X2 = X2.toarray() - n = X1.shape[1] - - # merge two data with a given time - t = 0.5 - X_all = np.hstack((X1, X1 + t * X2)) - - # parameter settings - maxIter = 20 - eps = 1e-3 - sigma = 0.001 - gamma = 10 - - # run DDRTree algorithm - Z, Y, stree, R, W, Q, C, objs = DDRTree(X_all, maxIter=maxIter, eps=eps, sigma=sigma, gamma=gamma) - - # draw velocity figure - - # quiver(Z(1, 1: 100), Z(2, 1: 100), Z(1, 101: 200)-Z(1, 1: 100), Z(2, 101: 200)-Z(2, 1: 100)); - # plot(Z(1, 1: 100), Z(2, 1: 100), 'ob'); - # plot(Z(1, 101: 200), Z(2, 101: 200), 'sr'); - G = stree - - sG = remove_velocity_points(G, n) - tree = sG - row = [] - col = [] - val = [] - for i in range(sG.shape[0]): - for j in range(sG.shape[1]): - if sG[i][j] != 0: - row = row + [i] - col = col + [j] - val = val + [sG[1][j]] - tree_fname = "tree.csv" - # write sG data to tree.csv - ####### - branch_fname = "branch.txt" - cmd = "python extract_branches.py" + tree_fname + branch_fname - - branch_cell = [] - fid = open(branch_fname, "r") - tline = next(fid) - while isinstance(tline, str): - path = re.regexp(tline, "\d*", "Match") ############ - branch_cell = branch_cell + [path] ################# - tline = next(fid) - fid.close() - - dG = np.zeros((n, n)) - for p in range(len(branch_cell)): - path = branch_cell[p] - pos_direct = 0 - for bp in range(len(path)): - u = path(bp) - v = u + n - - # find the shorest path on graph G(works for trees) - nodeid = u - ve_nodeid = v - shortest_mat = shortest_path( - csgraph=G, - directed=False, - indices=nodeid, - return_predecessors=True, - ) - velocity_path = [] - while ve_nodeid != nodeid: - velocity_path = [shortest_mat[nodeid][ve_nodeid]] + velocity_path - ve_nodeid = shortest_mat[nodeid][ve_nodeid] - velocity_path = [shortest_mat[nodeid][ve_nodeid]] + velocity_path - ###v_path = G.Nodes.Name(velocity_path) - - # check direction consistency between path and v_path - valid_idx = [] - for i in velocity_path: - if i <= n: - valid_idx = valid_idx + [i] - if len(valid_idx) == 1: - # compute direction matching - if bp < len(path): - tree_next_point = Z[:, path(bp)] - v_point = Z[:, v] - u_point = Z[:, u] - angle = calculate_angle(u_point, tree_next_point, v_point) - angle = angle / 3.14 * 180 - if angle < 90: - pos_direct = pos_direct + 1 - - else: - tree_pre_point = Z[:, path(bp - 1)] - v_point = Z[:, v] - u_point = Z[:, u] - angle = calculate_angle(u_point, tree_pre_point, v_point) - angle = angle / 3.14 * 180 - if angle > 90: - pos_direct = pos_direct + 1 - - else: - - if bp < len(path): - if path[bp + 1] == valid_idx[2]: - pos_direct = pos_direct + 1 - - else: - if path[bp - 1] != valid_idx[2]: - pos_direct = pos_direct + 1 - - neg_direct = len(path) - pos_direct - print( - "branch=" - + str(p) - + ", (" - + path[0] - + "->" - + path[-1] - + "), pos=" - + pos_direct - + ", neg=" - + neg_direct - + "\n" - ) - print(path) - print("\n") - - if pos_direct > neg_direct: - for bp in range(len(path) - 1): - dG[path[bp], path[bp + 1]] = 1 - - else: - for bp in range(len(path) - 1): - dG[path(bp + 1), path(bp)] = 1 - - # figure; - # plot(digraph(dG)); - # title('directed graph') figure; hold on; - row = [] - col = [] - for i in range(dG.shape[0]): - for j in range(dG.shape[1]): - if dG[i][j] != 0: - row = row + [i] - col = col + [j] - for tn in range(len(row)): - p1 = Y[:, row[tn]] - p2 = Y[:, col[tn]] - dp = p2 - p1 - h = plt.quiver(p1(1), p1(2), dp(1), dp(2), "LineWidth", 5) ###############need to plot it - set(h, "MaxHeadSize", 1e3, "AutoScaleFactor", 1) ############# - - for i in range(n): - plt.text(Y(1, i), Y(2, i), str(i)) ############## - plt.savefig("./results/t01_figure3.fig") ################## diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py index 51e399817..aae697957 100644 --- a/dynamo/tools/deprecated.py +++ b/dynamo/tools/deprecated.py @@ -1,3 +1,4 @@ +import re import warnings from typing import Callable, Iterable, List, Optional, Tuple, Union @@ -8,12 +9,17 @@ import functools +import matplotlib.pyplot as plt import numpy as np +import scipy from numpy import * from scipy.integrate import odeint from scipy.optimize import curve_fit, least_squares +from scipy.sparse import issparse +from scipy.sparse.csgraph import shortest_path from ..dynamo_logger import main_info, main_warning +from .DDRTree_py import DDRTree from .moments import moment_model from .utils import ( get_data_for_kin_params_estimation, @@ -24,8 +30,8 @@ set_param_ss, set_velocity, ) -from .utils_moments import moments -from .velocity import ss_estimation, velocity +from ..estimation.tsc.utils_moments import moments +from ..estimation.csc.velocity import ss_estimation, Velocity def deprecated(func): @@ -241,7 +247,7 @@ def _dynamics_legacy( log_unnormalized, NTR_vel, ) - vel = velocity(estimation=est) + vel = Velocity(estimation=est) vel_U = vel.vel_u(U) vel_S = vel.vel_s(U, S) vel_P = vel.vel_p(S, P) @@ -294,7 +300,7 @@ def fbar(x_a, x_i, a, b): alpha = fbar(alpha_a, alpha_i, a, b)[:, None] params = {"alpha": alpha, "beta": beta, "gamma": gamma, "t": t} - vel = velocity(**params) + vel = Velocity(**params) U, S = get_U_S_for_velocity_estimation( subset_adata, @@ -1286,3 +1292,310 @@ def fit_lsq( X.append(ret.x) i_min = argmin(costs) return X[i_min], costs[i_min] + + +# --------------------------------------------------------------------------------------------------- +# deprecated construct_velocity_tree.py +@deprecated +def remove_velocity_points(*args, **kwargs): + return _remove_velocity_points_legacy(*args, **kwargs) + + +def _remove_velocity_points_legacy(G: np.ndarray, n: int) -> np.ndarray: + """Modify a tree graph to remove the nodes themselves and recalculate the weights. + + Args: + G: A smooth tree graph embedded in the low dimension space. + n: The number of genes (column num of the original data) + + Returns: + The tree graph with a node itself removed and weight recalculated. + """ + for nodeid in range(n, 2 * n): + nb_ids = [] + for nb_id in range(len(G[0])): + if G[nodeid][nb_id] != 0: + nb_ids = nb_ids + [nb_id] + num_nbs = len(nb_ids) + + if num_nbs == 1: + G[nodeid][nb_ids[0]] = 0 + G[nb_ids[0]][nodeid] = 0 + else: + min_val = np.inf + for i in range(len(G[0])): + if G[nodeid][i] != 0: + if G[nodeid][i] < min_val: + min_val = G[nodeid][i] + min_ind = i + for i in nb_ids: + if i != min_ind: + new_weight = G[nodeid][i] + min_val + G[i][min_ind] = new_weight + G[min_ind][i] = new_weight + # print('Add ege %s, %s\n',G.Nodes.Name {nb_ids(i)}, G.Nodes.Name {nb_ids(min_ind)}); + G[nodeid][nb_ids[0]] = 0 + G[nb_ids[0]][nodeid] = 0 + + return G + + +@deprecated +def calculate_angle(*args, **kwargs): + return _calculate_angle_legacy(*args, **kwargs) + + +def _calculate_angle_legacy(o: np.ndarray, y: np.ndarray, x: np.ndarray) -> float: + """Calculate the angle between two vectors. + + Args: + o: Coordination of the origin. + y: End point of the first vector. + x: End point of the second vector. + + Returns: + The angle between the two vectors. + """ + + yo = y - o + norm_yo = yo / scipy.linalg.norm(yo) + xo = x - o + norm_xo = xo / scipy.linalg.norm(xo) + angle = np.arccos(norm_yo.T * norm_xo) + return angle + + +@deprecated +def construct_velocity_tree_py(*args, **kwargs): + return _construct_velocity_tree_py_legacy(*args, **kwargs) + + +def _construct_velocity_tree_py_legacy(X1: np.ndarray, X2: np.ndarray) -> None: + """Save a velocity tree graph with given data. + + Args: + X1: Expression matrix. + X2: Velocity matrix. + """ + if issparse(X1): + X1 = X1.toarray() + if issparse(X2): + X2 = X2.toarray() + n = X1.shape[1] + + # merge two data with a given time + t = 0.5 + X_all = np.hstack((X1, X1 + t * X2)) + + # parameter settings + maxIter = 20 + eps = 1e-3 + sigma = 0.001 + gamma = 10 + + # run DDRTree algorithm + Z, Y, stree, R, W, Q, C, objs = DDRTree(X_all, maxIter=maxIter, eps=eps, sigma=sigma, gamma=gamma) + + # draw velocity figure + + # quiver(Z(1, 1: 100), Z(2, 1: 100), Z(1, 101: 200)-Z(1, 1: 100), Z(2, 101: 200)-Z(2, 1: 100)); + # plot(Z(1, 1: 100), Z(2, 1: 100), 'ob'); + # plot(Z(1, 101: 200), Z(2, 101: 200), 'sr'); + G = stree + + sG = _remove_velocity_points_legacy(G, n) + tree = sG + row = [] + col = [] + val = [] + for i in range(sG.shape[0]): + for j in range(sG.shape[1]): + if sG[i][j] != 0: + row = row + [i] + col = col + [j] + val = val + [sG[1][j]] + tree_fname = "tree.csv" + # write sG data to tree.csv + ####### + branch_fname = "branch.txt" + cmd = "python extract_branches.py" + tree_fname + branch_fname + + branch_cell = [] + fid = open(branch_fname, "r") + tline = next(fid) + while isinstance(tline, str): + path = re.regexp(tline, "\d*", "Match") ############ + branch_cell = branch_cell + [path] ################# + tline = next(fid) + fid.close() + + dG = np.zeros((n, n)) + for p in range(len(branch_cell)): + path = branch_cell[p] + pos_direct = 0 + for bp in range(len(path)): + u = path(bp) + v = u + n + + # find the shorest path on graph G(works for trees) + nodeid = u + ve_nodeid = v + shortest_mat = shortest_path( + csgraph=G, + directed=False, + indices=nodeid, + return_predecessors=True, + ) + velocity_path = [] + while ve_nodeid != nodeid: + velocity_path = [shortest_mat[nodeid][ve_nodeid]] + velocity_path + ve_nodeid = shortest_mat[nodeid][ve_nodeid] + velocity_path = [shortest_mat[nodeid][ve_nodeid]] + velocity_path + ###v_path = G.Nodes.Name(velocity_path) + + # check direction consistency between path and v_path + valid_idx = [] + for i in velocity_path: + if i <= n: + valid_idx = valid_idx + [i] + if len(valid_idx) == 1: + # compute direction matching + if bp < len(path): + tree_next_point = Z[:, path(bp)] + v_point = Z[:, v] + u_point = Z[:, u] + angle = _calculate_angle_legacy(u_point, tree_next_point, v_point) + angle = angle / 3.14 * 180 + if angle < 90: + pos_direct = pos_direct + 1 + + else: + tree_pre_point = Z[:, path(bp - 1)] + v_point = Z[:, v] + u_point = Z[:, u] + angle = _calculate_angle_legacy(u_point, tree_pre_point, v_point) + angle = angle / 3.14 * 180 + if angle > 90: + pos_direct = pos_direct + 1 + + else: + + if bp < len(path): + if path[bp + 1] == valid_idx[2]: + pos_direct = pos_direct + 1 + + else: + if path[bp - 1] != valid_idx[2]: + pos_direct = pos_direct + 1 + + neg_direct = len(path) - pos_direct + print( + "branch=" + + str(p) + + ", (" + + path[0] + + "->" + + path[-1] + + "), pos=" + + pos_direct + + ", neg=" + + neg_direct + + "\n" + ) + print(path) + print("\n") + + if pos_direct > neg_direct: + for bp in range(len(path) - 1): + dG[path[bp], path[bp + 1]] = 1 + + else: + for bp in range(len(path) - 1): + dG[path(bp + 1), path(bp)] = 1 + + # figure; + # plot(digraph(dG)); + # title('directed graph') figure; hold on; + row = [] + col = [] + for i in range(dG.shape[0]): + for j in range(dG.shape[1]): + if dG[i][j] != 0: + row = row + [i] + col = col + [j] + for tn in range(len(row)): + p1 = Y[:, row[tn]] + p2 = Y[:, col[tn]] + dp = p2 - p1 + h = plt.quiver(p1(1), p1(2), dp(1), dp(2), "LineWidth", 5) ###############need to plot it + set(h, "MaxHeadSize", 1e3, "AutoScaleFactor", 1) ############# + + for i in range(n): + plt.text(Y(1, i), Y(2, i), str(i)) ############## + plt.savefig("./results/t01_figure3.fig") ################## + + +# --------------------------------------------------------------------------------------------------- +# deprecated pseudotime.py +@deprecated +def compute_partition(*args, **kwargs): + return _compute_partition_legacy(*args, **kwargs) + + +def _compute_partition_legacy(adata, transition_matrix, cell_membership, principal_g, group=None): + """Compute a partition of cells based on a minimum spanning tree and cell membership. + + Args: + adata: The anndata object containing the single-cell data. + transition_matrix: The matrix representing the transition probabilities between cells. + cell_membership: The matrix representing the cell membership information. + principal_g: The principal graph information saved as array. + group: The name of a categorical group in `adata.obs`. If provided, it is used to construct the + `cell_membership` matrix based on the specified group membership. Defaults to None. + + Returns: + A partition of cells represented as a matrix. + """ + + from scipy.sparse import csr_matrix + from scipy.sparse.csgraph import minimum_spanning_tree + + # http://active-analytics.com/blog/rvspythonwhyrisstillthekingofstatisticalcomputing/ + if group is not None and group in adata.obs.columns: + from patsy import dmatrix # dmatrices, dmatrix, demo_data + + data = adata.obs + data.columns[data.columns == group] = "group_" + + cell_membership = csr_matrix(dmatrix("~group_+0", data=data)) + + X = csr_matrix(principal_g > 0) + Tcsr = minimum_spanning_tree(X) + principal_g = Tcsr.toarray().astype(int) + + membership_matrix = cell_membership.T.dot(transition_matrix).dot(cell_membership) + + direct_principal_g = principal_g * membership_matrix + + # get the data: + # edges_per_module < - Matrix::rowSums(num_links) + # total_edges < - sum(num_links) + # + # theta < - (as.matrix(edges_per_module) / total_edges) % * % + # Matrix::t(edges_per_module / total_edges) + # + # var_null_num_links < - theta * (1 - theta) / total_edges + # num_links_ij < - num_links / total_edges - theta + # cluster_mat < - pnorm_over_mat(as.matrix(num_links_ij), var_null_num_links) + # + # num_links < - num_links_ij / total_edges + # + # cluster_mat < - matrix(stats::p.adjust(cluster_mat), + # nrow = length(louvain_modules), + # ncol = length(louvain_modules)) + # + # sig_links < - as.matrix(num_links) + # sig_links[cluster_mat > qval_thresh] = 0 + # diag(sig_links) < - 0 + + return direct_principal_g diff --git a/dynamo/tools/pseudotime.py b/dynamo/tools/pseudotime.py index 82211b7f7..10e98e13a 100755 --- a/dynamo/tools/pseudotime.py +++ b/dynamo/tools/pseudotime.py @@ -421,63 +421,3 @@ def _cal_ncenter(ncells, ncells_limit=100): return None else: return np.round(2 * ncells_limit * np.log(ncells) / (np.log(ncells) + np.log(ncells_limit))) - - -# make this function to also calculate the directed graph between clusters: -def compute_partition(adata, transition_matrix, cell_membership, principal_g, group=None): - """Compute a partition of cells based on a minimum spanning tree and cell membership. - - Args: - adata: The anndata object containing the single-cell data. - transition_matrix: The matrix representing the transition probabilities between cells. - cell_membership: The matrix representing the cell membership information. - principal_g: The principal graph information saved as array. - group: The name of a categorical group in `adata.obs`. If provided, it is used to construct the - `cell_membership` matrix based on the specified group membership. Defaults to None. - - Returns: - A partition of cells represented as a matrix. - """ - - from scipy.sparse import csr_matrix - from scipy.sparse.csgraph import minimum_spanning_tree - - # http://active-analytics.com/blog/rvspythonwhyrisstillthekingofstatisticalcomputing/ - if group is not None and group in adata.obs.columns: - from patsy import dmatrix # dmatrices, dmatrix, demo_data - - data = adata.obs - data.columns[data.columns == group] = "group_" - - cell_membership = csr_matrix(dmatrix("~group_+0", data=data)) - - X = csr_matrix(principal_g > 0) - Tcsr = minimum_spanning_tree(X) - principal_g = Tcsr.toarray().astype(int) - - membership_matrix = cell_membership.T.dot(transition_matrix).dot(cell_membership) - - direct_principal_g = principal_g * membership_matrix - - # get the data: - # edges_per_module < - Matrix::rowSums(num_links) - # total_edges < - sum(num_links) - # - # theta < - (as.matrix(edges_per_module) / total_edges) % * % - # Matrix::t(edges_per_module / total_edges) - # - # var_null_num_links < - theta * (1 - theta) / total_edges - # num_links_ij < - num_links / total_edges - theta - # cluster_mat < - pnorm_over_mat(as.matrix(num_links_ij), var_null_num_links) - # - # num_links < - num_links_ij / total_edges - # - # cluster_mat < - matrix(stats::p.adjust(cluster_mat), - # nrow = length(louvain_modules), - # ncol = length(louvain_modules)) - # - # sig_links < - as.matrix(num_links) - # sig_links[cluster_mat > qval_thresh] = 0 - # diag(sig_links) < - 0 - - return direct_principal_g From 4fea0984ea060f10611f389caca8e913b5faa1ab Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 13:43:15 -0500 Subject: [PATCH 05/14] reorganize DDRtree based functions and psl --- .gitignore | 4 +- dynamo/tools/{DDRTree_py.py => DDRTree.py} | 0 dynamo/tools/__init__.py | 7 +- dynamo/tools/deprecated.py | 2 +- dynamo/tools/pseudotime.py | 2 +- ...t_velocity_tree.py => pseudotime_graph.py} | 77 +++++++++++++++ dynamo/tools/{psl_py.py => psl.py} | 19 +--- dynamo/tools/time_series.py | 96 ------------------- dynamo/tools/utils.py | 17 ++++ dynamo/tools/utils_reduceDimension.py | 2 +- 10 files changed, 103 insertions(+), 123 deletions(-) rename dynamo/tools/{DDRTree_py.py => DDRTree.py} (100%) rename dynamo/tools/{construct_velocity_tree.py => pseudotime_graph.py} (70%) rename dynamo/tools/{psl_py.py => psl.py} (94%) delete mode 100755 dynamo/tools/time_series.py diff --git a/.gitignore b/.gitignore index be7480dc5..a2dcf092c 100755 --- a/.gitignore +++ b/.gitignore @@ -67,5 +67,5 @@ public/*.js # Docker related: debug/numbers_in_dynamo_first_revision.py -dynamo/tools/DDRTree_py.py -dynamo/tools/psl_py.py +dynamo/tools/DDRTree.py +dynamo/tools/psl.py diff --git a/dynamo/tools/DDRTree_py.py b/dynamo/tools/DDRTree.py similarity index 100% rename from dynamo/tools/DDRTree_py.py rename to dynamo/tools/DDRTree.py diff --git a/dynamo/tools/__init__.py b/dynamo/tools/__init__.py index 14125b101..2c8a98ab0 100755 --- a/dynamo/tools/__init__.py +++ b/dynamo/tools/__init__.py @@ -35,10 +35,9 @@ ) # Pseudotime related -from .construct_velocity_tree import construct_velocity_tree -from .DDRTree_py import DDRTree, cal_ncenter +from .pseudotime_graph import construct_velocity_tree, directed_pg +from .DDRTree import DDRTree, cal_ncenter from .pseudotime import order_cells -from .time_series import directed_pg # dimension reduction related from .dimension_reduction import reduceDimension # , run_umap @@ -79,7 +78,7 @@ from .metric_velocity import cell_wise_confidence, gene_wise_confidence from .moments import calc_1nd_moment, calc_2nd_moment, moments from .pseudotime_velocity import pseudotime_velocity -from .psl_py import psl +from .psl import psl # recipes: from .recipes import ( diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py index aae697957..4b5e600ed 100644 --- a/dynamo/tools/deprecated.py +++ b/dynamo/tools/deprecated.py @@ -19,7 +19,7 @@ from scipy.sparse.csgraph import shortest_path from ..dynamo_logger import main_info, main_warning -from .DDRTree_py import DDRTree +from .DDRTree import DDRTree from .moments import moment_model from .utils import ( get_data_for_kin_params_estimation, diff --git a/dynamo/tools/pseudotime.py b/dynamo/tools/pseudotime.py index 10e98e13a..00f643990 100755 --- a/dynamo/tools/pseudotime.py +++ b/dynamo/tools/pseudotime.py @@ -6,7 +6,7 @@ from scipy.spatial import distance from scipy.sparse.csgraph import minimum_spanning_tree -from .DDRTree_py import DDRTree +from .DDRTree import DDRTree from .utils import log1p_ from ..dynamo_logger import main_info, main_info_insert_adata_obs diff --git a/dynamo/tools/construct_velocity_tree.py b/dynamo/tools/pseudotime_graph.py similarity index 70% rename from dynamo/tools/construct_velocity_tree.py rename to dynamo/tools/pseudotime_graph.py index fbe6b01a5..5a194db1d 100755 --- a/dynamo/tools/construct_velocity_tree.py +++ b/dynamo/tools/pseudotime_graph.py @@ -4,7 +4,9 @@ import numpy as np from anndata import AnnData from scipy.sparse import issparse, csr_matrix +from scipy.sparse.csgraph import minimum_spanning_tree +from .DDRTree import cal_ncenter, DDRTree from ..dynamo_logger import main_info, main_info_insert_adata_uns @@ -194,3 +196,78 @@ def construct_velocity_tree(adata: AnnData, transition_matrix_key: str = "pearso adata.uns["directed_velocity_tree"] = directed_velocity_tree main_info_insert_adata_uns("directed_velocity_tree") return directed_velocity_tree + + +def directed_pg( + adata: AnnData, + basis: str = "umap", + transition_method: str = "pearson", + maxIter: int = 10, + sigma: float = 0.001, + Lambda: Optional[float] = None, + gamma: float = 10, + ncenter: Optional[int] = None, + raw_embedding: bool = True, +) -> AnnData: + """A function that learns a direct principal graph by integrating the transition matrix between and DDRTree. + + Args: + adata: An AnnData object, + basis: The dimension reduction method utilized. Defaults to "umap". + transition_method: The method to calculate the transition matrix and project high dimensional vector to low + dimension. + maxIter: The max iteration numbers. Defaults to 10. + sigma: The bandwidth parameter. Defaults to 0.001. + Lambda: The regularization parameter for inverse graph embedding. Defaults to None. + gamma: The regularization parameter for k-means. Defaults to 10. + ncenter: The number of centers to be considered. If None, number of centers would be calculated automatically. + Defaults to None. + raw_embedding: Whether to project the nodes on the principal graph into the original embedding. Defaults to + True. + + Raises: + Exception: invalid `basis`. + Exception: adata.uns["transition_matrix"] not defined. + + Returns: + An updated AnnData object that is updated with principal_g_transition, X__DDRTree and X_DDRTree_pg keys. + """ + + X = adata.obsm["X_" + basis] if "X_" + basis in adata.obsm.keys() else None + if X is None: + raise Exception("{} is not a key of obsm ({} dimension reduction is not performed yet.).".format(basis, basis)) + + transition_matrix = ( + adata.obsp[transition_method + "_transition_matrix"] + if transition_method + "_transition_matrix" in adata.obsp.keys() + else None + ) + if transition_matrix is None: + raise Exception("transition_matrix is not a key of uns. Please first run cell_velocity.") + + Lambda = 5 * X.shape[1] if Lambda is None else Lambda + ncenter = 250 if cal_ncenter(X.shape[1]) is None else ncenter + + Z, Y, principal_g, cell_membership, W, Q, C, objs = DDRTree( + X, + maxIter=maxIter, + Lambda=Lambda, + sigma=sigma, + gamma=gamma, + ncenter=ncenter, + ) + + X = csr_matrix(principal_g) + Tcsr = minimum_spanning_tree(X) + principal_g = Tcsr.toarray().astype(int) + + # here we can also identify siginificant links using methods related to PAGA + transition_matrix = transition_matrix.toarray() + principal_g_transition = cell_membership.T.dot(transition_matrix).dot(cell_membership) * principal_g + + adata.uns["principal_g_transition"] = principal_g_transition + adata.obsm["X_DDRTree"] = X.T if raw_embedding else Z + cell_membership = csr_matrix(cell_membership) + adata.uns["X_DDRTree_pg"] = cell_membership.dot(X.T) if raw_embedding else Y + + return adata diff --git a/dynamo/tools/psl_py.py b/dynamo/tools/psl.py similarity index 94% rename from dynamo/tools/psl_py.py rename to dynamo/tools/psl.py index 738282353..28e4def5e 100755 --- a/dynamo/tools/psl_py.py +++ b/dynamo/tools/psl.py @@ -9,7 +9,7 @@ from scipy.sparse import csr_matrix from scipy.sparse.linalg import eigs -from .DDRTree_py import repmat +from .DDRTree import repmat # from scikits.sparse.cholmod import cholesky @@ -200,20 +200,3 @@ def psl( Z = np.dot(U, tmp) return (S, Z) - - -def logdet(A: np.ndarray) -> float: - """Calculate log(det(A)). - - Compared with calculating log(det(A)) directly, this function avoid the overflow/underflow problems that are likely - to happen when applying det to large matrices. - - Args: - A: An square matrix. - - Returns: - log(det(A)). - """ - - v = 2 * sum(np.log(np.diag(np.linalg.cholesky(A)))) - return v diff --git a/dynamo/tools/time_series.py b/dynamo/tools/time_series.py deleted file mode 100755 index 3d7df9808..000000000 --- a/dynamo/tools/time_series.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import Optional - -import anndata -import numpy as np -from scipy.sparse import csr_matrix -from scipy.sparse.csgraph import minimum_spanning_tree - -from .DDRTree_py import DDRTree - - -def cal_ncenter(ncells: int, ncells_limit: int = 100) -> int: - """Calculate number of centers to be considered. - - Args: - ncells: The number of cells. - ncells_limit: The limitation of number of cells to be considered. Defaults to 100. - - Returns: - The number of centers to be considered - """ - return np.round(2 * ncells_limit * np.log(ncells) / (np.log(ncells) + np.log(ncells_limit))) - - -def directed_pg( - adata: anndata.AnnData, - basis: str = "umap", - transition_method: str = "pearson", - maxIter: int = 10, - sigma: float = 0.001, - Lambda: Optional[float] = None, - gamma: float = 10, - ncenter: Optional[int] = None, - raw_embedding: bool = True, -) -> anndata.AnnData: - """A function that learns a direct principal graph by integrating the transition matrix between and DDRTree. - - Args: - adata: An AnnData object, - basis: The dimension reduction method utilized. Defaults to "umap". - transition_method: The method to calculate the transition matrix and project high dimensional vector to low - dimension. - maxIter: The max iteration numbers. Defaults to 10. - sigma: The bandwidth parameter. Defaults to 0.001. - Lambda: The regularization parameter for inverse graph embedding. Defaults to None. - gamma: The regularization parameter for k-means. Defaults to 10. - ncenter: The number of centers to be considered. If None, number of centers would be calculated automatically. - Defaults to None. - raw_embedding: Whether to project the nodes on the principal graph into the original embedding. Defaults to - True. - - Raises: - Exception: invalid `basis`. - Exception: adata.uns["transition_matrix"] not defined. - - Returns: - An updated AnnData object that is updated with principal_g_transition, X__DDRTree and X_DDRTree_pg keys. - """ - - X = adata.obsm["X_" + basis] if "X_" + basis in adata.obsm.keys() else None - if X is None: - raise Exception("{} is not a key of obsm ({} dimension reduction is not performed yet.).".format(basis, basis)) - - transition_matrix = ( - adata.obsp[transition_method + "_transition_matrix"] - if transition_method + "_transition_matrix" in adata.obsp.keys() - else None - ) - if transition_matrix is None: - raise Exception("transition_matrix is not a key of uns. Please first run cell_velocity.") - - Lambda = 5 * X.shape[1] if Lambda is None else Lambda - ncenter = 250 if cal_ncenter(X.shape[1]) is None else ncenter - - Z, Y, principal_g, cell_membership, W, Q, C, objs = DDRTree( - X, - maxIter=maxIter, - Lambda=Lambda, - sigma=sigma, - gamma=gamma, - ncenter=ncenter, - ) - - X = csr_matrix(principal_g) - Tcsr = minimum_spanning_tree(X) - principal_g = Tcsr.toarray().astype(int) - - # here we can also identify siginificant links using methods related to PAGA - transition_matrix = transition_matrix.toarray() - principal_g_transition = cell_membership.T.dot(transition_matrix).dot(cell_membership) * principal_g - - adata.uns["principal_g_transition"] = principal_g_transition - adata.obsm["X_DDRTree"] = X.T if raw_embedding else Z - cell_membership = csr_matrix(cell_membership) - adata.uns["X_DDRTree_pg"] = cell_membership.dot(X.T) if raw_embedding else Y - - return adata diff --git a/dynamo/tools/utils.py b/dynamo/tools/utils.py index 16788e114..bb796d13b 100755 --- a/dynamo/tools/utils.py +++ b/dynamo/tools/utils.py @@ -496,6 +496,23 @@ def elem_prod( return np.multiply(X, Y) +def logdet(A: np.ndarray) -> float: + """Calculate log(det(A)). + + Compared with calculating log(det(A)) directly, this function avoid the overflow/underflow problems that are likely + to happen when applying det to large matrices. + + Args: + A: An square matrix. + + Returns: + log(det(A)). + """ + + v = 2 * sum(np.log(np.diag(np.linalg.cholesky(A)))) + return v + + def norm(x: Union[sp.csr_matrix, np.ndarray], **kwargs) -> np.ndarray: """Calculate the norm of an array or matrix diff --git a/dynamo/tools/utils_reduceDimension.py b/dynamo/tools/utils_reduceDimension.py index c971abf7e..66326b9fc 100644 --- a/dynamo/tools/utils_reduceDimension.py +++ b/dynamo/tools/utils_reduceDimension.py @@ -18,7 +18,7 @@ knn_to_adj, umap_conn_indices_dist_embedding, ) -from .psl_py import psl +from .psl import psl from .utils import log1p_, update_dict From 6fac27e170117a943bab42a1b115bed7b57fdcaa Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 14:29:24 -0500 Subject: [PATCH 06/14] fix format issue and add file docstrings --- dynamo/tools/DDRTree.py | 20 ++++++++++---------- dynamo/tools/graph_calculus.py | 1 + dynamo/tools/graph_operators.py | 9 ++++++--- dynamo/tools/utils.py | 2 +- dynamo/tools/velocyto_scvelo.py | 8 +++++++- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/dynamo/tools/DDRTree.py b/dynamo/tools/DDRTree.py index 35cabd014..3f97cdefb 100755 --- a/dynamo/tools/DDRTree.py +++ b/dynamo/tools/DDRTree.py @@ -10,7 +10,7 @@ from scipy.sparse.linalg import inv -def cal_ncenter(ncells: int, ncells_limit: int=100) -> int: +def cal_ncenter(ncells: int, ncells_limit: int = 100) -> int: """Calculate the number of cells to be most significant in the reduced space. Args: @@ -105,15 +105,15 @@ def eye(m: int, n: int) -> np.ndarray: def DDRTree( - X: np.ndarray, - maxIter: int, - sigma: float, - gamma: float, - eps: int=0, - dim: int=2, - Lambda: float=1.0, - ncenter: Optional[int]=None, - keep_history: bool=False + X: np.ndarray, + maxIter: int, + sigma: float, + gamma: float, + eps: int = 0, + dim: int = 2, + Lambda: float = 1.0, + ncenter: Optional[int] = None, + keep_history: bool = False, ) -> Union[ pd.DataFrame, Tuple[ diff --git a/dynamo/tools/graph_calculus.py b/dynamo/tools/graph_calculus.py index b43980ace..afa72542f 100644 --- a/dynamo/tools/graph_calculus.py +++ b/dynamo/tools/graph_calculus.py @@ -1,3 +1,4 @@ +"""This file implements the graph calculus functions using matrix as input.""" from typing import Callable, List, Optional, Tuple, Union try: diff --git a/dynamo/tools/graph_operators.py b/dynamo/tools/graph_operators.py index f7353685e..77b25ea5d 100644 --- a/dynamo/tools/graph_operators.py +++ b/dynamo/tools/graph_operators.py @@ -1,7 +1,10 @@ -# YEAR: 2019 -# COPYRIGHT HOLDER: ddhodge +""" This file implements graph operators using Graph object from iGraph as input. -# Code adapted from https://github.com/kazumits/ddhodge. +YEAR: 2019 +COPYRIGHT HOLDER: ddhodge + +Code adapted from https://github.com/kazumits/ddhodge. +""" from typing import List, Optional, Union from itertools import combinations diff --git a/dynamo/tools/utils.py b/dynamo/tools/utils.py index bb796d13b..ef5fc7999 100755 --- a/dynamo/tools/utils.py +++ b/dynamo/tools/utils.py @@ -2469,7 +2469,7 @@ def set_transition_genes( use_for_dynamics: bool = True, store_key: str = "use_for_transition", minimal_gene_num: int = 50, -) -> None: +) -> AnnData: """Set the transition genes in the AnnData object. Args: diff --git a/dynamo/tools/velocyto_scvelo.py b/dynamo/tools/velocyto_scvelo.py index 134ab64f7..37e435097 100755 --- a/dynamo/tools/velocyto_scvelo.py +++ b/dynamo/tools/velocyto_scvelo.py @@ -1,4 +1,10 @@ -# functions to run velocyto and scvelo +"""This file provides useful functions to interact with velocyto and scvelo. + +Implemented functions includes: + Run velocyto and scvelo analysis. + Convert adata to loom object or vice versa. + Convert Dynamo AnnData object to scvelo AnnData object or vice versa. +""" # from .moments import * from typing import List, Optional From 9610b8e044884a90df707de317fc26f63ed4b780 Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 14:48:09 -0500 Subject: [PATCH 07/14] deprecate leagcy moments functions --- dynamo/tools/deprecated.py | 267 ++++++++++++++++++++++++++++++++++++- dynamo/tools/moments.py | 253 ----------------------------------- 2 files changed, 266 insertions(+), 254 deletions(-) diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py index 4b5e600ed..cebd38dee 100644 --- a/dynamo/tools/deprecated.py +++ b/dynamo/tools/deprecated.py @@ -12,15 +12,17 @@ import matplotlib.pyplot as plt import numpy as np import scipy +from anndata import AnnData from numpy import * from scipy.integrate import odeint from scipy.optimize import curve_fit, least_squares from scipy.sparse import issparse from scipy.sparse.csgraph import shortest_path +from tqdm import tqdm from ..dynamo_logger import main_info, main_warning from .DDRTree import DDRTree -from .moments import moment_model +from .moments import calc_1nd_moment, strat_mom from .utils import ( get_data_for_kin_params_estimation, get_mapper, @@ -1599,3 +1601,266 @@ def _compute_partition_legacy(adata, transition_matrix, cell_membership, princip # diag(sig_links) < - 0 return direct_principal_g + + +# --------------------------------------------------------------------------------------------------- +# deprecated moments.py +@deprecated +def _calc_1nd_moment(*args, **kwargs): + return _calc_1nd_moment_legacy(*args, **kwargs) + + +def _calc_1nd_moment_legacy(X, W, normalize_W=True): + """deprecated""" + if normalize_W: + d = np.sum(W, 1) + W = np.diag(1 / d) @ W + return W @ X + + +@deprecated +def _calc_2nd_moment(*args, **kwargs): + return _calc_2nd_moment_legacy(*args, **kwargs) + + +def _calc_2nd_moment_legacy(X, Y, W, normalize_W=True, center=False, mX=None, mY=None): + """deprecated""" + if normalize_W: + d = np.sum(W, 1) + W = np.diag(1 / d) @ W + XY = np.multiply(W @ Y, X) + if center: + mX = calc_1nd_moment(X, W, False) if mX is None else mX + mY = calc_1nd_moment(Y, W, False) if mY is None else mY + XY = XY - np.multiply(mX, mY) + return XY + + +# old moment estimation code +class MomData(AnnData): + """deprecated""" + + def __init__(self, adata, time_key="Time", has_nan=False): + # self.data = adata + self.__dict__ = adata.__dict__ + # calculate first and second moments from data + self.times = np.array(self.obs[time_key].values, dtype=float) + self.uniq_times = np.unique(self.times) + nT = self.get_n_times() + ng = self.get_n_genes() + self.M = np.zeros((ng, nT)) # first moments (data) + self.V = np.zeros((ng, nT)) # second moments (data) + for g in tqdm(range(ng), desc="calculating 1/2 moments"): + tmp = self[:, g].layers["new"] + L = ( + np.array(tmp.A, dtype=float) if issparse(tmp) else np.array(tmp, dtype=float) + ) # consider using the `adata.obs_vector`, `adata.var_vector` methods or accessing the array directly. + if has_nan: + self.M[g] = strat_mom(L, self.times, np.nanmean) + self.V[g] = strat_mom(L, self.times, np.nanvar) + else: + self.M[g] = strat_mom(L, self.times, np.mean) + self.V[g] = strat_mom(L, self.times, np.var) + + def get_n_genes(self): + return self.var.shape[0] + + def get_n_cell(self): + return self.obs.shape[0] + + def get_n_times(self): + return len(self.uniq_times) + + +class Estimation: + """deprecated""" + + def __init__( + self, + adata, + adata_u=None, + time_key="Time", + normalize=True, + param_ranges=None, + has_nan=False, + ): + # initialize Estimation + self.data = MomData(adata, time_key, has_nan) + self.data_u = MomData(adata_u, time_key, has_nan) if adata_u is not None else None + if param_ranges is None: + param_ranges = { + "a": [0, 10], + "b": [0, 10], + "alpha_a": [10, 1000], + "alpha_i": [0, 10], + "beta": [0, 10], + "gamma": [0, 10], + } + self.normalize = normalize + self.param_ranges = param_ranges + self.n_params = len(param_ranges) + + def param_array2dict(self, parr): + if parr.ndim == 1: + return { + "a": parr[0], + "b": parr[1], + "alpha_a": parr[2], + "alpha_i": parr[3], + "beta": parr[4], + "gamma": parr[5], + } + else: + return { + "a": parr[:, 0], + "b": parr[:, 1], + "alpha_a": parr[:, 2], + "alpha_i": parr[:, 3], + "beta": parr[:, 4], + "gamma": parr[:, 5], + } + + def fit_gene(self, gene_no, n_p0=10): + from ..estimation.tsc.utils_moments import estimation + + estm = estimation(list(self.param_ranges.values())) + if self.data_u is None: + m = self.data.M[gene_no, :].T + v = self.data.V[gene_no, :].T + x_data = np.vstack((m, v)) + popt, cost = estm.fit_lsq( + self.data.uniq_times, + x_data, + p0=None, + n_p0=n_p0, + normalize=self.normalize, + experiment_type="nosplice", + ) + else: + mu = self.data_u.M[gene_no, :].T + ms = self.data.M[gene_no, :].T + vu = self.data_u.V[gene_no, :].T + vs = self.data.V[gene_no, :].T + x_data = np.vstack((mu, ms, vu, vs)) + popt, cost = estm.fit_lsq( + self.data.uniq_times, + x_data, + p0=None, + n_p0=n_p0, + normalize=self.normalize, + experiment_type=None, + ) + return popt, cost + + def fit(self, n_p0=10): + ng = self.data.get_n_genes() + params = np.zeros((ng, self.n_params)) + costs = np.zeros(ng) + for i in tqdm(range(ng), desc="fitting genes"): + params[i], costs[i] = self.fit_gene(i, n_p0) + return params, costs + + +# use for kinetic assumption with full data, deprecated +def moment_model(adata, subset_adata, _group, cur_grp, log_unnormalized, tkey): + """deprecated""" + # a few hard code to set up data for moment mode: + if "uu" in subset_adata.layers.keys() or "X_uu" in subset_adata.layers.keys(): + if log_unnormalized and "X_uu" not in subset_adata.layers.keys(): + if issparse(subset_adata.layers["uu"]): + ( + subset_adata.layers["uu"].data, + subset_adata.layers["ul"].data, + subset_adata.layers["su"].data, + subset_adata.layers["sl"].data, + ) = ( + np.log1p(subset_adata.layers["uu"].data), + np.log1p(subset_adata.layers["ul"].data), + np.log1p(subset_adata.layers["su"].data), + np.log1p(subset_adata.layers["sl"].data), + ) + else: + ( + subset_adata.layers["uu"], + subset_adata.layers["ul"], + subset_adata.layers["su"], + subset_adata.layers["sl"], + ) = ( + np.log1p(subset_adata.layers["uu"]), + np.log1p(subset_adata.layers["ul"]), + np.log1p(subset_adata.layers["su"]), + np.log1p(subset_adata.layers["sl"]), + ) + + subset_adata_u, subset_adata_s = ( + subset_adata.copy(), + subset_adata.copy(), + ) + del ( + subset_adata_u.layers["su"], + subset_adata_u.layers["sl"], + subset_adata_s.layers["uu"], + subset_adata_s.layers["ul"], + ) + ( + subset_adata_u.layers["new"], + subset_adata_u.layers["old"], + subset_adata_s.layers["new"], + subset_adata_s.layers["old"], + ) = ( + subset_adata_u.layers.pop("ul"), + subset_adata_u.layers.pop("uu"), + subset_adata_s.layers.pop("sl"), + subset_adata_s.layers.pop("su"), + ) + Moment, Moment_ = MomData(subset_adata_s, tkey), MomData(subset_adata_u, tkey) + if cur_grp == _group[0]: + t_ind = 0 + g_len, t_len = len(_group), len(np.unique(adata.obs[tkey])) + (adata.uns["M_sl"], adata.uns["V_sl"], adata.uns["M_ul"], adata.uns["V_ul"]) = ( + np.zeros((Moment.M.shape[0], g_len * t_len)), + np.zeros((Moment.M.shape[0], g_len * t_len)), + np.zeros((Moment.M.shape[0], g_len * t_len)), + np.zeros((Moment.M.shape[0], g_len * t_len)), + ) + + inds = np.arange((t_len * t_ind), (t_len * (t_ind + 1))) + ( + adata.uns["M_sl"][:, inds], + adata.uns["V_sl"][:, inds], + adata.uns["M_ul"][:, inds], + adata.uns["V_ul"][:, inds], + ) = (Moment.M, Moment.V, Moment_.M, Moment_.V) + + del Moment_ + Est = Estimation(Moment, adata_u=subset_adata_u, time_key=tkey, normalize=True) # # data is already normalized + else: + if log_unnormalized and "X_total" not in subset_adata.layers.keys(): + if issparse(subset_adata.layers["total"]): + (subset_adata.layers["new"].data, subset_adata.layers["total"].data,) = ( + np.log1p(subset_adata.layers["new"].data), + np.log1p(subset_adata.layers["total"].data), + ) + else: + subset_adata.layers["total"], subset_adata.layers["total"] = ( + np.log1p(subset_adata.layers["new"]), + np.log1p(subset_adata.layers["total"]), + ) + + Moment = MomData(subset_adata, tkey) + if cur_grp == _group[0]: + t_ind = 0 + g_len, t_len = len(_group), len(np.unique(adata.obs[tkey])) + adata.uns["M"], adata.uns["V"] = ( + np.zeros((adata.shape[1], g_len * t_len)), + np.zeros((adata.shape[1], g_len * t_len)), + ) + + inds = np.arange((t_len * t_ind), (t_len * (t_ind + 1))) + ( + adata.uns["M"][:, inds], + adata.uns["V"][:, inds], + ) = (Moment.M, Moment.V) + Est = Estimation(Moment, time_key=tkey, normalize=True) # # data is already normalized + + return adata, Est, t_ind diff --git a/dynamo/tools/moments.py b/dynamo/tools/moments.py index 8d56d597e..c09b328cc 100755 --- a/dynamo/tools/moments.py +++ b/dynamo/tools/moments.py @@ -1116,27 +1116,6 @@ def calc_mom_all_genes( return Mn, Mo, Mt, Mr -def _calc_1nd_moment(X, W, normalize_W=True): - """deprecated""" - if normalize_W: - d = np.sum(W, 1) - W = np.diag(1 / d) @ W - return W @ X - - -def _calc_2nd_moment(X, Y, W, normalize_W=True, center=False, mX=None, mY=None): - """deprecated""" - if normalize_W: - d = np.sum(W, 1) - W = np.diag(1 / d) @ W - XY = np.multiply(W @ Y, X) - if center: - mX = calc_1nd_moment(X, W, False) if mX is None else mX - mY = calc_1nd_moment(Y, W, False) if mY is None else mY - XY = XY - np.multiply(mX, mY) - return XY - - def gaussian_kernel( X: np.ndarray, nbr_idx: np.ndarray, sigma: int, k: Optional[int] = None, dists: Optional[np.ndarray] = None ) -> csr_matrix: @@ -1260,235 +1239,3 @@ def calc_2nd_moment( XY = XY - elem_prod(mX, mY) return XY - - -# --------------------------------------------------------------------------------------------------- -# old moment estimation code -class MomData(AnnData): - """deprecated""" - - def __init__(self, adata, time_key="Time", has_nan=False): - # self.data = adata - self.__dict__ = adata.__dict__ - # calculate first and second moments from data - self.times = np.array(self.obs[time_key].values, dtype=float) - self.uniq_times = np.unique(self.times) - nT = self.get_n_times() - ng = self.get_n_genes() - self.M = np.zeros((ng, nT)) # first moments (data) - self.V = np.zeros((ng, nT)) # second moments (data) - for g in tqdm(range(ng), desc="calculating 1/2 moments"): - tmp = self[:, g].layers["new"] - L = ( - np.array(tmp.A, dtype=float) if issparse(tmp) else np.array(tmp, dtype=float) - ) # consider using the `adata.obs_vector`, `adata.var_vector` methods or accessing the array directly. - if has_nan: - self.M[g] = strat_mom(L, self.times, np.nanmean) - self.V[g] = strat_mom(L, self.times, np.nanvar) - else: - self.M[g] = strat_mom(L, self.times, np.mean) - self.V[g] = strat_mom(L, self.times, np.var) - - def get_n_genes(self): - return self.var.shape[0] - - def get_n_cell(self): - return self.obs.shape[0] - - def get_n_times(self): - return len(self.uniq_times) - - -class Estimation: - """deprecated""" - - def __init__( - self, - adata, - adata_u=None, - time_key="Time", - normalize=True, - param_ranges=None, - has_nan=False, - ): - # initialize Estimation - self.data = MomData(adata, time_key, has_nan) - self.data_u = MomData(adata_u, time_key, has_nan) if adata_u is not None else None - if param_ranges is None: - param_ranges = { - "a": [0, 10], - "b": [0, 10], - "alpha_a": [10, 1000], - "alpha_i": [0, 10], - "beta": [0, 10], - "gamma": [0, 10], - } - self.normalize = normalize - self.param_ranges = param_ranges - self.n_params = len(param_ranges) - - def param_array2dict(self, parr): - if parr.ndim == 1: - return { - "a": parr[0], - "b": parr[1], - "alpha_a": parr[2], - "alpha_i": parr[3], - "beta": parr[4], - "gamma": parr[5], - } - else: - return { - "a": parr[:, 0], - "b": parr[:, 1], - "alpha_a": parr[:, 2], - "alpha_i": parr[:, 3], - "beta": parr[:, 4], - "gamma": parr[:, 5], - } - - def fit_gene(self, gene_no, n_p0=10): - from ..estimation.tsc.utils_moments import estimation - - estm = estimation(list(self.param_ranges.values())) - if self.data_u is None: - m = self.data.M[gene_no, :].T - v = self.data.V[gene_no, :].T - x_data = np.vstack((m, v)) - popt, cost = estm.fit_lsq( - self.data.uniq_times, - x_data, - p0=None, - n_p0=n_p0, - normalize=self.normalize, - experiment_type="nosplice", - ) - else: - mu = self.data_u.M[gene_no, :].T - ms = self.data.M[gene_no, :].T - vu = self.data_u.V[gene_no, :].T - vs = self.data.V[gene_no, :].T - x_data = np.vstack((mu, ms, vu, vs)) - popt, cost = estm.fit_lsq( - self.data.uniq_times, - x_data, - p0=None, - n_p0=n_p0, - normalize=self.normalize, - experiment_type=None, - ) - return popt, cost - - def fit(self, n_p0=10): - ng = self.data.get_n_genes() - params = np.zeros((ng, self.n_params)) - costs = np.zeros(ng) - for i in tqdm(range(ng), desc="fitting genes"): - params[i], costs[i] = self.fit_gene(i, n_p0) - return params, costs - - -# --------------------------------------------------------------------------------------------------- -# use for kinetic assumption with full data, deprecated -def moment_model(adata, subset_adata, _group, cur_grp, log_unnormalized, tkey): - """deprecated""" - # a few hard code to set up data for moment mode: - if "uu" in subset_adata.layers.keys() or "X_uu" in subset_adata.layers.keys(): - if log_unnormalized and "X_uu" not in subset_adata.layers.keys(): - if issparse(subset_adata.layers["uu"]): - ( - subset_adata.layers["uu"].data, - subset_adata.layers["ul"].data, - subset_adata.layers["su"].data, - subset_adata.layers["sl"].data, - ) = ( - np.log1p(subset_adata.layers["uu"].data), - np.log1p(subset_adata.layers["ul"].data), - np.log1p(subset_adata.layers["su"].data), - np.log1p(subset_adata.layers["sl"].data), - ) - else: - ( - subset_adata.layers["uu"], - subset_adata.layers["ul"], - subset_adata.layers["su"], - subset_adata.layers["sl"], - ) = ( - np.log1p(subset_adata.layers["uu"]), - np.log1p(subset_adata.layers["ul"]), - np.log1p(subset_adata.layers["su"]), - np.log1p(subset_adata.layers["sl"]), - ) - - subset_adata_u, subset_adata_s = ( - subset_adata.copy(), - subset_adata.copy(), - ) - del ( - subset_adata_u.layers["su"], - subset_adata_u.layers["sl"], - subset_adata_s.layers["uu"], - subset_adata_s.layers["ul"], - ) - ( - subset_adata_u.layers["new"], - subset_adata_u.layers["old"], - subset_adata_s.layers["new"], - subset_adata_s.layers["old"], - ) = ( - subset_adata_u.layers.pop("ul"), - subset_adata_u.layers.pop("uu"), - subset_adata_s.layers.pop("sl"), - subset_adata_s.layers.pop("su"), - ) - Moment, Moment_ = MomData(subset_adata_s, tkey), MomData(subset_adata_u, tkey) - if cur_grp == _group[0]: - t_ind = 0 - g_len, t_len = len(_group), len(np.unique(adata.obs[tkey])) - (adata.uns["M_sl"], adata.uns["V_sl"], adata.uns["M_ul"], adata.uns["V_ul"]) = ( - np.zeros((Moment.M.shape[0], g_len * t_len)), - np.zeros((Moment.M.shape[0], g_len * t_len)), - np.zeros((Moment.M.shape[0], g_len * t_len)), - np.zeros((Moment.M.shape[0], g_len * t_len)), - ) - - inds = np.arange((t_len * t_ind), (t_len * (t_ind + 1))) - ( - adata.uns["M_sl"][:, inds], - adata.uns["V_sl"][:, inds], - adata.uns["M_ul"][:, inds], - adata.uns["V_ul"][:, inds], - ) = (Moment.M, Moment.V, Moment_.M, Moment_.V) - - del Moment_ - Est = Estimation(Moment, adata_u=subset_adata_u, time_key=tkey, normalize=True) # # data is already normalized - else: - if log_unnormalized and "X_total" not in subset_adata.layers.keys(): - if issparse(subset_adata.layers["total"]): - (subset_adata.layers["new"].data, subset_adata.layers["total"].data,) = ( - np.log1p(subset_adata.layers["new"].data), - np.log1p(subset_adata.layers["total"].data), - ) - else: - subset_adata.layers["total"], subset_adata.layers["total"] = ( - np.log1p(subset_adata.layers["new"]), - np.log1p(subset_adata.layers["total"]), - ) - - Moment = MomData(subset_adata, tkey) - if cur_grp == _group[0]: - t_ind = 0 - g_len, t_len = len(_group), len(np.unique(adata.obs[tkey])) - adata.uns["M"], adata.uns["V"] = ( - np.zeros((adata.shape[1], g_len * t_len)), - np.zeros((adata.shape[1], g_len * t_len)), - ) - - inds = np.arange((t_len * t_ind), (t_len * (t_ind + 1))) - ( - adata.uns["M"][:, inds], - adata.uns["V"][:, inds], - ) = (Moment.M, Moment.V) - Est = Estimation(Moment, time_key=tkey, normalize=True) # # data is already normalized - - return adata, Est, t_ind From a2a52a35456d6473c01937496085d504a3e600f5 Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 15:50:51 -0500 Subject: [PATCH 08/14] reorganize multiomics and deprecate infomap --- dynamo/tools/clustering.py | 44 ---------------------------------- dynamo/tools/deprecated.py | 48 +++++++++++++++++++++++++++++++++++++- dynamo/tools/multiomics.py | 48 -------------------------------------- dynamo/tools/utils.py | 41 ++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 93 deletions(-) delete mode 100755 dynamo/tools/multiomics.py diff --git a/dynamo/tools/clustering.py b/dynamo/tools/clustering.py index 8f1b83bae..adc37f144 100644 --- a/dynamo/tools/clustering.py +++ b/dynamo/tools/clustering.py @@ -345,50 +345,6 @@ def louvain( ) -def infomap( - adata: AnnData, - use_weight: bool = True, - adj_matrix: Union[np.ndarray, csr_matrix, None] = None, - adj_matrix_key: Optional[str] = None, - result_key: Optional[str] = None, - layer: Optional[str] = None, - obsm_key: Optional[str] = None, - selected_cluster_subset: Optional[Tuple[str, str]] = None, - selected_cell_subset: Union[List[int], List[str], None] = None, - directed: bool = False, - copy: bool = False, - **kwargs -) -> AnnData: - """Apply infomap community detection algorithm to cluster adata. - - For other community detection general parameters, please refer to `dynamo`'s `tl.cluster_community` function. - "Infomap is based on ideas of information theory. The algorithm uses the probability flow of random walks on a - network as a proxy for information flows in the real system and it decomposes the network into modules by - compressing a description of the probability flow." - cdlib - - Args: - adata: an AnnData object. - use_weight: whether to use graph weight or not. False means to use connectivities only (0/1 integer values). - Defaults to True. - adj_matrix: adj_matrix used for clustering. Defaults to None. - adj_matrix_key: the key for adj_matrix stored in adata.obsp. Defaults to None. - result_key: the key where the results will be stored in obs. Defaults to None. - layer: the adata layer on which cluster algorithms will work. Defaults to None. - obsm_key: the key in obsm corresponding to the data that would be used for finding neighbors. Defaults to None. - selected_cluster_subset: a tuple of (cluster_key, allowed_clusters).Filtering cells in adata based on - cluster_key in adata.obs and only reserve cells in the allowed clusters. Defaults to None. - selected_cell_subset: a subset of cells in adata that would be clustered. Could be a list of indices or a list - of cell names. Defaults to None. - directed: whether the edges in the graph should be directed. Defaults to False. - copy: whether to return a new updated AnnData object or updated the original one inplace. Defaults to False. - - Returns: - An updated AnnData object if `copy` is set to be true. - """ - - raise NotImplementedError("infomap algorithm has been deprecated.") - - def cluster_community( adata: AnnData, method: Literal["leiden", "louvain"] = "leiden", diff --git a/dynamo/tools/deprecated.py b/dynamo/tools/deprecated.py index cebd38dee..850bfe003 100644 --- a/dynamo/tools/deprecated.py +++ b/dynamo/tools/deprecated.py @@ -16,7 +16,7 @@ from numpy import * from scipy.integrate import odeint from scipy.optimize import curve_fit, least_squares -from scipy.sparse import issparse +from scipy.sparse import csr_matrix, issparse from scipy.sparse.csgraph import shortest_path from tqdm import tqdm @@ -1864,3 +1864,49 @@ def moment_model(adata, subset_adata, _group, cur_grp, log_unnormalized, tkey): Est = Estimation(Moment, time_key=tkey, normalize=True) # # data is already normalized return adata, Est, t_ind + + +#--------------------------------------------------------------------------------------------------- +# deprecated clustering.py +def infomap( + adata: AnnData, + use_weight: bool = True, + adj_matrix: Union[np.ndarray, csr_matrix, None] = None, + adj_matrix_key: Optional[str] = None, + result_key: Optional[str] = None, + layer: Optional[str] = None, + obsm_key: Optional[str] = None, + selected_cluster_subset: Optional[Tuple[str, str]] = None, + selected_cell_subset: Union[List[int], List[str], None] = None, + directed: bool = False, + copy: bool = False, + **kwargs +) -> AnnData: + """Apply infomap community detection algorithm to cluster adata. + + For other community detection general parameters, please refer to `dynamo`'s `tl.cluster_community` function. + "Infomap is based on ideas of information theory. The algorithm uses the probability flow of random walks on a + network as a proxy for information flows in the real system and it decomposes the network into modules by + compressing a description of the probability flow." - cdlib + + Args: + adata: an AnnData object. + use_weight: whether to use graph weight or not. False means to use connectivities only (0/1 integer values). + Defaults to True. + adj_matrix: adj_matrix used for clustering. Defaults to None. + adj_matrix_key: the key for adj_matrix stored in adata.obsp. Defaults to None. + result_key: the key where the results will be stored in obs. Defaults to None. + layer: the adata layer on which cluster algorithms will work. Defaults to None. + obsm_key: the key in obsm corresponding to the data that would be used for finding neighbors. Defaults to None. + selected_cluster_subset: a tuple of (cluster_key, allowed_clusters).Filtering cells in adata based on + cluster_key in adata.obs and only reserve cells in the allowed clusters. Defaults to None. + selected_cell_subset: a subset of cells in adata that would be clustered. Could be a list of indices or a list + of cell names. Defaults to None. + directed: whether the edges in the graph should be directed. Defaults to False. + copy: whether to return a new updated AnnData object or updated the original one inplace. Defaults to False. + + Returns: + An updated AnnData object if `copy` is set to be true. + """ + + raise NotImplementedError("infomap algorithm has been deprecated.") \ No newline at end of file diff --git a/dynamo/tools/multiomics.py b/dynamo/tools/multiomics.py deleted file mode 100755 index 8557ace7f..000000000 --- a/dynamo/tools/multiomics.py +++ /dev/null @@ -1,48 +0,0 @@ -import anndata -import pandas as pd - -# 1. concatenate RNA/protein data -# 2. filter gene/protein for velocity -# 3. use steady state assumption to calculate protein velocity -# 4. use the PRL paper to estimate the parameters - - -def AddAssay(adata: anndata.AnnData, data: pd.DataFrame, key: str, slot: str = "obsm") -> anndata.AnnData: - """Add a new data as a key to the specified slot. - - Args: - adata: An AnnData object. - data: The data (in pandas DataFrame format) that will be added to adata. - key: The key name to be used for the new data. - slot: The slot of adata to store the new data. Defaults to "obsm". - - Returns: - An updated anndata object that are updated with a new data as a key to the specified slot. - """ - - if slot == "uns": - adata.uns[key] = data.loc[adata.obs.index, set(adata.var.index).intersection(data.columns)] - elif slot == "obsm": - adata.obsm[key] = data.loc[adata.obs.index, set(adata.var.index).intersection(data.columns)] - - return adata - - -def getAssay(adata: anndata.AnnData, key: str, slot: str = "obsm") -> pd.DataFrame: - """Retrieve a key named data from the specified slot. - - Args: - adata: An AnnData object. - key: The key name of the data to be retrieved. . - slot: The slot of adata to be retrieved from. Defaults to "obsm". - - Returns: - The data (in pd.DataFrame) that will be retrieved from adata. - """ - - if slot == "uns": - data = adata.uns[key] - elif slot == "obsm": - data = adata.obsm[key] - - return data diff --git a/dynamo/tools/utils.py b/dynamo/tools/utils.py index ef5fc7999..41667db9f 100755 --- a/dynamo/tools/utils.py +++ b/dynamo/tools/utils.py @@ -279,6 +279,47 @@ def create_layer( return new +def AddAssay(adata: AnnData, data: pd.DataFrame, key: str, slot: str = "obsm") -> AnnData: + """Add a new data as a key to the specified slot. + + Args: + adata: An AnnData object. + data: The data (in pandas DataFrame format) that will be added to adata. + key: The key name to be used for the new data. + slot: The slot of adata to store the new data. Defaults to "obsm". + + Returns: + An updated anndata object that are updated with a new data as a key to the specified slot. + """ + + if slot == "uns": + adata.uns[key] = data.loc[adata.obs.index, set(adata.var.index).intersection(data.columns)] + elif slot == "obsm": + adata.obsm[key] = data.loc[adata.obs.index, set(adata.var.index).intersection(data.columns)] + + return adata + + +def getAssay(adata: AnnData, key: str, slot: str = "obsm") -> pd.DataFrame: + """Retrieve a key named data from the specified slot. + + Args: + adata: An AnnData object. + key: The key name of the data to be retrieved. . + slot: The slot of adata to be retrieved from. Defaults to "obsm". + + Returns: + The data (in pd.DataFrame) that will be retrieved from adata. + """ + + if slot == "uns": + data = adata.uns[key] + elif slot == "obsm": + data = adata.obsm[key] + + return data + + def index_gene(adata: AnnData, arr: np.ndarray, genes: List[str]) -> np.ndarray: """A lightweight method for indexing adata arrays by genes. From 5796e7f6f60c28caf9d1230b4f4c611ff533d8ef Mon Sep 17 00:00:00 2001 From: sichao Date: Fri, 17 Nov 2023 17:40:21 -0500 Subject: [PATCH 09/14] rename pseudptime_graph to DDRTree_graph --- .../{pseudotime_graph.py => DDRTree_graph.py} | 25 +++++++++++++++++-- dynamo/tools/__init__.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) rename dynamo/tools/{pseudotime_graph.py => DDRTree_graph.py} (94%) diff --git a/dynamo/tools/pseudotime_graph.py b/dynamo/tools/DDRTree_graph.py similarity index 94% rename from dynamo/tools/pseudotime_graph.py rename to dynamo/tools/DDRTree_graph.py index 5a194db1d..2d4ea49e1 100755 --- a/dynamo/tools/pseudotime_graph.py +++ b/dynamo/tools/DDRTree_graph.py @@ -93,7 +93,17 @@ def _get_path( parents_dict: Dict, start: int, end_nodes: List, -): +) -> List: + """Get the path from the start node to the end node. + + Args: + parents_dict: The dictionary that maps each node to its parent node. + start: The start node. + end_nodes: The end nodes. + + Returns: + The path from the start node to the end node. + """ if parents_dict[start] == -1: return None cur = parents_dict[start] @@ -104,7 +114,18 @@ def _get_path( return path -def _get_all_segments(orders: Union[np.ndarray, List], parents: Union[np.ndarray, List]): +def _get_all_segments(orders: Union[np.ndarray, List], parents: Union[np.ndarray, List]) -> List: + """Get all segments from the minimum spanning tree. + + Segments is defined as a path without any bifurcations. + + Args: + orders: The order to traverse the minimum spanning tree. + parents: The parent node for each node. + + Returns: + A list of segments. + """ from collections import Counter leaf_nodes = [node for node in orders if node not in parents] diff --git a/dynamo/tools/__init__.py b/dynamo/tools/__init__.py index 2c8a98ab0..5a33085a5 100755 --- a/dynamo/tools/__init__.py +++ b/dynamo/tools/__init__.py @@ -35,7 +35,7 @@ ) # Pseudotime related -from .pseudotime_graph import construct_velocity_tree, directed_pg +from .DDRTree_graph import construct_velocity_tree, directed_pg from .DDRTree import DDRTree, cal_ncenter from .pseudotime import order_cells From e18d3a6e1e2eb5be59735eba64300c77f6d7535d Mon Sep 17 00:00:00 2001 From: sichao Date: Mon, 20 Nov 2023 19:06:40 -0500 Subject: [PATCH 10/14] rename _gen_neighbor_keys to generate_neighbor_keys --- dynamo/external/hodge.py | 4 ++-- dynamo/tools/cell_velocities.py | 4 ++-- dynamo/tools/clustering.py | 4 ++-- dynamo/tools/connectivity.py | 8 ++++---- dynamo/tools/dimension_reduction.py | 4 ++-- dynamo/tools/markers.py | 4 ++-- dynamo/tools/utils_reduceDimension.py | 4 ++-- dynamo/vectorfield/stochastic_process.py | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dynamo/external/hodge.py b/dynamo/external/hodge.py index 5e8928833..ac75d44c9 100644 --- a/dynamo/external/hodge.py +++ b/dynamo/external/hodge.py @@ -21,7 +21,7 @@ div, potential, )""" -from ..tools.connectivity import _gen_neighbor_keys, check_and_recompute_neighbors +from ..tools.connectivity import generate_neighbor_keys, check_and_recompute_neighbors def ddhodge( @@ -136,7 +136,7 @@ def func(x): main_info("graphizing vectorfield...") V_data = func(X_data) neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) if neighbor_key not in adata_.uns_keys() or to_downsample: existing_nbrs_idx = None else: diff --git a/dynamo/tools/cell_velocities.py b/dynamo/tools/cell_velocities.py index 3076247fa..0a3f82acb 100755 --- a/dynamo/tools/cell_velocities.py +++ b/dynamo/tools/cell_velocities.py @@ -16,7 +16,7 @@ from ..configuration import DKM from ..dynamo_logger import LoggerManager, main_info, main_warning from ..utils import areinstance, expr_to_pca -from .connectivity import _gen_neighbor_keys, adj_to_knn, check_and_recompute_neighbors, construct_mapper_umap +from .connectivity import generate_neighbor_keys, adj_to_knn, check_and_recompute_neighbors, construct_mapper_umap from .dimension_reduction import reduceDimension from .graph_calculus import calc_gaussian_weight, fp_operator, graphize_velocity from .Markov import ContinuousTimeMarkovChain, KernelMarkovChain, velocity_on_grid @@ -166,7 +166,7 @@ def cell_velocities( the Itô kernel method or similar methods from (La Manno et al. 2018). """ - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_key_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_key_prefix) mapper_r = get_mapper_inverse() layer = mapper_r[ekey] if (ekey is not None and ekey in mapper_r.keys()) else ekey ekey, vkey, layer = get_ekey_vkey_from_adata(adata) if (ekey is None or vkey is None) else (ekey, vkey, layer) diff --git a/dynamo/tools/clustering.py b/dynamo/tools/clustering.py index adc37f144..8136e0671 100644 --- a/dynamo/tools/clustering.py +++ b/dynamo/tools/clustering.py @@ -17,7 +17,7 @@ from ..preprocessing.pca import pca from ..preprocessing.transform import log1p from ..utils import LoggerManager, copy_adata -from .connectivity import _gen_neighbor_keys, neighbors +from .connectivity import generate_neighbor_keys, neighbors from .utils import update_dict from .utils_reduceDimension import prepare_dim_reduction, run_reduce_dim @@ -102,7 +102,7 @@ def hdbscan( reduction_method = basis.split("_")[-1] embedding_key = "X_" + reduction_method if layer is None else layer + "_" + reduction_method neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) adata = run_reduce_dim( adata, diff --git a/dynamo/tools/connectivity.py b/dynamo/tools/connectivity.py index 037cd3cfa..3492d2f84 100755 --- a/dynamo/tools/connectivity.py +++ b/dynamo/tools/connectivity.py @@ -594,7 +594,7 @@ def mnn( return adata -def _gen_neighbor_keys(result_prefix: str = "") -> Tuple[str, str, str]: +def generate_neighbor_keys(result_prefix: str = "") -> Tuple[str, str, str]: """Generate neighbor keys for other functions to store/access info in adata. Args: @@ -845,7 +845,7 @@ def neighbors( **kwargs, ) - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(result_prefix) logger.info_insert_adata(conn_key, adata_attr="obsp") logger.info_insert_adata(dist_key, adata_attr="obsp") adata.obsp[dist_key], adata.obsp[conn_key] = get_conn_dist_graph(knn, distances) @@ -890,7 +890,7 @@ def check_neighbors_completeness( """ is_valid = True - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(result_prefix) keys = [conn_key, dist_key, neighbor_key] # Old anndata version version @@ -956,7 +956,7 @@ def check_and_recompute_neighbors(adata: AnnData, result_prefix: str = "") -> No if result_prefix is None: result_prefix = "" - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(result_prefix) if not check_neighbors_completeness(adata, conn_key=conn_key, dist_key=dist_key, result_prefix=result_prefix): main_info("Neighbor graph is broken, recomputing....") diff --git a/dynamo/tools/dimension_reduction.py b/dynamo/tools/dimension_reduction.py index 5ceef3517..a586d85e4 100755 --- a/dynamo/tools/dimension_reduction.py +++ b/dynamo/tools/dimension_reduction.py @@ -5,7 +5,7 @@ from ..dynamo_logger import LoggerManager from ..utils import copy_adata -from .connectivity import _gen_neighbor_keys, neighbors +from .connectivity import generate_neighbor_keys, neighbors from .utils import update_dict from .utils_reduceDimension import prepare_dim_reduction, run_reduce_dim @@ -100,7 +100,7 @@ def reduceDimension( embedding_key = "X_" + reduction_method if layer is None else layer + "_" + reduction_method if neighbor_key is None: neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) if enforce or not has_basis: logger.info(f"[{reduction_method.upper()}] using {basis} with n_pca_components = {n_pca_components}", indent_level=1) diff --git a/dynamo/tools/markers.py b/dynamo/tools/markers.py index b7063e248..13d005da3 100755 --- a/dynamo/tools/markers.py +++ b/dynamo/tools/markers.py @@ -30,7 +30,7 @@ main_warning, ) from ..preprocessing.transform import _Freeman_Tukey -from ..tools.connectivity import _gen_neighbor_keys, check_and_recompute_neighbors +from ..tools.connectivity import generate_neighbor_keys, check_and_recompute_neighbors from .utils import fdr, fetch_X_data @@ -98,7 +98,7 @@ def moran_i( embedding_key = "X_umap" if layer is None else layer + "_umap" neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) if neighbor_key not in adata.uns.keys(): main_warning( diff --git a/dynamo/tools/utils_reduceDimension.py b/dynamo/tools/utils_reduceDimension.py index 66326b9fc..e7a8fa546 100644 --- a/dynamo/tools/utils_reduceDimension.py +++ b/dynamo/tools/utils_reduceDimension.py @@ -14,7 +14,7 @@ from ..dynamo_logger import main_info_insert_adata_obsm from ..preprocessing.pca import pca from .connectivity import ( - _gen_neighbor_keys, + generate_neighbor_keys, knn_to_adj, umap_conn_indices_dist_embedding, ) @@ -300,7 +300,7 @@ def run_reduce_dim( layer = neighbor_key.split("_")[0] if neighbor_key.__contains__("_") else None neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) adata.uns["umap_fit"] = { "X_data": X_data, diff --git a/dynamo/vectorfield/stochastic_process.py b/dynamo/vectorfield/stochastic_process.py index e9ca0a710..291dccc60 100644 --- a/dynamo/vectorfield/stochastic_process.py +++ b/dynamo/vectorfield/stochastic_process.py @@ -5,7 +5,7 @@ from sklearn.neighbors import NearestNeighbors from tqdm import tqdm -from ..tools.connectivity import _gen_neighbor_keys, check_and_recompute_neighbors, k_nearest_neighbors +from ..tools.connectivity import generate_neighbor_keys, check_and_recompute_neighbors, k_nearest_neighbors from ..tools.utils import log1p_ from .utils import VecFldDict, vecfld_from_adata, vector_field_function @@ -140,7 +140,7 @@ def diffusionMatrix( X_data, V_data = X_data[:, dims], V_data[:, dims] neighbor_result_prefix = "" if layer is None else layer - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) if neighbor_key not in adata.uns_keys() or (X_data is not None and V_data is not None): Idx, _ = k_nearest_neighbors( X_data, From 01741e39d6cbae1f14e3b7dc2a6e9d0a2c35e412 Mon Sep 17 00:00:00 2001 From: sichao Date: Tue, 21 Nov 2023 14:32:24 -0500 Subject: [PATCH 11/14] reorganize orders in files --- dynamo/tools/DDRTree.py | 220 ++-- dynamo/tools/DDRTree_graph.py | 264 ++--- dynamo/tools/Markov.py | 1954 +++++++++++++++---------------- dynamo/tools/connectivity.py | 511 ++++---- dynamo/tools/graph_calculus.py | 162 +-- dynamo/tools/graph_operators.py | 150 +-- dynamo/tools/metric_velocity.py | 129 +- 7 files changed, 1696 insertions(+), 1694 deletions(-) diff --git a/dynamo/tools/DDRTree.py b/dynamo/tools/DDRTree.py index 3f97cdefb..60b610b32 100755 --- a/dynamo/tools/DDRTree.py +++ b/dynamo/tools/DDRTree.py @@ -10,100 +10,6 @@ from scipy.sparse.linalg import inv -def cal_ncenter(ncells: int, ncells_limit: int = 100) -> int: - """Calculate the number of cells to be most significant in the reduced space. - - Args: - ncells: Total number of cells. - ncells_limit: The max number of cells to be considered. Defaults to 100. - - Returns: - The number of cells to be most significant in the reduced space. - """ - - res = np.round( - 2 * ncells_limit * np.log(ncells) / (np.log(ncells) + np.log(ncells_limit)) - ) - - return res - - -def pca_projection(C: np.ndarray, L: int) -> np.ndarray: - """Solve the problem size(C) = NxN, size(W) = NxL. max_W trace( W' C W ) : W' W = I - - Args: - C: The matrix to calculate eigenvalues. - L: The number of Eigenvalues. - - Returns: - The L largest Eigenvalues. - """ - - V, U = eig(C) - eig_idx = np.argsort(V).tolist() - eig_idx.reverse() - W = U.T[eig_idx[0:L]].T - return W - - -def sqdist(a: np.ndarray, b: np.ndarray) -> np.ndarray: - """Calculate the square distance between `a` and `b`. - - Args: - a: A matrix with dimension D x N - b: A matrix with dimension D x N - - Returns: - A numeric value for the difference between a and b. - """ - - aa = np.sum(a ** 2, axis=0) - bb = np.sum(b ** 2, axis=0) - ab = a.T.dot(b) - - aa_repmat = matlib.repmat(aa[:, None], 1, b.shape[1]) - bb_repmat = matlib.repmat(bb[None, :], a.shape[1], 1) - - dist = abs(aa_repmat + bb_repmat - 2 * ab) - - return dist - - -def repmat(X: np.ndarray, m: int, n: int) -> np.ndarray: - """This function returns an array containing m (n) copies of A in the row (column) dimensions. - - The size of B is size(A)*n when A is a matrix. For example, repmat(np.matrix(1:4), 2, 3) returns a 4-by-6 matrix. - - Args: - X: An array like matrix. - m: Number of copies on row dimension. - n: Number of copies on column dimension. - - Returns: - The constructed repmat. - """ - - xy_rep = matlib.repmat(X, m, n) - - return xy_rep - - -def eye(m: int, n: int) -> np.ndarray: - """Equivalent of eye (matlab). - - Return a m x n matrix with 0th diagonal to be 1 and the rest to be 0. - - Args: - m: Number of rows. - n: Number of columns. - - Returns: - The m x n eye matrix. - """ - mat = np.eye(m, n) - return mat - - def DDRTree( X: np.ndarray, maxIter: int, @@ -115,21 +21,21 @@ def DDRTree( ncenter: Optional[int] = None, keep_history: bool = False, ) -> Union[ - pd.DataFrame, + pd.DataFrame, Tuple[ - np.ndarray, - np.ndarray, - np.ndarray, - np.ndarray, - np.ndarray, - np.ndarray, - np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, List[np.ndarray], ], ]: - """Provides an implementation of the framework of reversed graph embedding (RGE). + """Provides an implementation of the framework of reversed graph embedding (RGE). - This function is a python version of the DDRTree algorithm originally written in R. + This function is a python version of the DDRTree algorithm originally written in R. (https://cran.r-project.org/web/packages/DDRTree/DDRTree.pdf) Args: @@ -144,13 +50,13 @@ def DDRTree( keep_history: Whether to keep relative parameters during each iteration and return. Defaults to False. Returns: - A dataframe containing `W`, `Z`, `Y`, `stree`, `R`, `objs` for each iterations if `keep_history` is True. - Otherwise, a tuple (Z, Y, stree, R, W, Q, C, objs). The items in the tuple is from the last iteration. `Z` is + A dataframe containing `W`, `Z`, `Y`, `stree`, `R`, `objs` for each iterations if `keep_history` is True. + Otherwise, a tuple (Z, Y, stree, R, W, Q, C, objs). The items in the tuple is from the last iteration. `Z` is the reduced dimension; `Y` is the latent points as the center of Z; `stree` is the smooth tree graph embedded in - the low dimension space; `R` is used to transform the hard assignments used in K-means into soft assignments; - `W` is the orthogonal set of d (dimensions) linear basis; `Q` is (I + lambda L)^(-1), where L = diag(B1) - B, a - Laplacian matrix. `C` equals to XQ^(-1)X^T; `objs` is a list containing convergency conditions during the - iterations. + the low dimension space; `R` is used to transform the hard assignments used in K-means into soft assignments; + `W` is the orthogonal set of d (dimensions) linear basis; `Q` is (I + lambda L)^(-1), where L = diag(B1) - B, a + Laplacian matrix. `C` equals to XQ^(-1)X^T; `objs` is a list containing convergency conditions during the + iterations. """ X = np.array(X).T @@ -245,3 +151,97 @@ def DDRTree( return history else: return Z, Y, stree, R, W, Q, C, objs + + +def cal_ncenter(ncells: int, ncells_limit: int = 100) -> int: + """Calculate the number of cells to be most significant in the reduced space. + + Args: + ncells: Total number of cells. + ncells_limit: The max number of cells to be considered. Defaults to 100. + + Returns: + The number of cells to be most significant in the reduced space. + """ + + res = np.round( + 2 * ncells_limit * np.log(ncells) / (np.log(ncells) + np.log(ncells_limit)) + ) + + return res + + +def pca_projection(C: np.ndarray, L: int) -> np.ndarray: + """Solve the problem size(C) = NxN, size(W) = NxL. max_W trace( W' C W ) : W' W = I + + Args: + C: The matrix to calculate eigenvalues. + L: The number of Eigenvalues. + + Returns: + The L largest Eigenvalues. + """ + + V, U = eig(C) + eig_idx = np.argsort(V).tolist() + eig_idx.reverse() + W = U.T[eig_idx[0:L]].T + return W + + +def sqdist(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Calculate the square distance between `a` and `b`. + + Args: + a: A matrix with dimension D x N + b: A matrix with dimension D x N + + Returns: + A numeric value for the difference between a and b. + """ + + aa = np.sum(a ** 2, axis=0) + bb = np.sum(b ** 2, axis=0) + ab = a.T.dot(b) + + aa_repmat = matlib.repmat(aa[:, None], 1, b.shape[1]) + bb_repmat = matlib.repmat(bb[None, :], a.shape[1], 1) + + dist = abs(aa_repmat + bb_repmat - 2 * ab) + + return dist + + +def repmat(X: np.ndarray, m: int, n: int) -> np.ndarray: + """This function returns an array containing m (n) copies of A in the row (column) dimensions. + + The size of B is size(A)*n when A is a matrix. For example, repmat(np.matrix(1:4), 2, 3) returns a 4-by-6 matrix. + + Args: + X: An array like matrix. + m: Number of copies on row dimension. + n: Number of copies on column dimension. + + Returns: + The constructed repmat. + """ + + xy_rep = matlib.repmat(X, m, n) + + return xy_rep + + +def eye(m: int, n: int) -> np.ndarray: + """Equivalent of eye (matlab). + + Return a m x n matrix with 0th diagonal to be 1 and the rest to be 0. + + Args: + m: Number of rows. + n: Number of columns. + + Returns: + The m x n eye matrix. + """ + mat = np.eye(m, n) + return mat diff --git a/dynamo/tools/DDRTree_graph.py b/dynamo/tools/DDRTree_graph.py index 2d4ea49e1..a932edcd1 100755 --- a/dynamo/tools/DDRTree_graph.py +++ b/dynamo/tools/DDRTree_graph.py @@ -10,6 +10,138 @@ from ..dynamo_logger import main_info, main_info_insert_adata_uns +def construct_velocity_tree(adata: AnnData, transition_matrix_key: str = "pearson"): + """Integrate pseudotime ordering with velocity to automatically assign the direction of the learned trajectory. + + Args: + adata: The anndata object containing the single-cell data. + transition_matrix_key (str, optional): Key to the transition matrix in the `adata.obsp` object that represents + the transition probabilities between cells. Defaults to "pearson". + + Raises: + KeyError: If the transition matrix or cell order information is not found in the `adata` object. + + Returns: + A directed velocity tree represented as a NumPy array. + """ + if transition_matrix_key + "_transition_matrix" not in adata.obsp.keys(): + raise KeyError("Transition matrix not found in anndata. Please call cell_velocities() before constructing " + "velocity tree") + + if "cell_order" not in adata.uns.keys(): + raise KeyError("Cell order information not found in anndata. Please call order_cells() before constructing " + "velocity tree.") + + main_info("Constructing velocity tree...") + + transition_matrix = adata.obsp[transition_matrix_key + "_transition_matrix"] + R = adata.uns["cell_order"]["R"] + orders = np.argsort(adata.uns["cell_order"]["centers_order"]) + parents = [adata.uns["cell_order"]["centers_parent"][node] for node in orders] + velocity_tree = adata.uns["cell_order"]["centers_minSpanningTree"] + directed_velocity_tree = velocity_tree.copy() + + segments = _get_all_segments(orders, parents) + center_transition_matrix = _compute_center_transition_matrix(transition_matrix, R) + + for segment in segments: + edge_pairs = _get_edges(segment) + edge_pairs_reversed = _get_edges(segment[::-1]) + segment_p = _calculate_segment_probability(center_transition_matrix, edge_pairs) + segment_p_reveresed = _calculate_segment_probability(center_transition_matrix, edge_pairs_reversed) + if segment_p[-1] > segment_p_reveresed[-1]: + for i, (r, c) in enumerate(edge_pairs): + directed_velocity_tree[r, c] = max(velocity_tree[r, c], velocity_tree[c, r]) + directed_velocity_tree[c, r] = 0 + elif segment_p[-1] < segment_p_reveresed[-1]: + for i, (r, c) in enumerate(edge_pairs): + directed_velocity_tree[c, r] = max(velocity_tree[r, c], velocity_tree[c, r]) + directed_velocity_tree[r, c] = 0 + else: + for i, (r, c) in enumerate(edge_pairs): + directed_velocity_tree[c, r] = velocity_tree[c, r] + directed_velocity_tree[r, c] = velocity_tree[r, c] + + adata.uns["directed_velocity_tree"] = directed_velocity_tree + main_info_insert_adata_uns("directed_velocity_tree") + return directed_velocity_tree + + +def directed_pg( + adata: AnnData, + basis: str = "umap", + transition_method: str = "pearson", + maxIter: int = 10, + sigma: float = 0.001, + Lambda: Optional[float] = None, + gamma: float = 10, + ncenter: Optional[int] = None, + raw_embedding: bool = True, +) -> AnnData: + """A function that learns a direct principal graph by integrating the transition matrix between and DDRTree. + + Args: + adata: An AnnData object, + basis: The dimension reduction method utilized. Defaults to "umap". + transition_method: The method to calculate the transition matrix and project high dimensional vector to low + dimension. + maxIter: The max iteration numbers. Defaults to 10. + sigma: The bandwidth parameter. Defaults to 0.001. + Lambda: The regularization parameter for inverse graph embedding. Defaults to None. + gamma: The regularization parameter for k-means. Defaults to 10. + ncenter: The number of centers to be considered. If None, number of centers would be calculated automatically. + Defaults to None. + raw_embedding: Whether to project the nodes on the principal graph into the original embedding. Defaults to + True. + + Raises: + Exception: invalid `basis`. + Exception: adata.uns["transition_matrix"] not defined. + + Returns: + An updated AnnData object that is updated with principal_g_transition, X__DDRTree and X_DDRTree_pg keys. + """ + + X = adata.obsm["X_" + basis] if "X_" + basis in adata.obsm.keys() else None + if X is None: + raise Exception("{} is not a key of obsm ({} dimension reduction is not performed yet.).".format(basis, basis)) + + transition_matrix = ( + adata.obsp[transition_method + "_transition_matrix"] + if transition_method + "_transition_matrix" in adata.obsp.keys() + else None + ) + if transition_matrix is None: + raise Exception("transition_matrix is not a key of uns. Please first run cell_velocity.") + + Lambda = 5 * X.shape[1] if Lambda is None else Lambda + ncenter = 250 if cal_ncenter(X.shape[1]) is None else ncenter + + Z, Y, principal_g, cell_membership, W, Q, C, objs = DDRTree( + X, + maxIter=maxIter, + Lambda=Lambda, + sigma=sigma, + gamma=gamma, + ncenter=ncenter, + ) + + X = csr_matrix(principal_g) + Tcsr = minimum_spanning_tree(X) + principal_g = Tcsr.toarray().astype(int) + + # here we can also identify siginificant links using methods related to PAGA + transition_matrix = transition_matrix.toarray() + principal_g_transition = cell_membership.T.dot(transition_matrix).dot(cell_membership) * principal_g + + adata.uns["principal_g_transition"] = principal_g_transition + adata.obsm["X_DDRTree"] = X.T if raw_embedding else Z + cell_membership = csr_matrix(cell_membership) + adata.uns["X_DDRTree_pg"] = cell_membership.dot(X.T) if raw_embedding else Y + + return adata + + def _compute_center_transition_matrix(transition_matrix: Union[csr_matrix, np.ndarray], R: np.ndarray) -> np.ndarray: """Calculate the transition matrix for DDRTree centers. @@ -160,135 +292,3 @@ def _get_all_segments(orders: Union[np.ndarray, List], parents: Union[np.ndarray segments.append(path) return segments - - -def construct_velocity_tree(adata: AnnData, transition_matrix_key: str = "pearson"): - """Integrate pseudotime ordering with velocity to automatically assign the direction of the learned trajectory. - - Args: - adata: The anndata object containing the single-cell data. - transition_matrix_key (str, optional): Key to the transition matrix in the `adata.obsp` object that represents - the transition probabilities between cells. Defaults to "pearson". - - Raises: - KeyError: If the transition matrix or cell order information is not found in the `adata` object. - - Returns: - A directed velocity tree represented as a NumPy array. - """ - if transition_matrix_key + "_transition_matrix" not in adata.obsp.keys(): - raise KeyError("Transition matrix not found in anndata. Please call cell_velocities() before constructing " - "velocity tree") - - if "cell_order" not in adata.uns.keys(): - raise KeyError("Cell order information not found in anndata. Please call order_cells() before constructing " - "velocity tree.") - - main_info("Constructing velocity tree...") - - transition_matrix = adata.obsp[transition_matrix_key + "_transition_matrix"] - R = adata.uns["cell_order"]["R"] - orders = np.argsort(adata.uns["cell_order"]["centers_order"]) - parents = [adata.uns["cell_order"]["centers_parent"][node] for node in orders] - velocity_tree = adata.uns["cell_order"]["centers_minSpanningTree"] - directed_velocity_tree = velocity_tree.copy() - - segments = _get_all_segments(orders, parents) - center_transition_matrix = _compute_center_transition_matrix(transition_matrix, R) - - for segment in segments: - edge_pairs = _get_edges(segment) - edge_pairs_reversed = _get_edges(segment[::-1]) - segment_p = _calculate_segment_probability(center_transition_matrix, edge_pairs) - segment_p_reveresed = _calculate_segment_probability(center_transition_matrix, edge_pairs_reversed) - if segment_p[-1] > segment_p_reveresed[-1]: - for i, (r, c) in enumerate(edge_pairs): - directed_velocity_tree[r, c] = max(velocity_tree[r, c], velocity_tree[c, r]) - directed_velocity_tree[c, r] = 0 - elif segment_p[-1] < segment_p_reveresed[-1]: - for i, (r, c) in enumerate(edge_pairs): - directed_velocity_tree[c, r] = max(velocity_tree[r, c], velocity_tree[c, r]) - directed_velocity_tree[r, c] = 0 - else: - for i, (r, c) in enumerate(edge_pairs): - directed_velocity_tree[c, r] = velocity_tree[c, r] - directed_velocity_tree[r, c] = velocity_tree[r, c] - - adata.uns["directed_velocity_tree"] = directed_velocity_tree - main_info_insert_adata_uns("directed_velocity_tree") - return directed_velocity_tree - - -def directed_pg( - adata: AnnData, - basis: str = "umap", - transition_method: str = "pearson", - maxIter: int = 10, - sigma: float = 0.001, - Lambda: Optional[float] = None, - gamma: float = 10, - ncenter: Optional[int] = None, - raw_embedding: bool = True, -) -> AnnData: - """A function that learns a direct principal graph by integrating the transition matrix between and DDRTree. - - Args: - adata: An AnnData object, - basis: The dimension reduction method utilized. Defaults to "umap". - transition_method: The method to calculate the transition matrix and project high dimensional vector to low - dimension. - maxIter: The max iteration numbers. Defaults to 10. - sigma: The bandwidth parameter. Defaults to 0.001. - Lambda: The regularization parameter for inverse graph embedding. Defaults to None. - gamma: The regularization parameter for k-means. Defaults to 10. - ncenter: The number of centers to be considered. If None, number of centers would be calculated automatically. - Defaults to None. - raw_embedding: Whether to project the nodes on the principal graph into the original embedding. Defaults to - True. - - Raises: - Exception: invalid `basis`. - Exception: adata.uns["transition_matrix"] not defined. - - Returns: - An updated AnnData object that is updated with principal_g_transition, X__DDRTree and X_DDRTree_pg keys. - """ - - X = adata.obsm["X_" + basis] if "X_" + basis in adata.obsm.keys() else None - if X is None: - raise Exception("{} is not a key of obsm ({} dimension reduction is not performed yet.).".format(basis, basis)) - - transition_matrix = ( - adata.obsp[transition_method + "_transition_matrix"] - if transition_method + "_transition_matrix" in adata.obsp.keys() - else None - ) - if transition_matrix is None: - raise Exception("transition_matrix is not a key of uns. Please first run cell_velocity.") - - Lambda = 5 * X.shape[1] if Lambda is None else Lambda - ncenter = 250 if cal_ncenter(X.shape[1]) is None else ncenter - - Z, Y, principal_g, cell_membership, W, Q, C, objs = DDRTree( - X, - maxIter=maxIter, - Lambda=Lambda, - sigma=sigma, - gamma=gamma, - ncenter=ncenter, - ) - - X = csr_matrix(principal_g) - Tcsr = minimum_spanning_tree(X) - principal_g = Tcsr.toarray().astype(int) - - # here we can also identify siginificant links using methods related to PAGA - transition_matrix = transition_matrix.toarray() - principal_g_transition = cell_membership.T.dot(transition_matrix).dot(cell_membership) * principal_g - - adata.uns["principal_g_transition"] = principal_g_transition - adata.obsm["X_DDRTree"] = X.T if raw_embedding else Z - cell_membership = csr_matrix(cell_membership) - adata.uns["X_DDRTree_pg"] = cell_membership.dot(X.T) if raw_embedding else Y - - return adata diff --git a/dynamo/tools/Markov.py b/dynamo/tools/Markov.py index 18001f5de..051b54384 100755 --- a/dynamo/tools/Markov.py +++ b/dynamo/tools/Markov.py @@ -16,433 +16,160 @@ from .utils import append_iterative_neighbor_indices, flatten -def markov_combination(x: np.ndarray, v: np.ndarray, X: np.ndarray) -> Tuple: - """Calculate the Markov combination by solving a 'cvxopt' library quadratic programming (QP) problem, which is - defined as: - minimize (1/2)*x'*P*x + q'*x - subject to G*x <= h +def prepare_velocity_grid_data( + X_emb: np.ndarray, + xy_grid_nums: List, + density: Optional[int] = None, + smooth: Optional[float] = None, + n_neighbors: Optional[int] = None, +) -> Tuple: + """Prepare the grid of data used to calculate the velocity embedding on grid. Args: - x: The cell data matrix. - v: The velocity data matrix. - X: The neighbors data matrix. + X_emb: The embedded data matrix. + xy_grid_nums: The number of grid points along each dimension for the velocity grid. + density: The density of grid points relative to the number of points in each dimension. + smooth: The smoothing factor for grid points relative to the range of each dimension. + n_neighbors: The number of neighbors to consider when estimating grid velocities. Returns: - A tuple containing the results of QP problem. + A tuple containing: + The grid points for the velocity. + The estimated probability mass for each grid point based on grid velocities. + The indices of neighbors for each grid point. + The weights corresponding to the neighbors for each grid point. """ - from cvxopt import matrix, solvers + n_obs, n_dim = X_emb.shape + density = 1 if density is None else density + smooth = 0.5 if smooth is None else smooth - n = X.shape[0] - R = matrix(X - x).T - H = R.T * R - f = matrix(v).T * R - G = np.vstack((-np.eye(n), np.ones(n))) - h = np.zeros(n + 1) - h[-1] = 1 - p = solvers.qp(H, -f.T, G=matrix(G), h=matrix(h))["x"] - u = R * p - return p, u + grs, scale = [], 0 + for dim_i in range(n_dim): + m, M = np.min(X_emb[:, dim_i]), np.max(X_emb[:, dim_i]) + m = m - 0.01 * np.abs(M - m) + M = M + 0.01 * np.abs(M - m) + gr = np.linspace(m, M, xy_grid_nums[dim_i] * density) + scale += gr[1] - gr[0] + grs.append(gr) + scale = scale / n_dim * smooth -def compute_markov_trans_prob( - x: np.ndarray, v: np.ndarray, X: np.ndarray, s: Optional[np.ndarray] = None, cont_time: bool = False -) -> np.ndarray: - """Calculate the Markov transition probabilities by solving a 'cvxopt' library quadratic programming (QP) problem, - which is defined as: - minimize (1/2)*x'*P*x + q'*x - subject to G*x <= h + meshes_tuple = np.meshgrid(*grs) + X_grid = np.vstack([i.flat for i in meshes_tuple]).T - Args: - x: The cell data matrix. - v: The velocity data matrix. - X: The neighbors data matrix. - s: Extra constraints added to the `q` in QP problem. - cont_time: Whether is continuous-time or not. + # estimate grid velocities + if n_neighbors is None: + n_neighbors = np.max([10, int(n_obs / 50)]) - Returns: - An array containing the optimal Markov transition probabilities computed by QP problem. - """ - from cvxopt import matrix, solvers + neighs, dists = k_nearest_neighbors( + X_emb, + query_X=X_grid, + k=n_neighbors - 1, + exclude_self=False, + pynn_rand_state=19491001, + ) - n = X.shape[0] - R = X - x - # normalize R, v, and s - Rn = np.array(R, copy=True) - vn = np.array(v, copy=True) - scale = np.abs(np.max(R, 0) - np.min(R, 0)) - Rn = Rn / scale - vn = vn / scale - if s is not None: - sn = np.array(s, copy=True) - sn = sn / scale - A = np.hstack((Rn, 0.5 * Rn * Rn)) - b = np.hstack((vn, 0.5 * sn * sn)) - else: - A = Rn - b = vn + weight = norm.pdf(x=dists, scale=scale) + p_mass = weight.sum(1) - H = A.dot(A.T) - f = b.dot(A.T) - if cont_time: - G = -np.eye(n) - h = np.zeros(n) - else: - G = np.vstack((-np.eye(n), np.ones(n))) - h = np.zeros(n + 1) - h[-1] = 1 - p = solvers.qp(matrix(H), matrix(-f), G=matrix(G), h=matrix(h))["x"] - p = np.array(p).flatten() - return p + return X_grid, p_mass, neighs, weight -@jit(nopython=True) -def compute_kernel_trans_prob( - x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float], cont_time: bool = False -) -> np.ndarray: - """Calculate the transition probabilities. +def grid_velocity_filter( + V_emb: np.ndarray, + neighs: np.ndarray, + p_mass: np.ndarray, + X_grid: np.ndarray, + V_grid: np.ndarray, + min_mass: Optional[float] = None, + autoscale: bool = False, + adjust_for_stream: bool = True, + V_threshold: Optional[float]=None, +) -> Tuple: + """Filter the grid velocities, adjusting for streamlines if needed. Args: - x: The cell data matrix representing current state. - v: The velocity data matrix. - X: An array of data points representing the neighbors. - inv_s: The inverse of the diffusion matrix or a scalar value. - cont_time: Whether to use continuous-time kernel computation. + V_emb: The velocity matrix which represents the velocity vectors associated with each data point in the embedding. + neighs: The indices of neighbors for each grid point. + p_mass: The estimated probability mass for each grid point based on grid velocities. + X_grid: The grid points for the velocity. + V_grid: The estimated grid velocities. + min_mass: The minimum probability mass threshold to filter grid points based on p_mass. + autoscale: Whether to autoscale the grid velocities based on the grid points and their velocities. + adjust_for_stream: Whether to adjust the grid velocities to show streamlines. + V_threshold: The velocity threshold to filter grid points based on velocity magnitude. Returns: - The computed transition probabilities for each state in the Markov chain. + The filtered grid points and the filtered estimated grid velocities. """ - n = X.shape[0] - p = np.zeros(n) - for i in range(n): - d = X[i] - x - p[i] = np.exp(-0.25 * (d - v) @ inv_s @ (d - v).T) - p /= np.sum(p) - return p + if adjust_for_stream: + X_grid = np.stack([np.unique(X_grid[:, 0]), np.unique(X_grid[:, 1])]) + ns = int(np.sqrt(V_grid.shape[0])) + V_grid = V_grid.T.reshape(2, ns, ns) + mass = np.sqrt((V_grid**2).sum(0)) + if V_threshold is not None: + V_grid[0][mass.reshape(V_grid[0].shape) < V_threshold] = np.nan + else: + if min_mass is None: + min_mass = 1e-5 + min_mass = np.clip(min_mass, None, np.max(mass) * 0.9) + cutoff = mass.reshape(V_grid[0].shape) < min_mass -# @jit(nopython=True) -def compute_drift_kernel(x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float]) -> np.ndarray: - """Compute the kernal representing the drift based on input data and parameters. + if neighs is not None: + length = np.sum(np.mean(np.abs(V_emb[neighs]), axis=1), axis=1).T.reshape(ns, ns) + cutoff |= length < np.percentile(length, 5) - Args: - x: The cell data matrix representing current state. - v: The velocity data matrix. - X: An array of data points representing the neighbors. - inv_s: The inverse of the diffusion matrix or a scalar value. + V_grid[0][cutoff] = np.nan + else: + from ..plot.utils import quiver_autoscaler - Returns: - The computed drift kernel values for each state in the Markov chain. - """ - n = X.shape[0] - k = np.zeros(n) - for i in range(n): - d = X[i] - x - if np.isscalar(inv_s): - k[i] = np.exp(-0.25 * inv_s * (d - v).dot(d - v)) + if p_mass is None: + p_mass = np.sqrt((V_grid**2).sum(1)) + if min_mass is None: + min_mass = np.clip(np.percentile(p_mass, 5), 1e-5, None) else: - k[i] = np.exp(-0.25 * (d - v) @ inv_s @ (d - v).T) - return k - + if min_mass is None: + min_mass = np.clip(np.percentile(p_mass, 99) / 100, 1e-5, None) + X_grid, V_grid = X_grid[p_mass > min_mass], V_grid[p_mass > min_mass] -"""def compute_drift_local_kernel(x, v, X, inv_s): - n = X.shape[0] - k = np.zeros(n) - # compute tau - D = X - x - dists = np.zeros(n) - vds = np.zeros(n) - for (i, d) in enumerate(D): - dists[i] = np.linalg.norm(d) - if dists[i] > 0: - vds[i] = v.dot(d) / dists[i] - i_dir = np.logical_and(vds >= np.quantile(vds, 0.7), vds > 0) - tau = np.mean(dists[i_dir] / vds[i_dir]) - if np.isnan(tau): tau = 1 - if tau > 1e2: tau = 1e2 + if autoscale: + V_grid /= 3 * quiver_autoscaler(X_grid, V_grid) - tau_v = tau * v - tau_invs = (1 / (tau * np.linalg.norm(v))) * inv_s - for i in range(n): - d = D[i] - k[i] = np.exp(-0.25 * (d-tau_v) @ tau_invs @ (d-tau_v).T) - return k, tau_invs""" + return X_grid, V_grid -# @jit(nopython=True) -def compute_drift_local_kernel(x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float]) -> np.ndarray: - """Compute a local kernel representing the drift based on input data and parameters. +def velocity_on_grid( + X_emb: np.ndarray, + V_emb: np.ndarray, + xy_grid_nums: List, + density: Optional[int] = None, + smooth: Optional[float] = None, + n_neighbors: Optional[int] = None, + min_mass: Optional[float] = None, + autoscale: bool = False, + adjust_for_stream: bool = True, + V_threshold: Optional[float] = None, + cut_off_velocity: bool = True, +) -> Tuple: + """Function to calculate the velocity vectors on a grid for grid vector field quiver plot and streamplot, adapted + from scVelo. Args: - x: The cell data matrix representing current state. - v: The velocity data matrix. - X: An array of data points representing the neighbors. - inv_s: The inverse of the diffusion matrix or a scalar value. + X_emb: The low-dimensional embedding which represents the coordinates of the data points in the embedding space. + V_emb: The velocity matrix which represents the velocity vectors associated with each data point in the embedding. + xy_grid_nums: The number of grid points along each dimension of the embedding space. + density: The number of density grid points for each dimension. + smooth: The smoothing parameter for grid points along each dimension. + n_neighbors: The number of neighbors to consider for estimating grid velocities. + min_mass: The minimum probability mass threshold to filter grid points based on p_mass. + autoscale: Whether to autoscale the grid velocities based on the grid points and their velocities. + adjust_for_stream: Whether to adjust the grid velocities to show streamlines. + V_threshold: The velocity threshold to filter grid points based on velocity magnitude. + cut_off_velocity: Whether to cut off the grid velocities or return the entire grid. Returns: - The computed drift kernel values. - """ - n = X.shape[0] - k = np.zeros(n) - # compute tau - D = X - x - dists = np.zeros(n) - vds = np.zeros(n) - for (i, d) in enumerate(D): - dists[i] = np.linalg.norm(d) - if dists[i] > 0: - vds[i] = v.dot(d) / dists[i] - i_dir = np.logical_and(vds >= np.quantile(vds, 0.7), vds > 0) - if np.any(i_dir): - tau = np.mean(dists[i_dir] / vds[i_dir]) - if tau > 1e2: - tau = 1e2 - tau_v = tau * v - tau_invs = (1 / (tau * v.dot(v))) * inv_s - else: - tau_v = 0 - tau_invs = (1 / (1e2 * v.dot(v))) * inv_s - for i in range(n): - d = D[i] - if np.isscalar(tau_invs): - k[i] = np.exp(-0.25 * tau_invs * (d - tau_v).dot(d - tau_v)) - else: - k[i] = np.exp(-0.25 * (d - tau_v) @ tau_invs @ (d - tau_v).T) - return k - - -@jit(nopython=True) -def compute_density_kernel(x: np.ndarray, X: np.ndarray, inv_eps: float) -> np.ndarray: - """Compute the density kernel values. - - Args: - x: The cell data matrix representing current state. - X: An array of data points representing the neighbors. - inv_eps: The inverse of the epsilon. - - Returns: - The computed density kernel values for each state. - """ - n = X.shape[0] - k = np.zeros(n) - for i in range(n): - d = X[i] - x - k[i] = np.exp(-0.25 * inv_eps * d.dot(d)) - return k - - -@jit(nopython=True) -def makeTransitionMatrix(Qnn: np.ndarray, I_vec: np.ndarray, tol: float = 0.0) -> np.ndarray: # Qnn, I, tol=0.0 - """Create the transition matrix based on the transition rate matrix `Qnn` and the indexing vector `I_vec`. - - Args: - Qnn: The matrix which represents the transition rates between different states. - I_vec: The indexing vector to map the rows to the appropriate positions in the transition matrix. - tol: A numerical tolerance value to consider rate matrix elements as zero. - - Returns: - The computed transition matrix based on `Qnn` and `I_vec`. - """ - n = Qnn.shape[0] - M = np.zeros((n, n)) - - for i in range(n): - q = Qnn[i] - q[q < tol] = 0 - M[I_vec[i], i] = q - M[i, i] = 1 - np.sum(q) - return M - - -@jit(nopython=True) -def compute_tau(X: np.ndarray, V: np.ndarray, k: int = 100, nbr_idx: Optional[np.ndarray] = None) -> Tuple: - """Compute the tau values for each state in `X` based on the local density and velocity magnitudes. - - Args: - X: The data matrix which represents the states of the system. - V: The velocity matrix which represents the velocity vectors associated with each state in `X`. - k: The number of neighbors to consider when estimating local density. Default is 100. - nbr_idx: The indices of neighbors for each state in `X`. - - Returns: - The computed tau values representing the timescale of transitions for each state in `X`. The computed velocity - magnitudes for each state in `X`. - """ - - if nbr_idx is None: - _, dists = k_nearest_neighbors( - X, - k=k - 1, - exclude_self=False, - pynn_rand_state=19491001, - n_jobs=-1, - ) - else: - dists = np.zeros(nbr_idx.shape) - for i in range(nbr_idx.shape[0]): - for j in range(nbr_idx.shape[1]): - x = X[i] - y = X[nbr_idx[i, j]] - dists[i, j] = np.sqrt((x - y).dot(x - y)) - d = np.mean(dists[:, 1:], 1) - v = np.linalg.norm(V, axis=1) - tau = d / v - return tau, v - - -def prepare_velocity_grid_data( - X_emb: np.ndarray, - xy_grid_nums: List, - density: Optional[int] = None, - smooth: Optional[float] = None, - n_neighbors: Optional[int] = None, -) -> Tuple: - """Prepare the grid of data used to calculate the velocity embedding on grid. - - Args: - X_emb: The embedded data matrix. - xy_grid_nums: The number of grid points along each dimension for the velocity grid. - density: The density of grid points relative to the number of points in each dimension. - smooth: The smoothing factor for grid points relative to the range of each dimension. - n_neighbors: The number of neighbors to consider when estimating grid velocities. - - Returns: - A tuple containing: - The grid points for the velocity. - The estimated probability mass for each grid point based on grid velocities. - The indices of neighbors for each grid point. - The weights corresponding to the neighbors for each grid point. - """ - n_obs, n_dim = X_emb.shape - density = 1 if density is None else density - smooth = 0.5 if smooth is None else smooth - - grs, scale = [], 0 - for dim_i in range(n_dim): - m, M = np.min(X_emb[:, dim_i]), np.max(X_emb[:, dim_i]) - m = m - 0.01 * np.abs(M - m) - M = M + 0.01 * np.abs(M - m) - gr = np.linspace(m, M, xy_grid_nums[dim_i] * density) - scale += gr[1] - gr[0] - grs.append(gr) - - scale = scale / n_dim * smooth - - meshes_tuple = np.meshgrid(*grs) - X_grid = np.vstack([i.flat for i in meshes_tuple]).T - - # estimate grid velocities - if n_neighbors is None: - n_neighbors = np.max([10, int(n_obs / 50)]) - - neighs, dists = k_nearest_neighbors( - X_emb, - query_X=X_grid, - k=n_neighbors - 1, - exclude_self=False, - pynn_rand_state=19491001, - ) - - weight = norm.pdf(x=dists, scale=scale) - p_mass = weight.sum(1) - - return X_grid, p_mass, neighs, weight - - -def grid_velocity_filter( - V_emb: np.ndarray, - neighs: np.ndarray, - p_mass: np.ndarray, - X_grid: np.ndarray, - V_grid: np.ndarray, - min_mass: Optional[float] = None, - autoscale: bool = False, - adjust_for_stream: bool = True, - V_threshold: Optional[float]=None, -) -> Tuple: - """Filter the grid velocities, adjusting for streamlines if needed. - - Args: - V_emb: The velocity matrix which represents the velocity vectors associated with each data point in the embedding. - neighs: The indices of neighbors for each grid point. - p_mass: The estimated probability mass for each grid point based on grid velocities. - X_grid: The grid points for the velocity. - V_grid: The estimated grid velocities. - min_mass: The minimum probability mass threshold to filter grid points based on p_mass. - autoscale: Whether to autoscale the grid velocities based on the grid points and their velocities. - adjust_for_stream: Whether to adjust the grid velocities to show streamlines. - V_threshold: The velocity threshold to filter grid points based on velocity magnitude. - - Returns: - The filtered grid points and the filtered estimated grid velocities. - """ - if adjust_for_stream: - X_grid = np.stack([np.unique(X_grid[:, 0]), np.unique(X_grid[:, 1])]) - ns = int(np.sqrt(V_grid.shape[0])) - V_grid = V_grid.T.reshape(2, ns, ns) - - mass = np.sqrt((V_grid**2).sum(0)) - if V_threshold is not None: - V_grid[0][mass.reshape(V_grid[0].shape) < V_threshold] = np.nan - else: - if min_mass is None: - min_mass = 1e-5 - min_mass = np.clip(min_mass, None, np.max(mass) * 0.9) - cutoff = mass.reshape(V_grid[0].shape) < min_mass - - if neighs is not None: - length = np.sum(np.mean(np.abs(V_emb[neighs]), axis=1), axis=1).T.reshape(ns, ns) - cutoff |= length < np.percentile(length, 5) - - V_grid[0][cutoff] = np.nan - else: - from ..plot.utils import quiver_autoscaler - - if p_mass is None: - p_mass = np.sqrt((V_grid**2).sum(1)) - if min_mass is None: - min_mass = np.clip(np.percentile(p_mass, 5), 1e-5, None) - else: - if min_mass is None: - min_mass = np.clip(np.percentile(p_mass, 99) / 100, 1e-5, None) - X_grid, V_grid = X_grid[p_mass > min_mass], V_grid[p_mass > min_mass] - - if autoscale: - V_grid /= 3 * quiver_autoscaler(X_grid, V_grid) - - return X_grid, V_grid - - -def velocity_on_grid( - X_emb: np.ndarray, - V_emb: np.ndarray, - xy_grid_nums: List, - density: Optional[int] = None, - smooth: Optional[float] = None, - n_neighbors: Optional[int] = None, - min_mass: Optional[float] = None, - autoscale: bool = False, - adjust_for_stream: bool = True, - V_threshold: Optional[float] = None, - cut_off_velocity: bool = True, -) -> Tuple: - """Function to calculate the velocity vectors on a grid for grid vector field quiver plot and streamplot, adapted - from scVelo. - - Args: - X_emb: The low-dimensional embedding which represents the coordinates of the data points in the embedding space. - V_emb: The velocity matrix which represents the velocity vectors associated with each data point in the embedding. - xy_grid_nums: The number of grid points along each dimension of the embedding space. - density: The number of density grid points for each dimension. - smooth: The smoothing parameter for grid points along each dimension. - n_neighbors: The number of neighbors to consider for estimating grid velocities. - min_mass: The minimum probability mass threshold to filter grid points based on p_mass. - autoscale: Whether to autoscale the grid velocities based on the grid points and their velocities. - adjust_for_stream: Whether to adjust the grid velocities to show streamlines. - V_threshold: The velocity threshold to filter grid points based on velocity magnitude. - cut_off_velocity: Whether to cut off the grid velocities or return the entire grid. - - Returns: - A tuple containing the grid points, the filtered estimated grid velocities, the diffusion matrix D of shape. + A tuple containing the grid points, the filtered estimated grid velocities, the diffusion matrix D of shape. """ from ..vectorfield.stochastic_process import diffusionMatrix2D @@ -471,165 +198,21 @@ def velocity_on_grid( V_grid, min_mass=min_mass, autoscale=autoscale, - adjust_for_stream=adjust_for_stream, - V_threshold=V_threshold, - ) - else: - X_grid, V_grid = ( - np.array([np.unique(X_grid[:, 0]), np.unique(X_grid[:, 1])]), - np.array( - [ - V_grid[:, 0].reshape(xy_grid_nums), - V_grid[:, 1].reshape(xy_grid_nums), - ] - ), - ) - - return X_grid, V_grid, D - - -# we might need a separate module/file for discrete vector field and markovian methods in the future -def graphize_velocity( - V: np.ndarray, - X: np.ndarray, - nbrs_idx: Optional[list] = None, - k: int = 30, - normalize_v: bool = False, - E_func: Optional[Union[Callable, str]] = None -) -> Tuple: - """The function generates a graph based on the velocity data. The flow from i- to j-th - node is returned as the edge matrix E[i, j], and E[i, j] = -E[j, i]. - - Args: - V: The velocities for all cells. - X: The coordinates for all cells. - nbrs_idx: A list of neighbor indices for each cell. If None a KNN will be performed instead. - k: The number of neighbors for the KNN search. - normalize_v: Whether to normalize the velocity vectors. - E_func: A variance stabilizing function for reducing the variance of the flows. - If a string is passed, there are two options: - 'sqrt': the `numpy.sqrt` square root function; - 'exp': the `numpy.exp` exponential function. - - Returns: - The edge matrix and the neighbor indices. - """ - n, d = X.shape - - nbrs = None - if nbrs_idx is None: - nbrs_idx, _ = k_nearest_neighbors( - X, - k=k, - exclude_self=False, - pynn_rand_state=19491001, - ) - - if type(E_func) is str: - if E_func == "sqrt": - E_func = np.sqrt - elif E_func == "exp": - E_func = np.exp - else: - raise NotImplementedError("The specified edge function is not implemented.") - - # E = sp.csr_matrix((n, n)) # Making E a csr_matrix will slow down this process. Try lil_matrix maybe? - E = np.zeros((n, n)) - for i in range(n): - x = flatten(X[i]) - idx = nbrs_idx[i] - if len(idx) > 0 and idx[0] == i: # excluding the node itself from the neighbors - idx = idx[1:] - vi = flatten(V[i]) - if normalize_v: - vi_norm = np.linalg.norm(vi) - if vi_norm > 0: - vi /= vi_norm - - # normalized differences - U = X[idx] - x - U_norm = np.linalg.norm(U, axis=1) - U_norm[U_norm == 0] = 1 - U /= U_norm[:, None] - - for jj, j in enumerate(idx): - vj = flatten(V[j]) - if normalize_v: - vj_norm = np.linalg.norm(vj) - if vj_norm > 0: - vj /= vj_norm - u = flatten(U[jj]) - v = np.mean((vi.dot(u), vj.dot(u))) - - if E_func is not None: - v = np.sign(v) * E_func(np.abs(v)) - E[i, j] = v - E[j, i] = -v - - return E, nbrs_idx - - -def calc_Laplacian(E: np.ndarray, convention: str = "graph") -> np.ndarray: - """Calculate the Laplacian matrix of a given matrix of edge weights. - - Args: - E: The matrix of edge weights which represents the weights of edges in a graph. - convention: The convention used to compute the Laplacian matrix. - If "graph", the Laplacian matrix will be calculated as the diagonal matrix of node degrees minus the adjacency matrix. - If "diffusion", the Laplacian matrix will be calculated as the negative of the graph Laplacian. - Default is "graph". - - Returns: - The Laplacian matrix. - """ - A = np.abs(np.sign(E)) - L = np.diag(np.sum(A, 0)) - A - - if convention == "diffusion": - L = -L - - return L - - -def fp_operator(E: np.ndarray, D: Union[int, float]) -> np.ndarray: - """Calculate the Fokker-Planck operator based on the given matrix of edge weights (E) and diffusion coefficient (D). - - Args: - E: The matrix of edge weights. - D: The diffusion coefficient used in the Fokker-Planck operator. - - Returns: - The Fokker-Planck operator matrix. - """ - # drift - Mu = E.T.copy() - Mu[Mu < 0] = 0 - Mu = np.diag(np.sum(Mu, 0)) - Mu - # diffusion - L = calc_Laplacian(E, convention="diffusion") - - return -Mu + D * L - - -def divergence(E: np.ndarray, tol: float = 1e-5) -> np.ndarray: - """Calculate the divergence for each node in a given matrix of edge weights. - - Args: - E: The matrix of edge weights. - tol: The tolerance value. Edge weights below this value will be treated as zero. - - Returns: - The divergence values for each node. - """ - n = E.shape[0] - div = np.zeros(n) - # optimize for sparse matrices later... - for i in range(n): - for j in range(i + 1, n): - if np.abs(E[i, j]) > tol: - div[i] += E[i, j] - E[j, i] + adjust_for_stream=adjust_for_stream, + V_threshold=V_threshold, + ) + else: + X_grid, V_grid = ( + np.array([np.unique(X_grid[:, 0]), np.unique(X_grid[:, 1])]), + np.array( + [ + V_grid[:, 0].reshape(xy_grid_nums), + V_grid[:, 1].reshape(xy_grid_nums), + ] + ), + ) - return div + return X_grid, V_grid, D class MarkovChain: @@ -1019,565 +602,982 @@ def compute_theta(self, p_st: Optional[np.ndarray] = None) -> sp.csr_matrix: """Compute the matrix 'Theta' for the Markov chain. Args: - p_st: The stationary distribution of the Markov chain. + p_st: The stationary distribution of the Markov chain. + + Returns: + The computed matrix 'Theta' as a sparse matrix. + """ + p_st = self.compute_stationary_distribution() if p_st is None else p_st + Pi = sp.csr_matrix(np.diag(np.sqrt(p_st))) + Pi_right = sp.csc_matrix(np.diag(np.sqrt(p_st))) + # Pi_inv = sp.csr_matrix(np.linalg.pinv(Pi)) + # Pi_inv_right = sp.csc_matrix(np.linalg.pinv(Pi)) + Pi_inv = sp.csr_matrix(np.diag(np.sqrt(1 / p_st))) + Pi_inv_right = sp.csc_matrix(np.diag(np.sqrt(1 / p_st))) + Theta = 0.5 * (Pi @ self.P @ Pi_inv_right + Pi_inv @ self.P.T @ Pi_right) + + return Theta + + +class DiscreteTimeMarkovChain(MarkovChain): + """DiscreteTimeMarkovChain class represents a discrete-time Markov chain.""" + def __init__(self, P: Optional[np.ndarray] = None, eignum: Optional[int] = None, sumto: int = 1, **kwargs): + """Initialize the DiscreteTimeMarkovChain instance. + + Args: + P: The transition matrix of the Markov chain. + eignum: Number of eigenvalues/eigenvectors to compute. + sumto: The value that each column of the transition matrix should sum to. + **kwargs: Additional keyword arguments to be passed to the base class MarkovChain's constructor. + + Returns: + An instance of DiscreteTimeMarkovChain. + """ + super().__init__(P, eignum=eignum, sumto=sumto, **kwargs) + # self.Kd = None + + """def fit(self, X, V, k, s=None, method="qp", eps=None, tol=1e-4): # pass index + # the parameter k will be replaced by a connectivity matrix in the future. + self.__reset__() + # knn clustering + if X.shape[0] > 200000 and X.shape[1] > 2: + from pynndescent import NNDescent + + nbrs = NNDescent( + X, + metric="euclidean", + n_neighbors=k, + n_jobs=-1, + random_state=19491001, + ) + Idx, _ = nbrs.query(X, k=k) + else: + alg = "ball_tree" if X.shape[1] > 10 else "kd_tree" + nbrs = NearestNeighbors(n_neighbors=k, algorithm=alg, n_jobs=-1).fit(X) + _, Idx = nbrs.kneighbors(X) + # compute transition prob. + n = X.shape[0] + self.P = np.zeros((n, n)) + if method == "kernel": + inv_s = np.linalg.inv(s) + # compute density kernel + if eps is not None: + self.Kd = np.zeros((n, n)) + inv_eps = 1 / eps + for i in range(n): + self.Kd[i, Idx[i]] = compute_density_kernel(X[i], X[Idx[i]], inv_eps) + D = np.sum(self.Kd, 0) + for i in range(n): + y = X[i] + v = V[i] + if method == "qp": + Y = X[Idx[i, 1:]] + p = compute_markov_trans_prob(y, v, Y, s) + p[p <= tol] = 0 # tolerance check + self.P[Idx[i, 1:], i] = p + self.P[i, i] = 1 - np.sum(p) + else: + Y = X[Idx[i]] + # p = compute_kernel_trans_prob(y, v, Y, inv_s) + k = compute_drift_kernel(y, v, Y, inv_s) + if eps is not None: + k /= D[Idx[i]] + p = k / np.sum(k) + p[p <= tol] = 0 # tolerance check + p = p / np.sum(p) + self.P[Idx[i], i] = p""" + + def propagate_P(self, num_prop: int) -> np.ndarray: + """Propagate the transition matrix 'P' for a given number of steps. + + Args: + num_prop: The number of propagation steps. + + Returns: + The propagated transition matrix after 'num_prop' steps. + """ + ret = np.array(self.P, copy=True) + for _ in range(num_prop - 1): + ret = self.P @ ret + return ret + + def compute_drift(self, X: np.ndarray, num_prop: int = 1) -> np.ndarray: + """Compute the drift for each state in the Markov chain. + + Args: + X: The data matrix which represents the states of the Markov chain. + num_prop: The number of propagation steps used for drift computation. Default is 1. + + Returns: + The computed drift values for each state in the Markov chain. + """ + n = self.get_num_states() + V = np.zeros_like(X) + P = self.propagate_P(num_prop) + for i in range(n): + V[i] = (X - X[i]).T.dot(P[:, i]) + return V + + def compute_density_corrected_drift( + self, X: np.ndarray, k: Optional[int] = None, normalize_vector: bool = False + ) -> np.ndarray: + """Compute density-corrected drift for each state in the Markov chain. + + Args: + X: The data matrix whicj represents the states of the Markov chain. + k: The number of nearest neighbors used for computing the mean of kernel probabilities. + normalize_vector: whether to normalize the drift vector for each state. + + Returns: + The computed density-corrected drift values for each state in the Markov chain. + """ + n = self.get_num_states() + if k is None: + k = n + V = np.zeros_like(X) + for i in range(n): + d = X - X[i] # no self.nbrs_idx[i] is here.... may be wrong? + if normalize_vector: + d /= np.linalg.norm(d) + V[i] = d.T.dot(self.P[:, i] - 1 / k) + return V + + def solve_distribution(self, p0: np.ndarray, n: int, method: str = "naive") -> np.ndarray: + """Solve the distribution for the Markov chain. + + Args: + p0: The initial probability distribution vector. + n: The number of steps for distribution propagation. + method: The method used for solving the distribution. + + Returns: + The computed probability distribution vector after 'n' steps. + """ + if method == "naive": + p = p0 + for _ in range(n): + p = self.P.dot(p) + else: + if self.D is None: + self.eigsys() + p = np.real(self.W @ np.diag(self.D**n) @ np.linalg.inv(self.W)).dot(p0) + return p + + def compute_stationary_distribution(self, method: str = "eig") -> np.ndarray: + """Compute the stationary distribution of the Markov chain. + + Args: + method: The method used for computing the stationary distribution. + + Returns: + The computed stationary distribution as a probability vector. + """ + if method == "solve": + p = np.real(null_space(self.P - np.eye(self.P.shape[0])[:, 0])[:, 0].flatten()) + else: + if self.W is None: + self.eigsys() + p = np.abs(np.real(self.W[:, 0])) + p = p / np.sum(p) + return p + + def lump(self, labels: np.ndarray, M_weight: Optional[np.ndarray] = None) -> np.ndarray: + """Markov chain lumping based on: + K. Hoffmanna and P. Salamon, Bounding the lumping error in Markov chain dynamics, Appl Math Lett, (2009) + + Args: + labels: The lumping labels. + M_weight: The weighting matrix. If None, it is computed using the stationary distribution. + + Returns: + The lumped transition matrix after the lumping operation. + """ + k = len(labels) + M_part = np.zeros((k, self.get_num_states())) + + for i in range(len(labels)): + M_part[labels[i], i] = 1 + + n_node = self.get_num_states() + if M_weight is None: + p_st = self.compute_stationary_distribution() + M_weight = np.multiply(M_part, p_st) + M_weight = np.divide(M_weight.T, M_weight @ np.ones(n_node)) + P_lumped = M_part @ self.P @ M_weight + + return P_lumped + + def naive_lump(self, x: np.ndarray, grp: np.ndarray) -> np.ndarray: + """Perform naive Markov chain lumping based on given data and group labels. + + Args: + x: The data matrix. + grp: The group labels. + + Returns: + The lumped transition matrix after the lumping operation. + """ + k = len(np.unique(grp)) + y = np.zeros((k, k)) + for i in range(len(y)): + for j in range(len(y)): + y[i, j] = x[grp == i, :][:, grp == j].mean() + + return y + + def diffusion_map_embedding(self, n_dims: int = 2, t: int = 1) -> np.ndarray: + """Perform diffusion map embedding for the Markov chain. + + Args: + n_dims: The number of dimensions for the diffusion map embedding. + t: The diffusion time parameter used in the embedding. + + Returns: + The diffusion map embedding of the Markov chain as a matrix of shape (n_states, n_dims). + """ + if self.W is None: + self.eigsys() + + ind = np.arange(1, n_dims + 1) + Y = np.real(self.D[ind] ** t) * np.real(self.U[:, ind]) + return Y + + def simulate_random_walk(self, init_idx: int, num_steps: int) -> np.ndarray: + """Simulate a random walk on the Markov chain from a given initial state. + + Args: + init_idx: The index of the initial state for the random walk. + num_steps: The number of steps for the random walk. Returns: - The computed matrix 'Theta' as a sparse matrix. + The sequence of state indices resulting from the random walk. """ - p_st = self.compute_stationary_distribution() if p_st is None else p_st - Pi = sp.csr_matrix(np.diag(np.sqrt(p_st))) - Pi_right = sp.csc_matrix(np.diag(np.sqrt(p_st))) - # Pi_inv = sp.csr_matrix(np.linalg.pinv(Pi)) - # Pi_inv_right = sp.csc_matrix(np.linalg.pinv(Pi)) - Pi_inv = sp.csr_matrix(np.diag(np.sqrt(1 / p_st))) - Pi_inv_right = sp.csc_matrix(np.diag(np.sqrt(1 / p_st))) - Theta = 0.5 * (Pi @ self.P @ Pi_inv_right + Pi_inv @ self.P.T @ Pi_right) + P = self.P.copy() - return Theta + seq = np.ones(num_steps + 1, dtype=int) * -1 + seq[0] = init_idx + for i in range(1, num_steps + 1): + cur_state = seq[i - 1] + r = np.random.rand() + seq[i] = np.cumsum(P[:, cur_state]).searchsorted(r) + return seq -class DiscreteTimeMarkovChain(MarkovChain): - """DiscreteTimeMarkovChain class represents a discrete-time Markov chain.""" - def __init__(self, P: Optional[np.ndarray] = None, eignum: Optional[int] = None, sumto: int = 1, **kwargs): - """Initialize the DiscreteTimeMarkovChain instance. + +class ContinuousTimeMarkovChain(MarkovChain): + """ContinuousTimeMarkovChain class represents a continuous-time Markov chain.""" + def __init__(self, P: Optional[np.ndarray] = None, eignum: Optional[int] = None, **kwargs): + """Initialize the ContinuousTimeMarkovChain instance. Args: P: The transition matrix of the Markov chain. eignum: Number of eigenvalues/eigenvectors to compute. - sumto: The value that each column of the transition matrix should sum to. **kwargs: Additional keyword arguments to be passed to the base class MarkovChain's constructor. Returns: - An instance of DiscreteTimeMarkovChain. + An instance of ContinuousTimeMarkovChain. """ - super().__init__(P, eignum=eignum, sumto=sumto, **kwargs) - # self.Kd = None - - """def fit(self, X, V, k, s=None, method="qp", eps=None, tol=1e-4): # pass index - # the parameter k will be replaced by a connectivity matrix in the future. - self.__reset__() - # knn clustering - if X.shape[0] > 200000 and X.shape[1] > 2: - from pynndescent import NNDescent - - nbrs = NNDescent( - X, - metric="euclidean", - n_neighbors=k, - n_jobs=-1, - random_state=19491001, - ) - Idx, _ = nbrs.query(X, k=k) - else: - alg = "ball_tree" if X.shape[1] > 10 else "kd_tree" - nbrs = NearestNeighbors(n_neighbors=k, algorithm=alg, n_jobs=-1).fit(X) - _, Idx = nbrs.kneighbors(X) - # compute transition prob. - n = X.shape[0] - self.P = np.zeros((n, n)) - if method == "kernel": - inv_s = np.linalg.inv(s) - # compute density kernel - if eps is not None: - self.Kd = np.zeros((n, n)) - inv_eps = 1 / eps - for i in range(n): - self.Kd[i, Idx[i]] = compute_density_kernel(X[i], X[Idx[i]], inv_eps) - D = np.sum(self.Kd, 0) - for i in range(n): - y = X[i] - v = V[i] - if method == "qp": - Y = X[Idx[i, 1:]] - p = compute_markov_trans_prob(y, v, Y, s) - p[p <= tol] = 0 # tolerance check - self.P[Idx[i, 1:], i] = p - self.P[i, i] = 1 - np.sum(p) - else: - Y = X[Idx[i]] - # p = compute_kernel_trans_prob(y, v, Y, inv_s) - k = compute_drift_kernel(y, v, Y, inv_s) - if eps is not None: - k /= D[Idx[i]] - p = k / np.sum(k) - p[p <= tol] = 0 # tolerance check - p = p / np.sum(p) - self.P[Idx[i], i] = p""" + super().__init__(P, eignum=eignum, sumto=0, **kwargs) + self.Q = None # embedded markov chain transition matrix + self.Kd = None # density kernel for density adjustment + self.p_st = None # stationary distribution - def propagate_P(self, num_prop: int) -> np.ndarray: - """Propagate the transition matrix 'P' for a given number of steps. + def check_transition_rate_matrix(self, P: np.ndarray, tol: float = 1e-6) -> np.ndarray: + """Check if the input matrix is a valid transition rate matrix. Args: - num_prop: The number of propagation steps. + P: The transition rate matrix to be checked. + tol: Tolerance threshold for zero row- or column-sum check. Returns: - The propagated transition matrix after 'num_prop' steps. + The checked transition rate matrix. + + Raises: + ValueError: If the input transition rate matrix has non-zero row- and column-sums. """ - ret = np.array(self.P, copy=True) - for _ in range(num_prop - 1): - ret = self.P @ ret - return ret + if np.any(flatten(np.abs(np.sum(P, 0))) <= tol): + return P + elif np.any(flatten(np.abs(np.sum(P, 1))) <= tol): + return P.T + else: + raise ValueError("The input transition rate matrix must have a zero row- or column-sum.") - def compute_drift(self, X: np.ndarray, num_prop: int = 1) -> np.ndarray: - """Compute the drift for each state in the Markov chain. + def compute_drift(self, X: np.ndarray, t: float, n_top: int = 5, normalize_vector: bool = False) -> np.ndarray: + """Compute the drift for each state in the continuous-time Markov chain. Args: - X: The data matrix which represents the states of the Markov chain. - num_prop: The number of propagation steps used for drift computation. Default is 1. + X: The data matrix of shape which represents the states of the Markov chain. + t: The time at which the drift is computed. + n_top: The number of top states to consider for drift computation. + normalize_vector: Whether to normalize the drift vector for each state. Returns: - The computed drift values for each state in the Markov chain. + The computed drift values for each state in the continuous-time Markov chain. """ n = self.get_num_states() V = np.zeros_like(X) - P = self.propagate_P(num_prop) + P = self.compute_transition_matrix(t) + for i in range(n): - V[i] = (X - X[i]).T.dot(P[:, i]) + if n_top is None: + d = (X - X[i]).T + if normalize_vector: + d = d / np.linalg.norm(d, axis=0) + V[i] = d.dot(P[:, i]) + else: + idx = np.argsort(P[:, i])[-n_top:] + d = (X[idx] - X[i]).T + if normalize_vector: + d = d / np.linalg.norm(d, axis=0) + V[i] = d.dot(P[idx, i]) + # q = P[idx, i] / np.sum(P[idx, i]) + # V[i] = d.dot(q) return V def compute_density_corrected_drift( - self, X: np.ndarray, k: Optional[int] = None, normalize_vector: bool = False + self, X: np.ndarray, t: float, k: Optional[int] = None, normalize_vector: bool = False ) -> np.ndarray: - """Compute density-corrected drift for each state in the Markov chain. + """Compute density-corrected drift for each state in the continuous-time Markov chain. Args: - X: The data matrix whicj represents the states of the Markov chain. - k: The number of nearest neighbors used for computing the mean of kernel probabilities. - normalize_vector: whether to normalize the drift vector for each state. + X: The data matrix of shape which represents the states of the Markov chain. + t: The time at which the density-corrected drift is computed. + k: The number of nearest neighbors used for computing the correction term. + normalize_vector: Whether to normalize the drift vector for each state. Returns: - The computed density-corrected drift values for each state in the Markov chain. + The computed density-corrected drift values for each state in the continuous-time Markov chain. """ n = self.get_num_states() - if k is None: - k = n V = np.zeros_like(X) + P = self.compute_transition_matrix(t) for i in range(n): - d = X - X[i] # no self.nbrs_idx[i] is here.... may be wrong? + P[i, i] = 0 + P[:, i] /= np.sum(P[:, i]) + d = X - X[i] if normalize_vector: d /= np.linalg.norm(d) - V[i] = d.T.dot(self.P[:, i] - 1 / k) + correction = 1 / k if k is not None else np.mean(P[:, i]) + V[i] = d.T.dot(P[:, i] - correction) return V - def solve_distribution(self, p0: np.ndarray, n: int, method: str = "naive") -> np.ndarray: - """Solve the distribution for the Markov chain. + def compute_transition_matrix(self, t: float) -> np.ndarray: + """Compute the transition matrix for a given time. + + Args: + t: The time at which the transition matrix is computed. + + Returns: + The computed transition matrix. + """ + + if self.D is None: + self.eigsys() + P = np.real(self.W @ np.diag(np.exp(self.D * t)) @ self.W_inv) + return P + + def compute_embedded_transition_matrix(self) -> np.ndarray: + """Compute the embedded Markov chain transition matrix. + + Returns: + The computed embedded Markov chain transition matrix. + """ + self.Q = np.array(self.P, copy=True) + for i in range(self.Q.shape[1]): + self.Q[i, i] = 0 + self.Q[:, i] /= np.sum(self.Q[:, i]) + return self.Q + + def solve_distribution(self, p0: np.ndarray, t: float) -> np.ndarray: + """Solve the distribution for the continuous-time Markov chain at a given time. Args: p0: The initial probability distribution vector. - n: The number of steps for distribution propagation. - method: The method used for solving the distribution. + t: The time at which the distribution is solved. Returns: - The computed probability distribution vector after 'n' steps. + The computed probability distribution vector at time. """ - if method == "naive": - p = p0 - for _ in range(n): - p = self.P.dot(p) - else: - if self.D is None: - self.eigsys() - p = np.real(self.W @ np.diag(self.D**n) @ np.linalg.inv(self.W)).dot(p0) + P = self.compute_transition_matrix(t) + p = P @ p0 + p = p / np.sum(p) return p - def compute_stationary_distribution(self, method: str = "eig") -> np.ndarray: - """Compute the stationary distribution of the Markov chain. + def compute_stationary_distribution(self, method: str = "eig"): + """Compute the stationary distribution of the continuous-time Markov chain. + + Args: + method: The method used for computing the stationary distribution. + + Returns: + The computed stationary distribution of the continuous-time Markov chain. + """ + if self.p_st is None: + if method == "null": + p = np.abs(np.real(null_space(self.P)[:, 0].flatten())) + p = p / np.sum(p) + self.p_st = p + else: + if self.D is None: + self.eigsys() + p = np.abs(np.real(self.W[:, 0])) + p = p / np.sum(p) + self.p_st = p + return self.p_st + + def simulate_random_walk(self, init_idx: int, tspan: np.ndarray) -> Tuple: + """Simulate a random walk on the continuous-time Markov chain from a given initial state. + + Args: + init_idx: The index of the initial state for the random walk. + tspan: The time span for the random walk as a 1D array. + + Returns: + A tuple containing two arrays: + The value at each time point. + The corresponding time points during the random walk. + """ + P = self.P.copy() + + def prop_func(c): + a = P[:, c[0]] + a[c[0]] = 0 + return a + + def update_func(c, mu): + return np.array([mu]) + + T, C = directMethod(prop_func, update_func, tspan=tspan, C0=np.array([init_idx])) + + return C, T + + def compute_mean_exit_time(self, p0: np.ndarray, sinks: np.ndarray) -> float: + """Compute the mean exit time given a initial distribution p0 and a set of sink nodes using: + met = inv(K) @ p0_ + where K is the transition rate matrix (P) where the columns and rows corresponding to the sinks are removed, + and p0_ the initial distribution w/o the sinks. Args: - method: The method used for computing the stationary distribution. + p0: The initial probability distribution vector. + sinks: The indices of the sink nodes. Returns: - The computed stationary distribution as a probability vector. + The computed mean exit time. """ - if method == "solve": - p = np.real(null_space(self.P - np.eye(self.P.shape[0])[:, 0])[:, 0].flatten()) - else: - if self.W is None: - self.eigsys() - p = np.abs(np.real(self.W[:, 0])) - p = p / np.sum(p) - return p + states = [] + for i in range(self.get_num_states()): + if not i in sinks: + states.append(i) + K = self.P[states][:, states] # submatrix of P excluding the sinks + p0_ = p0[states] + met = np.sum(-np.linalg.inv(K) @ p0_) + return met - def lump(self, labels: np.ndarray, M_weight: Optional[np.ndarray] = None) -> np.ndarray: - """Markov chain lumping based on: - K. Hoffmanna and P. Salamon, Bounding the lumping error in Markov chain dynamics, Appl Math Lett, (2009) + def compute_mean_first_passage_time(self, p0: np.ndarray, target: int, sinks: np.ndarray) -> float: + """Compute the mean first passage time given an initial distribution, a target node, and a set of sink nodes. Args: - labels: The lumping labels. - M_weight: The weighting matrix. If None, it is computed using the stationary distribution. + p0: The initial probability distribution vector of shape (n_states,). + target: The index of the target node. + sinks: The indices of the sink nodes. Returns: - The lumped transition matrix after the lumping operation. + The computed mean first passage time. """ - k = len(labels) - M_part = np.zeros((k, self.get_num_states())) - - for i in range(len(labels)): - M_part[labels[i], i] = 1 + states = [] + all_sinks = np.hstack((target, sinks)) + for i in range(self.get_num_states()): + if not i in all_sinks: + states.append(i) + K = self.P[states][:, states] # submatrix of P excluding the sinks + p0_ = p0[states] - n_node = self.get_num_states() - if M_weight is None: - p_st = self.compute_stationary_distribution() - M_weight = np.multiply(M_part, p_st) - M_weight = np.divide(M_weight.T, M_weight @ np.ones(n_node)) - P_lumped = M_part @ self.P @ M_weight + # find transition prob. from states to target + k = np.zeros(len(states)) + for i, state in enumerate(states): + k[i] = np.sum(self.P[target, state]) - return P_lumped + K_inv = np.linalg.inv(K) + mfpt = -(k @ (K_inv @ K_inv @ p0_)) / (k @ (K_inv @ p0_)) + return mfpt - def naive_lump(self, x: np.ndarray, grp: np.ndarray) -> np.ndarray: - """Perform naive Markov chain lumping based on given data and group labels. + def compute_hitting_time(self, p_st: Optional[np.ndarray] = None, return_Z: bool = False) -> Union[Tuple, np.ndarray]: + """Compute the hitting time of the continuous-time Markov chain. Args: - x: The data matrix. - grp: The group labels. + p_st: The stationary distribution of the continuous-time Markov chain. + return_Z: Whether to return the matrix Z in addition to the hitting time matrix. Returns: - The lumped transition matrix after the lumping operation. + The computed hitting time matrix. """ - k = len(np.unique(grp)) - y = np.zeros((k, k)) - for i in range(len(y)): - for j in range(len(y)): - y[i, j] = x[grp == i, :][:, grp == j].mean() - - return y + p_st = self.compute_stationary_distribution() if p_st is None else p_st + n_nodes = len(p_st) + W = np.ones((n_nodes, 1)) * p_st + Z = np.linalg.inv(-self.P.T + W) + H = np.ones((n_nodes, 1)) * np.diag(Z).T - Z + H = H / W + H = H.T + if return_Z: + return H, Z + else: + return H - def diffusion_map_embedding(self, n_dims: int = 2, t: int = 1) -> np.ndarray: - """Perform diffusion map embedding for the Markov chain. + def diffusion_map_embedding(self, n_dims: int = 2, t: Union[int, float] = 1, n_pca_dims: Optional[int] = None) -> np.ndarray: + """Perform diffusion map embedding for the continuous-time Markov chain. Args: n_dims: The number of dimensions for the diffusion map embedding. t: The diffusion time parameter used in the embedding. + n_pca_dims: The number of dimensions for PCA before diffusion map embedding. Returns: - The diffusion map embedding of the Markov chain as a matrix of shape (n_states, n_dims). + The diffusion map embedding of the continuous-time Markov chain. """ - if self.W is None: + if self.D is None: self.eigsys() ind = np.arange(1, n_dims + 1) - Y = np.real(self.D[ind] ** t) * np.real(self.U[:, ind]) + Y = np.real(np.exp(self.D[ind] ** t)) * np.real(self.U[:, ind]) + if n_pca_dims is not None: + pca = PCA(n_components=n_pca_dims) + Y = pca.fit_transform(Y) return Y - def simulate_random_walk(self, init_idx: int, num_steps: int) -> np.ndarray: - """Simulate a random walk on the Markov chain from a given initial state. + """def fit(self, X, V, k, s=None, tol=1e-4): + self.__reset__() + # knn clustering + if self.nbrs_idx is None: + nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(X) + _, Idx = nbrs.kneighbors(X) + self.nbrs_idx = Idx + else: + Idx = self.nbrs_idx + # compute transition prob. + n = X.shape[0] + self.P = np.zeros((n, n)) + for i in range(n): + y = X[i] + v = V[i] + Y = X[Idx[i, 1:]] + p = compute_markov_trans_prob(y, v, Y, s, cont_time=True) + p[p<=tol] = 0 # tolerance check + self.P[Idx[i, 1:], i] = p + self.P[i, i] = - np.sum(p)""" - Args: - init_idx: The index of the initial state for the random walk. - num_steps: The number of steps for the random walk. - Returns: - The sequence of state indices resulting from the random walk. - """ - P = self.P.copy() +def markov_combination(x: np.ndarray, v: np.ndarray, X: np.ndarray) -> Tuple: + """Calculate the Markov combination by solving a 'cvxopt' library quadratic programming (QP) problem, which is + defined as: + minimize (1/2)*x'*P*x + q'*x + subject to G*x <= h - seq = np.ones(num_steps + 1, dtype=int) * -1 - seq[0] = init_idx - for i in range(1, num_steps + 1): - cur_state = seq[i - 1] - r = np.random.rand() - seq[i] = np.cumsum(P[:, cur_state]).searchsorted(r) + Args: + x: The cell data matrix. + v: The velocity data matrix. + X: The neighbors data matrix. + + Returns: + A tuple containing the results of QP problem. + """ + from cvxopt import matrix, solvers + + n = X.shape[0] + R = matrix(X - x).T + H = R.T * R + f = matrix(v).T * R + G = np.vstack((-np.eye(n), np.ones(n))) + h = np.zeros(n + 1) + h[-1] = 1 + p = solvers.qp(H, -f.T, G=matrix(G), h=matrix(h))["x"] + u = R * p + return p, u + + +def compute_markov_trans_prob( + x: np.ndarray, v: np.ndarray, X: np.ndarray, s: Optional[np.ndarray] = None, cont_time: bool = False +) -> np.ndarray: + """Calculate the Markov transition probabilities by solving a 'cvxopt' library quadratic programming (QP) problem, + which is defined as: + minimize (1/2)*x'*P*x + q'*x + subject to G*x <= h + + Args: + x: The cell data matrix. + v: The velocity data matrix. + X: The neighbors data matrix. + s: Extra constraints added to the `q` in QP problem. + cont_time: Whether is continuous-time or not. + + Returns: + An array containing the optimal Markov transition probabilities computed by QP problem. + """ + from cvxopt import matrix, solvers + + n = X.shape[0] + R = X - x + # normalize R, v, and s + Rn = np.array(R, copy=True) + vn = np.array(v, copy=True) + scale = np.abs(np.max(R, 0) - np.min(R, 0)) + Rn = Rn / scale + vn = vn / scale + if s is not None: + sn = np.array(s, copy=True) + sn = sn / scale + A = np.hstack((Rn, 0.5 * Rn * Rn)) + b = np.hstack((vn, 0.5 * sn * sn)) + else: + A = Rn + b = vn + + H = A.dot(A.T) + f = b.dot(A.T) + if cont_time: + G = -np.eye(n) + h = np.zeros(n) + else: + G = np.vstack((-np.eye(n), np.ones(n))) + h = np.zeros(n + 1) + h[-1] = 1 + p = solvers.qp(matrix(H), matrix(-f), G=matrix(G), h=matrix(h))["x"] + p = np.array(p).flatten() + return p + + +@jit(nopython=True) +def compute_kernel_trans_prob( + x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float], cont_time: bool = False +) -> np.ndarray: + """Calculate the transition probabilities. + + Args: + x: The cell data matrix representing current state. + v: The velocity data matrix. + X: An array of data points representing the neighbors. + inv_s: The inverse of the diffusion matrix or a scalar value. + cont_time: Whether to use continuous-time kernel computation. + + Returns: + The computed transition probabilities for each state in the Markov chain. + """ + n = X.shape[0] + p = np.zeros(n) + for i in range(n): + d = X[i] - x + p[i] = np.exp(-0.25 * (d - v) @ inv_s @ (d - v).T) + p /= np.sum(p) + return p + + +# @jit(nopython=True) +def compute_drift_kernel(x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float]) -> np.ndarray: + """Compute the kernal representing the drift based on input data and parameters. - return seq + Args: + x: The cell data matrix representing current state. + v: The velocity data matrix. + X: An array of data points representing the neighbors. + inv_s: The inverse of the diffusion matrix or a scalar value. + Returns: + The computed drift kernel values for each state in the Markov chain. + """ + n = X.shape[0] + k = np.zeros(n) + for i in range(n): + d = X[i] - x + if np.isscalar(inv_s): + k[i] = np.exp(-0.25 * inv_s * (d - v).dot(d - v)) + else: + k[i] = np.exp(-0.25 * (d - v) @ inv_s @ (d - v).T) + return k -class ContinuousTimeMarkovChain(MarkovChain): - """ContinuousTimeMarkovChain class represents a continuous-time Markov chain.""" - def __init__(self, P: Optional[np.ndarray] = None, eignum: Optional[int] = None, **kwargs): - """Initialize the ContinuousTimeMarkovChain instance. - Args: - P: The transition matrix of the Markov chain. - eignum: Number of eigenvalues/eigenvectors to compute. - **kwargs: Additional keyword arguments to be passed to the base class MarkovChain's constructor. +"""def compute_drift_local_kernel(x, v, X, inv_s): + n = X.shape[0] + k = np.zeros(n) + # compute tau + D = X - x + dists = np.zeros(n) + vds = np.zeros(n) + for (i, d) in enumerate(D): + dists[i] = np.linalg.norm(d) + if dists[i] > 0: + vds[i] = v.dot(d) / dists[i] + i_dir = np.logical_and(vds >= np.quantile(vds, 0.7), vds > 0) + tau = np.mean(dists[i_dir] / vds[i_dir]) + if np.isnan(tau): tau = 1 + if tau > 1e2: tau = 1e2 - Returns: - An instance of ContinuousTimeMarkovChain. - """ - super().__init__(P, eignum=eignum, sumto=0, **kwargs) - self.Q = None # embedded markov chain transition matrix - self.Kd = None # density kernel for density adjustment - self.p_st = None # stationary distribution + tau_v = tau * v + tau_invs = (1 / (tau * np.linalg.norm(v))) * inv_s + for i in range(n): + d = D[i] + k[i] = np.exp(-0.25 * (d-tau_v) @ tau_invs @ (d-tau_v).T) + return k, tau_invs""" - def check_transition_rate_matrix(self, P: np.ndarray, tol: float = 1e-6) -> np.ndarray: - """Check if the input matrix is a valid transition rate matrix. - Args: - P: The transition rate matrix to be checked. - tol: Tolerance threshold for zero row- or column-sum check. +# @jit(nopython=True) +def compute_drift_local_kernel(x: np.ndarray, v: np.ndarray, X: np.ndarray, inv_s: Union[np.ndarray, float]) -> np.ndarray: + """Compute a local kernel representing the drift based on input data and parameters. - Returns: - The checked transition rate matrix. + Args: + x: The cell data matrix representing current state. + v: The velocity data matrix. + X: An array of data points representing the neighbors. + inv_s: The inverse of the diffusion matrix or a scalar value. - Raises: - ValueError: If the input transition rate matrix has non-zero row- and column-sums. - """ - if np.any(flatten(np.abs(np.sum(P, 0))) <= tol): - return P - elif np.any(flatten(np.abs(np.sum(P, 1))) <= tol): - return P.T + Returns: + The computed drift kernel values. + """ + n = X.shape[0] + k = np.zeros(n) + # compute tau + D = X - x + dists = np.zeros(n) + vds = np.zeros(n) + for (i, d) in enumerate(D): + dists[i] = np.linalg.norm(d) + if dists[i] > 0: + vds[i] = v.dot(d) / dists[i] + i_dir = np.logical_and(vds >= np.quantile(vds, 0.7), vds > 0) + if np.any(i_dir): + tau = np.mean(dists[i_dir] / vds[i_dir]) + if tau > 1e2: + tau = 1e2 + tau_v = tau * v + tau_invs = (1 / (tau * v.dot(v))) * inv_s + else: + tau_v = 0 + tau_invs = (1 / (1e2 * v.dot(v))) * inv_s + for i in range(n): + d = D[i] + if np.isscalar(tau_invs): + k[i] = np.exp(-0.25 * tau_invs * (d - tau_v).dot(d - tau_v)) else: - raise ValueError("The input transition rate matrix must have a zero row- or column-sum.") - - def compute_drift(self, X: np.ndarray, t: float, n_top: int = 5, normalize_vector: bool = False) -> np.ndarray: - """Compute the drift for each state in the continuous-time Markov chain. - - Args: - X: The data matrix of shape which represents the states of the Markov chain. - t: The time at which the drift is computed. - n_top: The number of top states to consider for drift computation. - normalize_vector: Whether to normalize the drift vector for each state. + k[i] = np.exp(-0.25 * (d - tau_v) @ tau_invs @ (d - tau_v).T) + return k - Returns: - The computed drift values for each state in the continuous-time Markov chain. - """ - n = self.get_num_states() - V = np.zeros_like(X) - P = self.compute_transition_matrix(t) - for i in range(n): - if n_top is None: - d = (X - X[i]).T - if normalize_vector: - d = d / np.linalg.norm(d, axis=0) - V[i] = d.dot(P[:, i]) - else: - idx = np.argsort(P[:, i])[-n_top:] - d = (X[idx] - X[i]).T - if normalize_vector: - d = d / np.linalg.norm(d, axis=0) - V[i] = d.dot(P[idx, i]) - # q = P[idx, i] / np.sum(P[idx, i]) - # V[i] = d.dot(q) - return V +@jit(nopython=True) +def compute_density_kernel(x: np.ndarray, X: np.ndarray, inv_eps: float) -> np.ndarray: + """Compute the density kernel values. - def compute_density_corrected_drift( - self, X: np.ndarray, t: float, k: Optional[int] = None, normalize_vector: bool = False - ) -> np.ndarray: - """Compute density-corrected drift for each state in the continuous-time Markov chain. + Args: + x: The cell data matrix representing current state. + X: An array of data points representing the neighbors. + inv_eps: The inverse of the epsilon. - Args: - X: The data matrix of shape which represents the states of the Markov chain. - t: The time at which the density-corrected drift is computed. - k: The number of nearest neighbors used for computing the correction term. - normalize_vector: Whether to normalize the drift vector for each state. + Returns: + The computed density kernel values for each state. + """ + n = X.shape[0] + k = np.zeros(n) + for i in range(n): + d = X[i] - x + k[i] = np.exp(-0.25 * inv_eps * d.dot(d)) + return k - Returns: - The computed density-corrected drift values for each state in the continuous-time Markov chain. - """ - n = self.get_num_states() - V = np.zeros_like(X) - P = self.compute_transition_matrix(t) - for i in range(n): - P[i, i] = 0 - P[:, i] /= np.sum(P[:, i]) - d = X - X[i] - if normalize_vector: - d /= np.linalg.norm(d) - correction = 1 / k if k is not None else np.mean(P[:, i]) - V[i] = d.T.dot(P[:, i] - correction) - return V - def compute_transition_matrix(self, t: float) -> np.ndarray: - """Compute the transition matrix for a given time. +@jit(nopython=True) +def makeTransitionMatrix(Qnn: np.ndarray, I_vec: np.ndarray, tol: float = 0.0) -> np.ndarray: # Qnn, I, tol=0.0 + """Create the transition matrix based on the transition rate matrix `Qnn` and the indexing vector `I_vec`. - Args: - t: The time at which the transition matrix is computed. + Args: + Qnn: The matrix which represents the transition rates between different states. + I_vec: The indexing vector to map the rows to the appropriate positions in the transition matrix. + tol: A numerical tolerance value to consider rate matrix elements as zero. - Returns: - The computed transition matrix. - """ + Returns: + The computed transition matrix based on `Qnn` and `I_vec`. + """ + n = Qnn.shape[0] + M = np.zeros((n, n)) - if self.D is None: - self.eigsys() - P = np.real(self.W @ np.diag(np.exp(self.D * t)) @ self.W_inv) - return P + for i in range(n): + q = Qnn[i] + q[q < tol] = 0 + M[I_vec[i], i] = q + M[i, i] = 1 - np.sum(q) + return M - def compute_embedded_transition_matrix(self) -> np.ndarray: - """Compute the embedded Markov chain transition matrix. - Returns: - The computed embedded Markov chain transition matrix. - """ - self.Q = np.array(self.P, copy=True) - for i in range(self.Q.shape[1]): - self.Q[i, i] = 0 - self.Q[:, i] /= np.sum(self.Q[:, i]) - return self.Q +@jit(nopython=True) +def compute_tau(X: np.ndarray, V: np.ndarray, k: int = 100, nbr_idx: Optional[np.ndarray] = None) -> Tuple: + """Compute the tau values for each state in `X` based on the local density and velocity magnitudes. - def solve_distribution(self, p0: np.ndarray, t: float) -> np.ndarray: - """Solve the distribution for the continuous-time Markov chain at a given time. + Args: + X: The data matrix which represents the states of the system. + V: The velocity matrix which represents the velocity vectors associated with each state in `X`. + k: The number of neighbors to consider when estimating local density. Default is 100. + nbr_idx: The indices of neighbors for each state in `X`. - Args: - p0: The initial probability distribution vector. - t: The time at which the distribution is solved. + Returns: + The computed tau values representing the timescale of transitions for each state in `X`. The computed velocity + magnitudes for each state in `X`. + """ - Returns: - The computed probability distribution vector at time. - """ - P = self.compute_transition_matrix(t) - p = P @ p0 - p = p / np.sum(p) - return p + if nbr_idx is None: + _, dists = k_nearest_neighbors( + X, + k=k - 1, + exclude_self=False, + pynn_rand_state=19491001, + n_jobs=-1, + ) + else: + dists = np.zeros(nbr_idx.shape) + for i in range(nbr_idx.shape[0]): + for j in range(nbr_idx.shape[1]): + x = X[i] + y = X[nbr_idx[i, j]] + dists[i, j] = np.sqrt((x - y).dot(x - y)) + d = np.mean(dists[:, 1:], 1) + v = np.linalg.norm(V, axis=1) + tau = d / v + return tau, v - def compute_stationary_distribution(self, method: str = "eig"): - """Compute the stationary distribution of the continuous-time Markov chain. - Args: - method: The method used for computing the stationary distribution. +# we might need a separate module/file for discrete vector field and markovian methods in the future +def graphize_velocity( + V: np.ndarray, + X: np.ndarray, + nbrs_idx: Optional[list] = None, + k: int = 30, + normalize_v: bool = False, + E_func: Optional[Union[Callable, str]] = None +) -> Tuple: + """The function generates a graph based on the velocity data. The flow from i- to j-th + node is returned as the edge matrix E[i, j], and E[i, j] = -E[j, i]. - Returns: - The computed stationary distribution of the continuous-time Markov chain. - """ - if self.p_st is None: - if method == "null": - p = np.abs(np.real(null_space(self.P)[:, 0].flatten())) - p = p / np.sum(p) - self.p_st = p - else: - if self.D is None: - self.eigsys() - p = np.abs(np.real(self.W[:, 0])) - p = p / np.sum(p) - self.p_st = p - return self.p_st + Args: + V: The velocities for all cells. + X: The coordinates for all cells. + nbrs_idx: A list of neighbor indices for each cell. If None a KNN will be performed instead. + k: The number of neighbors for the KNN search. + normalize_v: Whether to normalize the velocity vectors. + E_func: A variance stabilizing function for reducing the variance of the flows. + If a string is passed, there are two options: + 'sqrt': the `numpy.sqrt` square root function; + 'exp': the `numpy.exp` exponential function. - def simulate_random_walk(self, init_idx: int, tspan: np.ndarray) -> Tuple: - """Simulate a random walk on the continuous-time Markov chain from a given initial state. + Returns: + The edge matrix and the neighbor indices. + """ + n, d = X.shape - Args: - init_idx: The index of the initial state for the random walk. - tspan: The time span for the random walk as a 1D array. + nbrs = None + if nbrs_idx is None: + nbrs_idx, _ = k_nearest_neighbors( + X, + k=k, + exclude_self=False, + pynn_rand_state=19491001, + ) - Returns: - A tuple containing two arrays: - The value at each time point. - The corresponding time points during the random walk. - """ - P = self.P.copy() + if type(E_func) is str: + if E_func == "sqrt": + E_func = np.sqrt + elif E_func == "exp": + E_func = np.exp + else: + raise NotImplementedError("The specified edge function is not implemented.") - def prop_func(c): - a = P[:, c[0]] - a[c[0]] = 0 - return a + # E = sp.csr_matrix((n, n)) # Making E a csr_matrix will slow down this process. Try lil_matrix maybe? + E = np.zeros((n, n)) + for i in range(n): + x = flatten(X[i]) + idx = nbrs_idx[i] + if len(idx) > 0 and idx[0] == i: # excluding the node itself from the neighbors + idx = idx[1:] + vi = flatten(V[i]) + if normalize_v: + vi_norm = np.linalg.norm(vi) + if vi_norm > 0: + vi /= vi_norm - def update_func(c, mu): - return np.array([mu]) + # normalized differences + U = X[idx] - x + U_norm = np.linalg.norm(U, axis=1) + U_norm[U_norm == 0] = 1 + U /= U_norm[:, None] - T, C = directMethod(prop_func, update_func, tspan=tspan, C0=np.array([init_idx])) + for jj, j in enumerate(idx): + vj = flatten(V[j]) + if normalize_v: + vj_norm = np.linalg.norm(vj) + if vj_norm > 0: + vj /= vj_norm + u = flatten(U[jj]) + v = np.mean((vi.dot(u), vj.dot(u))) - return C, T + if E_func is not None: + v = np.sign(v) * E_func(np.abs(v)) + E[i, j] = v + E[j, i] = -v - def compute_mean_exit_time(self, p0: np.ndarray, sinks: np.ndarray) -> float: - """Compute the mean exit time given a initial distribution p0 and a set of sink nodes using: - met = inv(K) @ p0_ - where K is the transition rate matrix (P) where the columns and rows corresponding to the sinks are removed, - and p0_ the initial distribution w/o the sinks. + return E, nbrs_idx - Args: - p0: The initial probability distribution vector. - sinks: The indices of the sink nodes. - Returns: - The computed mean exit time. - """ - states = [] - for i in range(self.get_num_states()): - if not i in sinks: - states.append(i) - K = self.P[states][:, states] # submatrix of P excluding the sinks - p0_ = p0[states] - met = np.sum(-np.linalg.inv(K) @ p0_) - return met +def calc_Laplacian(E: np.ndarray, convention: str = "graph") -> np.ndarray: + """Calculate the Laplacian matrix of a given matrix of edge weights. - def compute_mean_first_passage_time(self, p0: np.ndarray, target: int, sinks: np.ndarray) -> float: - """Compute the mean first passage time given an initial distribution, a target node, and a set of sink nodes. + Args: + E: The matrix of edge weights which represents the weights of edges in a graph. + convention: The convention used to compute the Laplacian matrix. + If "graph", the Laplacian matrix will be calculated as the diagonal matrix of node degrees minus the adjacency matrix. + If "diffusion", the Laplacian matrix will be calculated as the negative of the graph Laplacian. + Default is "graph". - Args: - p0: The initial probability distribution vector of shape (n_states,). - target: The index of the target node. - sinks: The indices of the sink nodes. + Returns: + The Laplacian matrix. + """ + A = np.abs(np.sign(E)) + L = np.diag(np.sum(A, 0)) - A - Returns: - The computed mean first passage time. - """ - states = [] - all_sinks = np.hstack((target, sinks)) - for i in range(self.get_num_states()): - if not i in all_sinks: - states.append(i) - K = self.P[states][:, states] # submatrix of P excluding the sinks - p0_ = p0[states] + if convention == "diffusion": + L = -L - # find transition prob. from states to target - k = np.zeros(len(states)) - for i, state in enumerate(states): - k[i] = np.sum(self.P[target, state]) + return L - K_inv = np.linalg.inv(K) - mfpt = -(k @ (K_inv @ K_inv @ p0_)) / (k @ (K_inv @ p0_)) - return mfpt - def compute_hitting_time(self, p_st: Optional[np.ndarray] = None, return_Z: bool = False) -> Union[Tuple, np.ndarray]: - """Compute the hitting time of the continuous-time Markov chain. +def fp_operator(E: np.ndarray, D: Union[int, float]) -> np.ndarray: + """Calculate the Fokker-Planck operator based on the given matrix of edge weights (E) and diffusion coefficient (D). - Args: - p_st: The stationary distribution of the continuous-time Markov chain. - return_Z: Whether to return the matrix Z in addition to the hitting time matrix. + Args: + E: The matrix of edge weights. + D: The diffusion coefficient used in the Fokker-Planck operator. - Returns: - The computed hitting time matrix. - """ - p_st = self.compute_stationary_distribution() if p_st is None else p_st - n_nodes = len(p_st) - W = np.ones((n_nodes, 1)) * p_st - Z = np.linalg.inv(-self.P.T + W) - H = np.ones((n_nodes, 1)) * np.diag(Z).T - Z - H = H / W - H = H.T - if return_Z: - return H, Z - else: - return H + Returns: + The Fokker-Planck operator matrix. + """ + # drift + Mu = E.T.copy() + Mu[Mu < 0] = 0 + Mu = np.diag(np.sum(Mu, 0)) - Mu + # diffusion + L = calc_Laplacian(E, convention="diffusion") - def diffusion_map_embedding(self, n_dims: int = 2, t: Union[int, float] = 1, n_pca_dims: Optional[int] = None) -> np.ndarray: - """Perform diffusion map embedding for the continuous-time Markov chain. + return -Mu + D * L - Args: - n_dims: The number of dimensions for the diffusion map embedding. - t: The diffusion time parameter used in the embedding. - n_pca_dims: The number of dimensions for PCA before diffusion map embedding. - Returns: - The diffusion map embedding of the continuous-time Markov chain. - """ - if self.D is None: - self.eigsys() +def divergence(E: np.ndarray, tol: float = 1e-5) -> np.ndarray: + """Calculate the divergence for each node in a given matrix of edge weights. - ind = np.arange(1, n_dims + 1) - Y = np.real(np.exp(self.D[ind] ** t)) * np.real(self.U[:, ind]) - if n_pca_dims is not None: - pca = PCA(n_components=n_pca_dims) - Y = pca.fit_transform(Y) - return Y + Args: + E: The matrix of edge weights. + tol: The tolerance value. Edge weights below this value will be treated as zero. - """def fit(self, X, V, k, s=None, tol=1e-4): - self.__reset__() - # knn clustering - if self.nbrs_idx is None: - nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(X) - _, Idx = nbrs.kneighbors(X) - self.nbrs_idx = Idx - else: - Idx = self.nbrs_idx - # compute transition prob. - n = X.shape[0] - self.P = np.zeros((n, n)) - for i in range(n): - y = X[i] - v = V[i] - Y = X[Idx[i, 1:]] - p = compute_markov_trans_prob(y, v, Y, s, cont_time=True) - p[p<=tol] = 0 # tolerance check - self.P[Idx[i, 1:], i] = p - self.P[i, i] = - np.sum(p)""" + Returns: + The divergence values for each node. + """ + n = E.shape[0] + div = np.zeros(n) + # optimize for sparse matrices later... + for i in range(n): + for j in range(i + 1, n): + if np.abs(E[i, j]) > tol: + div[i] += E[i, j] - E[j, i] + + return div diff --git a/dynamo/tools/connectivity.py b/dynamo/tools/connectivity.py index 3492d2f84..be07a3145 100755 --- a/dynamo/tools/connectivity.py +++ b/dynamo/tools/connectivity.py @@ -95,131 +95,22 @@ def knn_to_adj(knn_indices: np.ndarray, knn_weights: np.ndarray) -> csr_matrix: return adj -def get_conn_dist_graph(knn: np.ndarray, distances: np.ndarray) -> Tuple[csr_matrix, csr_matrix]: - """Compute connection and distance sparse matrix. - - Args: - knn: A matrix (n x n_neighbors) storing the indices for each node's n_neighbors nearest neighbors in knn graph. - distances: The distances to the n_neighbors the closest points in knn graph. - - Returns: - A tuple (distances, connectivities), where distance is the distance sparse matrix and connectivities is the - connectivity sparse matrix. - """ - - n_obs, n_neighbors = knn.shape - distances = csr_matrix( - ( - distances.flatten(), - (np.repeat(np.arange(n_obs), n_neighbors), knn.flatten()), - ), - shape=(n_obs, n_obs), - ) - connectivities = distances.copy() - connectivities.data[connectivities.data > 0] = 1 - - distances.eliminate_zeros() - connectivities.eliminate_zeros() - - return distances, connectivities - - -def construct_mapper_umap( - X: np.ndarray, - n_neighbors: int = 30, - n_components: int = 2, - metric: Union[str, Callable] = "euclidean", - min_dist: float = 0.1, - spread: float = 1.0, - max_iter: Optional[int] = None, - alpha: float = 1.0, - gamma: float = 1.0, - negative_sample_rate: float = 5, - init_pos: Union[Literal["spectral", "random"], np.ndarray] = "spectral", - random_state: Union[int, np.random.RandomState, None] = 0, - verbose: bool = False, - **umap_kwargs, -) -> UMAP: - """Construct a UMAP object. +def normalize_knn_graph(knn: csr_matrix) -> csr_matrix: + """Normalize the knn graph so that each row will be sum up to 1. Args: - X: the expression matrix (n_cell x n_genes). - n_neighbors: the number of nearest neighbors to compute for each sample in `X`. Defaults to 30. - n_components: the dimension of the space to embed into. Defaults to 2. - metric: the metric to use for the computation. Defaults to "euclidean". - min_dist: the effective minimum distance between embedded points. Smaller values will result in a more - clustered/clumped embedding where nearby points on the manifold are drawn closer together, while larger - values will result on a more even dispersal of points. The value should be set relative to the `spread` - value, which determines the scale at which embedded points will be spread out. Defaults to 0.1. - spread: the effective scale of embedded points. In combination with min_dist this determines how - clustered/clumped the embedded points are. Defaults to 1.0. - max_iter: the number of training epochs to be used in optimizing the low dimensional embedding. Larger values - result in more accurate embeddings. If None is specified a value will be selected based on the size of the - input dataset (200 for large datasets, 500 for small). This argument was refactored from n_epochs from - UMAP-learn to account for recent API changes in UMAP-learn 0.5.2. Defaults to None. - alpha: initial learning rate for the SGD. Defaults to 1.0. - gamma: weight to apply to negative samples. Values higher than one will result in greater weight being given to - negative samples. Defaults to 1.0. - negative_sample_rate: the number of negative samples to select per positive sample in the optimization process. - Increasing this value will result in greater repulsive force being applied, greater optimization cost, but - slightly more accuracy. The number of negative edge/1-simplex samples to use per positive edge/1-simplex - sample in optimizing the low dimensional embedding. Defaults to 5. - init_pos: the method to initialize the low dimensional embedding. Where: - "spectral": use a spectral embedding of the fuzzy 1-skeleton. - "random": assign initial embedding positions at random. - np.ndarray: the array to define the initial position. - Defaults to "spectral". - random_state: the method to generate random numbers. If int, random_state is the seed used by the random number - generator; If RandomState instance, random_state is the random number generator; If None, the random number - generator is the RandomState instance used by `numpy.random`. Defaults to 0. - verbose: whether to log verbosely. Defaults to False. + knn: The sparse matrix containing the indices of nearest neighbors of each cell. Returns: - A `mapper` that is the data mapped onto umap space. + The normalized matrix. """ - import umap.umap_ as umap - from sklearn.utils import check_random_state - - from .utils import update_dict - - # also see github issue at: https://github.com/lmcinnes/umap/issues/798 - default_epochs = 500 if X.shape[0] <= 10000 else 200 - max_iter = default_epochs if max_iter is None else max_iter - - random_state = check_random_state(random_state) - - _umap_kwargs = { - "angular_rp_forest": False, - "local_connectivity": 1.0, - "metric_kwds": None, - "set_op_mix_ratio": 1.0, - "target_metric": "categorical", - "target_metric_kwds": None, - "target_n_neighbors": -1, - "target_weight": 0.5, - "transform_queue_size": 4.0, - "transform_seed": 42, - } - umap_kwargs = update_dict(_umap_kwargs, umap_kwargs) - - mapper = umap.UMAP( - n_neighbors=n_neighbors, - n_components=n_components, - metric=metric, - min_dist=min_dist, - spread=spread, - n_epochs=max_iter, - learning_rate=alpha, - repulsion_strength=gamma, - negative_sample_rate=negative_sample_rate, - init=init_pos, - random_state=random_state, - verbose=verbose, - **umap_kwargs, - ).fit(X) + """normalize the knn graph so that each row will be sum up to 1.""" + knn.setdiag(1) + knn = knn.astype("float32") + sparsefuncs.inplace_row_scale(knn, 1 / knn.sum(axis=1).A1) - return mapper + return knn @docstrings.get_sectionsf("umap_ann") @@ -467,131 +358,102 @@ def umap_conn_indices_dist_embedding( return graph, knn_indices, knn_dists, embedding_ -CsrOrNdarray = TypeVar("CsrOrNdarray", csr_matrix, np.ndarray) - - -def mnn_from_list(knn_graph_list: List[CsrOrNdarray]) -> CsrOrNdarray: - """Apply `reduce` function to calculate the mutual kNN. - - Args: - knn_graph_list: A list of ndarray or csr_matrix representing a series of knn graphs. - - Returns: - The calculated mutual knn, in same type as the input (ndarray of csr_matrix). - """ - - import functools - - mnn = ( - functools.reduce(scipy.sparse.csr.csr_matrix.minimum, knn_graph_list) - if issparse(knn_graph_list[0]) - else functools.reduce(scipy.minimum, knn_graph_list) - ) - - return mnn - - -def normalize_knn_graph(knn: csr_matrix) -> csr_matrix: - """Normalize the knn graph so that each row will be sum up to 1. - - Args: - knn: The sparse matrix containing the indices of nearest neighbors of each cell. - - Returns: - The normalized matrix. - """ - - """normalize the knn graph so that each row will be sum up to 1.""" - knn.setdiag(1) - knn = knn.astype("float32") - sparsefuncs.inplace_row_scale(knn, 1 / knn.sum(axis=1).A1) - - return knn - - -def mnn( - adata: AnnData, - n_pca_components: int = 30, - n_neighbors: int = 250, - layers: Union[str, List[str]] = "all", - use_pca_fit: bool = True, - save_all_to_adata: bool = False, -) -> AnnData: - """Calculate mutual nearest neighbor graph across specific data layers. +def construct_mapper_umap( + X: np.ndarray, + n_neighbors: int = 30, + n_components: int = 2, + metric: Union[str, Callable] = "euclidean", + min_dist: float = 0.1, + spread: float = 1.0, + max_iter: Optional[int] = None, + alpha: float = 1.0, + gamma: float = 1.0, + negative_sample_rate: float = 5, + init_pos: Union[Literal["spectral", "random"], np.ndarray] = "spectral", + random_state: Union[int, np.random.RandomState, None] = 0, + verbose: bool = False, + **umap_kwargs, +) -> UMAP: + """Construct a UMAP object. Args: - adata: An AnnData object. - n_pca_components: The number of PCA components. Defaults to 30. - n_neighbors: The number of nearest neighbors to compute for each sample. Defaults to 250. - layers: The layer(s) to be normalized. When set to `'all'`, it will include RNA (X, raw) or spliced, unspliced, - protein, etc. Defaults to "all". - use_pca_fit: Whether to use the precomputed pca model to transform different data layers or calculate pca for - each data layer separately. Defaults to True. - save_all_to_adata: Whether to save_fig all calculated data to adata object. Defaults to False. - - Raises: - Exception: No PCA fit result in .uns. + X: the expression matrix (n_cell x n_genes). + n_neighbors: the number of nearest neighbors to compute for each sample in `X`. Defaults to 30. + n_components: the dimension of the space to embed into. Defaults to 2. + metric: the metric to use for the computation. Defaults to "euclidean". + min_dist: the effective minimum distance between embedded points. Smaller values will result in a more + clustered/clumped embedding where nearby points on the manifold are drawn closer together, while larger + values will result on a more even dispersal of points. The value should be set relative to the `spread` + value, which determines the scale at which embedded points will be spread out. Defaults to 0.1. + spread: the effective scale of embedded points. In combination with min_dist this determines how + clustered/clumped the embedded points are. Defaults to 1.0. + max_iter: the number of training epochs to be used in optimizing the low dimensional embedding. Larger values + result in more accurate embeddings. If None is specified a value will be selected based on the size of the + input dataset (200 for large datasets, 500 for small). This argument was refactored from n_epochs from + UMAP-learn to account for recent API changes in UMAP-learn 0.5.2. Defaults to None. + alpha: initial learning rate for the SGD. Defaults to 1.0. + gamma: weight to apply to negative samples. Values higher than one will result in greater weight being given to + negative samples. Defaults to 1.0. + negative_sample_rate: the number of negative samples to select per positive sample in the optimization process. + Increasing this value will result in greater repulsive force being applied, greater optimization cost, but + slightly more accuracy. The number of negative edge/1-simplex samples to use per positive edge/1-simplex + sample in optimizing the low dimensional embedding. Defaults to 5. + init_pos: the method to initialize the low dimensional embedding. Where: + "spectral": use a spectral embedding of the fuzzy 1-skeleton. + "random": assign initial embedding positions at random. + np.ndarray: the array to define the initial position. + Defaults to "spectral". + random_state: the method to generate random numbers. If int, random_state is the seed used by the random number + generator; If RandomState instance, random_state is the random number generator; If None, the random number + generator is the RandomState instance used by `numpy.random`. Defaults to 0. + verbose: whether to log verbosely. Defaults to False. Returns: - An updated anndata object that are updated with the `mnn` or other relevant data that are calculated during mnn - calculation. + A `mapper` that is the data mapped onto umap space. """ - if use_pca_fit: - if "PCs" in adata.uns.keys(): - PCs = adata.uns["PCs"] - else: - raise Exception("use_pca_fit is set to be True, but there is no pca fit results in .uns attribute.") - - layers = DynamoAdataKeyManager.get_available_layer_keys(adata, layers, False, False) - layers = [ - layer - for layer in layers - if layer.startswith("X_") and (not layer.endswith("_matrix") and not layer.endswith("_ambiguous")) - ] - knn_graph_list = [] - for layer in layers: - layer_X = adata.layers[layer] - layer_X = log1p_(adata, layer_X) - if use_pca_fit: - layer_pca = expr_to_pca(layer_X, PCs=PCs, mean=layer_X.mean(0))[:, 1:] - else: - transformer = TruncatedSVD(n_components=n_pca_components + 1, random_state=0) - layer_pca = transformer.fit_transform(layer_X)[:, 1:] + import umap.umap_ as umap + from sklearn.utils import check_random_state - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - ( - graph, - knn_indices, - knn_dists, - X_dim, - ) = umap_conn_indices_dist_embedding(layer_pca, n_neighbors=n_neighbors, return_mapper=False) + from .utils import update_dict - if save_all_to_adata: - adata.obsm[layer + "_pca"], adata.obsm[layer + "_umap"] = ( - layer_pca, - X_dim, - ) - n_neighbors = signature(umap_conn_indices_dist_embedding).parameters["n_neighbors"] + # also see github issue at: https://github.com/lmcinnes/umap/issues/798 + default_epochs = 500 if X.shape[0] <= 10000 else 200 + max_iter = default_epochs if max_iter is None else max_iter - adata.uns[layer + "_neighbors"] = { - "params": {"n_neighbors": eval(n_neighbors), "method": "umap"}, - # "connectivities": None, - # "distances": None, - "indices": knn_indices, - } - ( - adata.obsp[layer + "_connectivities"], - adata.obsp[layer + "_distances"], - ) = (graph, knn_dists) + random_state = check_random_state(random_state) - knn_graph_list.append(graph > 0) + _umap_kwargs = { + "angular_rp_forest": False, + "local_connectivity": 1.0, + "metric_kwds": None, + "set_op_mix_ratio": 1.0, + "target_metric": "categorical", + "target_metric_kwds": None, + "target_n_neighbors": -1, + "target_weight": 0.5, + "transform_queue_size": 4.0, + "transform_seed": 42, + } + umap_kwargs = update_dict(_umap_kwargs, umap_kwargs) - mnn = mnn_from_list(knn_graph_list) - adata.uns["mnn"] = normalize_knn_graph(mnn) + mapper = umap.UMAP( + n_neighbors=n_neighbors, + n_components=n_components, + metric=metric, + min_dist=min_dist, + spread=spread, + n_epochs=max_iter, + learning_rate=alpha, + repulsion_strength=gamma, + negative_sample_rate=negative_sample_rate, + init=init_pos, + random_state=random_state, + verbose=verbose, + **umap_kwargs, + ).fit(X) - return adata + return mapper def generate_neighbor_keys(result_prefix: str = "") -> Tuple[str, str, str]: @@ -618,30 +480,6 @@ def generate_neighbor_keys(result_prefix: str = "") -> Tuple[str, str, str]: return conn_key, dist_key, neighbor_key -def correct_hnsw_neighbors(knn_hn: np.ndarray, distances_hn: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - """Corrects the indices and corresponding distances obtained from a hnswlib by manually adding self neighbors. - - Args: - knn_hn: Array containing the k-NN indices obtained from the hnswlib. - distances_hn: Array containing the distances corresponding to the k-NN indices obtained from the HNSW index. - - Returns: - A tuple containing the corrected indices and distances. - """ - mask = knn_hn[:, 0] == np.arange(knn_hn.shape[0]) - target_indices = np.where(mask)[0] - def roll(arr, value=0): - arr = np.roll(arr, 1, axis=0) - arr[0] = value - return arr - - knn_corrected = [knn_hn[i] if i in target_indices else roll(knn_hn[i], i) for i in range(knn_hn.shape[0])] - distances_corrected = [ - distances_hn[i] if i in target_indices else roll(distances_hn[i]) for i in range(distances_hn.shape[0]) - ] - return np.vstack(knn_corrected), np.vstack(distances_corrected) - - def k_nearest_neighbors( X: np.ndarray, k: int, @@ -961,3 +799,166 @@ def check_and_recompute_neighbors(adata: AnnData, result_prefix: str = "") -> No if not check_neighbors_completeness(adata, conn_key=conn_key, dist_key=dist_key, result_prefix=result_prefix): main_info("Neighbor graph is broken, recomputing....") neighbors(adata, result_prefix=result_prefix) + + +def correct_hnsw_neighbors(knn_hn: np.ndarray, distances_hn: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Corrects the indices and corresponding distances obtained from a hnswlib by manually adding self neighbors. + + Args: + knn_hn: Array containing the k-NN indices obtained from the hnswlib. + distances_hn: Array containing the distances corresponding to the k-NN indices obtained from the HNSW index. + + Returns: + A tuple containing the corrected indices and distances. + """ + mask = knn_hn[:, 0] == np.arange(knn_hn.shape[0]) + target_indices = np.where(mask)[0] + + def roll(arr, value=0): + arr = np.roll(arr, 1, axis=0) + arr[0] = value + return arr + + knn_corrected = [knn_hn[i] if i in target_indices else roll(knn_hn[i], i) for i in range(knn_hn.shape[0])] + distances_corrected = [ + distances_hn[i] if i in target_indices else roll(distances_hn[i]) for i in range(distances_hn.shape[0]) + ] + return np.vstack(knn_corrected), np.vstack(distances_corrected) + + +CsrOrNdarray = TypeVar("CsrOrNdarray", csr_matrix, np.ndarray) + + +def mnn_from_list(knn_graph_list: List[CsrOrNdarray]) -> CsrOrNdarray: + """Apply `reduce` function to calculate the mutual kNN. + + Args: + knn_graph_list: A list of ndarray or csr_matrix representing a series of knn graphs. + + Returns: + The calculated mutual knn, in same type as the input (ndarray of csr_matrix). + """ + + import functools + + mnn = ( + functools.reduce(scipy.sparse.csr.csr_matrix.minimum, knn_graph_list) + if issparse(knn_graph_list[0]) + else functools.reduce(scipy.minimum, knn_graph_list) + ) + + return mnn + + +def mnn( + adata: AnnData, + n_pca_components: int = 30, + n_neighbors: int = 250, + layers: Union[str, List[str]] = "all", + use_pca_fit: bool = True, + save_all_to_adata: bool = False, +) -> AnnData: + """Calculate mutual nearest neighbor graph across specific data layers. + + Args: + adata: An AnnData object. + n_pca_components: The number of PCA components. Defaults to 30. + n_neighbors: The number of nearest neighbors to compute for each sample. Defaults to 250. + layers: The layer(s) to be normalized. When set to `'all'`, it will include RNA (X, raw) or spliced, unspliced, + protein, etc. Defaults to "all". + use_pca_fit: Whether to use the precomputed pca model to transform different data layers or calculate pca for + each data layer separately. Defaults to True. + save_all_to_adata: Whether to save_fig all calculated data to adata object. Defaults to False. + + Raises: + Exception: No PCA fit result in .uns. + + Returns: + An updated anndata object that are updated with the `mnn` or other relevant data that are calculated during mnn + calculation. + """ + + if use_pca_fit: + if "PCs" in adata.uns.keys(): + PCs = adata.uns["PCs"] + else: + raise Exception("use_pca_fit is set to be True, but there is no pca fit results in .uns attribute.") + + layers = DynamoAdataKeyManager.get_available_layer_keys(adata, layers, False, False) + layers = [ + layer + for layer in layers + if layer.startswith("X_") and (not layer.endswith("_matrix") and not layer.endswith("_ambiguous")) + ] + knn_graph_list = [] + for layer in layers: + layer_X = adata.layers[layer] + layer_X = log1p_(adata, layer_X) + if use_pca_fit: + layer_pca = expr_to_pca(layer_X, PCs=PCs, mean=layer_X.mean(0))[:, 1:] + else: + transformer = TruncatedSVD(n_components=n_pca_components + 1, random_state=0) + layer_pca = transformer.fit_transform(layer_X)[:, 1:] + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ( + graph, + knn_indices, + knn_dists, + X_dim, + ) = umap_conn_indices_dist_embedding(layer_pca, n_neighbors=n_neighbors, return_mapper=False) + + if save_all_to_adata: + adata.obsm[layer + "_pca"], adata.obsm[layer + "_umap"] = ( + layer_pca, + X_dim, + ) + n_neighbors = signature(umap_conn_indices_dist_embedding).parameters["n_neighbors"] + + adata.uns[layer + "_neighbors"] = { + "params": {"n_neighbors": eval(n_neighbors), "method": "umap"}, + # "connectivities": None, + # "distances": None, + "indices": knn_indices, + } + ( + adata.obsp[layer + "_connectivities"], + adata.obsp[layer + "_distances"], + ) = (graph, knn_dists) + + knn_graph_list.append(graph > 0) + + mnn = mnn_from_list(knn_graph_list) + adata.uns["mnn"] = normalize_knn_graph(mnn) + + return adata + + +def get_conn_dist_graph(knn: np.ndarray, distances: np.ndarray) -> Tuple[csr_matrix, csr_matrix]: + """Compute connection and distance sparse matrix. + + Args: + knn: A matrix (n x n_neighbors) storing the indices for each node's n_neighbors nearest neighbors in knn graph. + distances: The distances to the n_neighbors the closest points in knn graph. + + Returns: + A tuple (distances, connectivities), where distance is the distance sparse matrix and connectivities is the + connectivity sparse matrix. + """ + + n_obs, n_neighbors = knn.shape + distances = csr_matrix( + ( + distances.flatten(), + (np.repeat(np.arange(n_obs), n_neighbors), knn.flatten()), + ), + shape=(n_obs, n_obs), + ) + connectivities = distances.copy() + connectivities.data[connectivities.data > 0] = 1 + + distances.eliminate_zeros() + connectivities.eliminate_zeros() + + return distances, connectivities diff --git a/dynamo/tools/graph_calculus.py b/dynamo/tools/graph_calculus.py index afa72542f..075e2c077 100644 --- a/dynamo/tools/graph_calculus.py +++ b/dynamo/tools/graph_calculus.py @@ -322,24 +322,6 @@ def symmetrize_discrete_vector_field(F: np.ndarray, mode: Literal["asym", "sym"] return E_ -def dist_mat_to_gaussian_weight(dist: np.ndarray, sigma: float) -> np.ndarray: - """Calculate the corresponding Gaussian weight for each distance element in a distance matrix. - - Args: - dist: The distance matrix. Each element represents a distance to the mean. - sigma: The standard deviation of the gaussian distribution. - - Returns: - A matrix with each element corresponding to the Gaussian weight of the distance matrix. - """ - - dist = symmetrize_symmetric_matrix(dist) - W = elem_prod(dist, dist) / sigma**2 - W[W.nonzero()] = np.exp(-0.5 * W.data) - - return W - - def calc_gaussian_weight( nbrs_idx: list, dists: np.ndarray, @@ -445,6 +427,24 @@ def calc_laplacian( return np.asarray(L) if type(L) == np.matrix else L +def dist_mat_to_gaussian_weight(dist: np.ndarray, sigma: float) -> np.ndarray: + """Calculate the corresponding Gaussian weight for each distance element in a distance matrix. + + Args: + dist: The distance matrix. Each element represents a distance to the mean. + sigma: The standard deviation of the gaussian distribution. + + Returns: + A matrix with each element corresponding to the Gaussian weight of the distance matrix. + """ + + dist = symmetrize_symmetric_matrix(dist) + W = elem_prod(dist, dist) / sigma**2 + W[W.nonzero()] = np.exp(-0.5 * W.data) + + return W + + def fp_operator( F: np.ndarray, D: np.ndarray, @@ -501,6 +501,56 @@ def fp_operator( return Q +def potential( + F: Union[sp.csr_matrix, np.ndarray], + E: Optional[Union[sp.csr_matrix, np.ndarray]] = None, + W: Optional[Union[sp.csr_matrix, np.ndarray]] = None, + div: Optional[np.ndarray] = None, + method: Literal["inv", "pinv", "qr_pinv", "lsq"] = "lsq", +) -> np.ndarray: + """Calculate potential of a weighted graph. + + Potential is related to the intrinsic time. Note that the returned value from this function is the negative of + potential. Thus, small potential is related to smaller intrinsic time and vice versa. + + Args: + F: The graph vector field. F_ij encodes the flow on edge e_ij (from vector i to j). + E: The length of edges of the graph. If None, all edges are assumed to have length of 1. Defaults to None. + W: The edge weight of the graph. If None, all edges are assumed to have weight of 1. Defaults to None. + div: The divergence of the graph. If None, it would be calculated based on the graph's vector field and weight. + Defaults to None. + method: The method to be used to calculate the potential. Can be following: + 1. inv: using inverse of the laplacian matrix. + 2. pinv: using pseudo inverse of the laplacian matrix. + 3. qr_pinv: perform QR decomposition of the laplacian matrix first, then perform pinv. + 4. lsq: solve the least square problem between the laplacian matrix and the divergence. + `Defaults to "lsq". + + Returns: + Potential of this graph. + """ + + if W is None: + W = abs(F.sign()) if sp.issparse(F) else np.abs(np.sign(F)) + div_neg = -divergence(F, W=W) if div is None else -div + L = calc_laplacian(W, E=E, weight_mode="naive") + + if method == "inv": + p = np.linalg.inv(L).dot(div_neg) + elif method == "pinv": + p = np.linalg.pinv(L).dot(div_neg) + elif method == "qr_pinv": + Q, R = qr(L) + L_inv = np.linalg.pinv(R).dot(Q.T) + p = L_inv.dot(div_neg) + elif method == "lsq": + res = lsq_linear(L, div_neg) + p = res["x"] + + p -= p.min() + return p + + def divergence( E: np.ndarray, W: Optional[np.ndarray] = None, method: Literal["direct", "operator"] = "operator", weighted: bool = False ) -> np.ndarray: @@ -543,6 +593,19 @@ def divergence( return div +def divop(W: Union[sp.csr_matrix, np.ndarray]) -> np.ndarray: + """Return the divergence operator in matrix form. + + Args: + W: The edge weight of the graph. + + Returns: + The operator used to calculate the divergence of the graph. + """ + + return -0.5 * gradop(W).T + + def gradop(adj: Union[sp.csr_matrix, np.ndarray]) -> sp.csr_matrix: """Return the gradient operator of a weighted graph in matrix form. @@ -574,69 +637,6 @@ def gradient(E: Union[sp.csr_matrix, np.ndarray], p: np.ndarray) -> np.ndarray: return gradop(E).dot(p) -def divop(W: Union[sp.csr_matrix, np.ndarray]) -> np.ndarray: - """Return the divergence operator in matrix form. - - Args: - W: The edge weight of the graph. - - Returns: - The operator used to calculate the divergence of the graph. - """ - - return -0.5 * gradop(W).T - - -def potential( - F: Union[sp.csr_matrix, np.ndarray], - E: Optional[Union[sp.csr_matrix, np.ndarray]] = None, - W: Optional[Union[sp.csr_matrix, np.ndarray]] = None, - div: Optional[np.ndarray] = None, - method: Literal["inv", "pinv", "qr_pinv", "lsq"] = "lsq", -) -> np.ndarray: - """Calculate potential of a weighted graph. - - Potential is related to the intrinsic time. Note that the returned value from this function is the negative of - potential. Thus, small potential is related to smaller intrinsic time and vice versa. - - Args: - F: The graph vector field. F_ij encodes the flow on edge e_ij (from vector i to j). - E: The length of edges of the graph. If None, all edges are assumed to have length of 1. Defaults to None. - W: The edge weight of the graph. If None, all edges are assumed to have weight of 1. Defaults to None. - div: The divergence of the graph. If None, it would be calculated based on the graph's vector field and weight. - Defaults to None. - method: The method to be used to calculate the potential. Can be following: - 1. inv: using inverse of the laplacian matrix. - 2. pinv: using pseudo inverse of the laplacian matrix. - 3. qr_pinv: perform QR decomposition of the laplacian matrix first, then perform pinv. - 4. lsq: solve the least square problem between the laplacian matrix and the divergence. - `Defaults to "lsq". - - Returns: - Potential of this graph. - """ - - if W is None: - W = abs(F.sign()) if sp.issparse(F) else np.abs(np.sign(F)) - div_neg = -divergence(F, W=W) if div is None else -div - L = calc_laplacian(W, E=E, weight_mode="naive") - - if method == "inv": - p = np.linalg.inv(L).dot(div_neg) - elif method == "pinv": - p = np.linalg.pinv(L).dot(div_neg) - elif method == "qr_pinv": - Q, R = qr(L) - L_inv = np.linalg.pinv(R).dot(Q.T) - p = L_inv.dot(div_neg) - elif method == "lsq": - res = lsq_linear(L, div_neg) - p = res["x"] - - p -= p.min() - return p - - class GraphVectorField: """An object representing a graph vector field, storing its edges, edge lengths, and edge weights. diff --git a/dynamo/tools/graph_operators.py b/dynamo/tools/graph_operators.py index 77b25ea5d..fa10fd0b3 100644 --- a/dynamo/tools/graph_operators.py +++ b/dynamo/tools/graph_operators.py @@ -15,6 +15,47 @@ from scipy.sparse import csr_matrix +def potential(g: Graph, div_neg: Optional[csr_matrix] = None) -> np.ndarray: + """Calculate potential for each cell. + + The potential is related to the intrinsic time. Note that the returned value from this function is the negative of + potential. Thus, small potential is related to smaller intrinsic time and vice versa. + + Args: + g: Graph object. + div_neg: Negative divergence. If None, it will be calculated from the graph. + + Returns: + An array representing the potential. + """ + + div_neg = -div(g) if div_neg is None else div_neg + g_undirected = g.copy() + g_undirected.to_undirected() + L = np.array(g_undirected.laplacian()) + Q, R = qr(L) + p = np.linalg.pinv(R).dot(Q.T).dot(div_neg) + + res = p - p.min() + return res + + +def grad(g: Graph, div_neg: Optional[csr_matrix] = None) -> np.ndarray: + """Compute the gradient of a potential field on a graph. + + The gradient of a potential field on a graph represents the rate of change of the potential at each vertex. It is + obtained by multiplying the gradient operator with the potential field. + + Args: + g: Graph object. + div_neg: Negative divergence. If None, it will be calculated from the graph. + + Returns: + An array representing the gradient. + """ + return gradop(g).dot(potential(g, div_neg)) + + def gradop(g: Graph) -> csr_matrix: """Compute the gradient operator for a graph. @@ -31,6 +72,22 @@ def gradop(g: Graph) -> csr_matrix: return csr_matrix((x, (i, j)), shape=(ne, g.vcount())) +def div(g: Graph) -> np.ndarray: + """Calculate divergence for each cell. + + Negative values correspond to potential sink while positive corresponds to potential source. + https://en.wikipedia.org/wiki/Divergence + + Args: + g: Graph object. + + Returns: + The divergence of the graph + """ + weight = np.array(g.es.get_attribute_values("weight")) + return divop(g).dot(weight) + + def divop(g: Graph) -> csr_matrix: """Compute the divergence operator for a graph. @@ -43,6 +100,24 @@ def divop(g: Graph) -> csr_matrix: return -gradop(g).T +def curl(g: Graph) -> np.ndarray: + """Calculate curl for each cell. + + On 2d, negative values correspond to clockwise rotation while positive corresponds to anticlockwise rotation. + https://www.khanacademy.org/math/multivariable-calculus/greens-theorem-and-stokes-theorem/formal-definitions-of + -divergence-and-curl/a/defining-curl + + Args: + g: Graph object. + + Returns: + The curl of the graph. + """ + + weight = np.array(g.es.get_attribute_values("weight")) + return curlop(g).dot(weight) + + def curlop(g: Graph) -> csr_matrix: """Compute the curl operator for a graph. @@ -105,81 +180,6 @@ def laplacian1(g: Graph) -> csr_matrix: return cur_mat.T.dot(cur_mat) - grad_mat.dot(grad_mat.T) -def potential(g: Graph, div_neg: Optional[csr_matrix] = None) -> np.ndarray: - """Calculate potential for each cell. - - The potential is related to the intrinsic time. Note that the returned value from this function is the negative of - potential. Thus, small potential is related to smaller intrinsic time and vice versa. - - Args: - g: Graph object. - div_neg: Negative divergence. If None, it will be calculated from the graph. - - Returns: - An array representing the potential. - """ - - div_neg = -div(g) if div_neg is None else div_neg - g_undirected = g.copy() - g_undirected.to_undirected() - L = np.array(g_undirected.laplacian()) - Q, R = qr(L) - p = np.linalg.pinv(R).dot(Q.T).dot(div_neg) - - res = p - p.min() - return res - - -def grad(g: Graph, div_neg: Optional[csr_matrix] = None) -> np.ndarray: - """Compute the gradient of a potential field on a graph. - - The gradient of a potential field on a graph represents the rate of change of the potential at each vertex. It is - obtained by multiplying the gradient operator with the potential field. - - Args: - g: Graph object. - div_neg: Negative divergence. If None, it will be calculated from the graph. - - Returns: - An array representing the gradient. - """ - return gradop(g).dot(potential(g, div_neg)) - - -def div(g: Graph) -> np.ndarray: - """Calculate divergence for each cell. - - Negative values correspond to potential sink while positive corresponds to potential source. - https://en.wikipedia.org/wiki/Divergence - - Args: - g: Graph object. - - Returns: - The divergence of the graph - """ - weight = np.array(g.es.get_attribute_values("weight")) - return divop(g).dot(weight) - - -def curl(g: Graph) -> np.ndarray: - """Calculate curl for each cell. - - On 2d, negative values correspond to clockwise rotation while positive corresponds to anticlockwise rotation. - https://www.khanacademy.org/math/multivariable-calculus/greens-theorem-and-stokes-theorem/formal-definitions-of - -divergence-and-curl/a/defining-curl - - Args: - g: Graph object. - - Returns: - The curl of the graph. - """ - - weight = np.array(g.es.get_attribute_values("weight")) - return curlop(g).dot(weight) - - def triangles(g: Graph) -> List[int]: """Count the number of triangles each vertex participates in within a graph using cliques. A triangle is a cycle of length 3 in an undirected graph. diff --git a/dynamo/tools/metric_velocity.py b/dynamo/tools/metric_velocity.py index 4ad3c5a50..4c812e252 100755 --- a/dynamo/tools/metric_velocity.py +++ b/dynamo/tools/metric_velocity.py @@ -193,70 +193,6 @@ def cell_wise_confidence( return adata -def jaccard( - X: np.ndarray, V: np.ndarray, n_pca_components: int, n_neigh: int, X_neighbors: csr_matrix -) -> Tuple[np.ndarray, csr_matrix, np.ndarray]: - """Calculate cell-wise confidence matrix with Jaccard method. - - This method measures how well each velocity vector meets the geometric constraints defined by the local neighborhood - structure. Jaccard index is calculated as the fraction of the number of the intersected set of nearest neighbors - from each cell at current expression state (X) and that from the future expression state (X + V) over the number of - the union of these two sets. - - Args: - X: The expression states of single cells (or expression states in reduced dimension, like pca, of single cells). - V: The RNA velocity of single cells (or velocity estimates projected to reduced dimension, like pca, of single - cells). Note that X, V_mat need to have the exact dimensionalities. - n_pca_components: The number of PCA components of the expression data. - n_neigh: The number of neighbors to be found. - X_neighbors: The neighbor matrix. - - Returns: - The cell wise velocity confidence metric. - """ - from sklearn.decomposition import TruncatedSVD - - transformer = TruncatedSVD(n_components=n_pca_components + 1, random_state=0) - Xt = X + V - if issparse(Xt): - Xt.data[Xt.data < 0] = 0 - Xt.data = np.log2(Xt.data + 1) - else: - Xt = np.log2(Xt + 1) - X_fit = transformer.fit(Xt) - Xt_pca = X_fit.transform(Xt)[:, 1:] - - V_neighbors, _, _, _ = umap_conn_indices_dist_embedding(Xt_pca, n_neighbors=n_neigh, return_mapper=False) - X_neighbors_, V_neighbors_ = ( - X_neighbors.dot(X_neighbors), - V_neighbors.dot(V_neighbors), - ) - union_ = X_neighbors_ + V_neighbors_ > 0 - intersect_ = mnn_from_list([X_neighbors_, V_neighbors_]) > 0 - - jaccard = (intersect_.sum(1) / union_.sum(1)).A1 if issparse(X) else intersect_.sum(1) / union_.sum(1) - - return jaccard, intersect_, union_ - - -def consensus(x: np.ndarray, y: np.ndarray) -> np.ndarray: - """Calculate the consensus with expression matrix and velocity matrix. - - Args: - x: Expression matrix (genes x cells). - y: Velocity vectors y_i for gene i. - - Returns: - The consensus matrix. - """ - x_norm, y_norm = np.linalg.norm(x), np.linalg.norm(y) - consensus = ( - einsum_correlation(x[None, :], y, type="cosine")[0, 0] * np.min([x_norm, y_norm]) / np.max([x_norm, y_norm]) - ) - - return consensus - - def gene_wise_confidence( adata: AnnData, group: str, @@ -431,3 +367,68 @@ def gene_wise_confidence( adata.var.loc[genes, "avg_mature_confidence"] = avg.loc[genes, "mature_confidence"] adata.uns["gene_wise_confidence"] = confidence + + +def jaccard( + X: np.ndarray, V: np.ndarray, n_pca_components: int, n_neigh: int, X_neighbors: csr_matrix +) -> Tuple[np.ndarray, csr_matrix, np.ndarray]: + """Calculate cell-wise confidence matrix with Jaccard method. + + This method measures how well each velocity vector meets the geometric constraints defined by the local neighborhood + structure. Jaccard index is calculated as the fraction of the number of the intersected set of nearest neighbors + from each cell at current expression state (X) and that from the future expression state (X + V) over the number of + the union of these two sets. + + Args: + X: The expression states of single cells (or expression states in reduced dimension, like pca, of single cells). + V: The RNA velocity of single cells (or velocity estimates projected to reduced dimension, like pca, of single + cells). Note that X, V_mat need to have the exact dimensionalities. + n_pca_components: The number of PCA components of the expression data. + n_neigh: The number of neighbors to be found. + X_neighbors: The neighbor matrix. + + Returns: + The cell wise velocity confidence metric. + """ + from sklearn.decomposition import TruncatedSVD + + transformer = TruncatedSVD(n_components=n_pca_components + 1, random_state=0) + Xt = X + V + if issparse(Xt): + Xt.data[Xt.data < 0] = 0 + Xt.data = np.log2(Xt.data + 1) + else: + Xt = np.log2(Xt + 1) + X_fit = transformer.fit(Xt) + Xt_pca = X_fit.transform(Xt)[:, 1:] + + V_neighbors, _, _, _ = umap_conn_indices_dist_embedding(Xt_pca, n_neighbors=n_neigh, return_mapper=False) + X_neighbors_, V_neighbors_ = ( + X_neighbors.dot(X_neighbors), + V_neighbors.dot(V_neighbors), + ) + union_ = X_neighbors_ + V_neighbors_ > 0 + intersect_ = mnn_from_list([X_neighbors_, V_neighbors_]) > 0 + + jaccard = (intersect_.sum(1) / union_.sum(1)).A1 if issparse(X) else intersect_.sum(1) / union_.sum(1) + + return jaccard, intersect_, union_ + + +def consensus(x: np.ndarray, y: np.ndarray) -> np.ndarray: + """Calculate the consensus with expression matrix and velocity matrix. + + Args: + x: Expression matrix (genes x cells). + y: Velocity vectors y_i for gene i. + + Returns: + The consensus matrix. + """ + x_norm, y_norm = np.linalg.norm(x), np.linalg.norm(y) + consensus = ( + einsum_correlation(x[None, :], y, type="cosine")[0, 0] * np.min([x_norm, y_norm]) / np.max([x_norm, y_norm]) + ) + + return consensus + From 220bbcb1ef25e568a4d754a34d2aa7404f34766c Mon Sep 17 00:00:00 2001 From: sichao Date: Tue, 21 Nov 2023 14:46:01 -0500 Subject: [PATCH 12/14] reorganize the order in more files --- dynamo/tools/moments.py | 224 ++++++++++++------------ dynamo/tools/pseudotime.py | 258 ++++++++++++++-------------- dynamo/tools/pseudotime_velocity.py | 166 +++++++++--------- dynamo/tools/psl.py | 32 ++-- dynamo/tools/recipes.py | 226 ++++++++++++------------ dynamo/tools/sampling.py | 90 +++++----- 6 files changed, 497 insertions(+), 499 deletions(-) diff --git a/dynamo/tools/moments.py b/dynamo/tools/moments.py index c09b328cc..939cf5fdc 100755 --- a/dynamo/tools/moments.py +++ b/dynamo/tools/moments.py @@ -1050,39 +1050,100 @@ def prepare_data_mix_no_splicing( # --------------------------------------------------------------------------------------------------- # moment related: +def calc_1nd_moment( + X: np.ndarray, W: np.ndarray, normalize_W: bool = True +) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]: + """Calculate first moment for the layers. + Args: + X: The layer to calculate the moment. + W: The connectivity graph that will be used for moment calculations. + normalize_W: Whether to normalize W before calculation. Defaults to True. -def stratify(arr: np.ndarray, strata: np.ndarray) -> List[np.ndarray]: - """Stratify the given array with the given reference strata. + Returns: + The first moment of the layer. + """ + if normalize_W: + if type(W) == np.ndarray: + d = np.sum(W, 1).flatten() + else: + d = np.sum(W, 1).A.flatten() + W = diags(1 / d) @ W if issparse(W) else np.diag(1 / d) @ W + return W @ X, W + else: + return W @ X + + +def calc_2nd_moment( + X: np.ndarray, + Y: np.ndarray, + W: np.ndarray, + normalize_W: bool = True, + center: bool = False, + mX: np.ndarray = None, + mY: np.ndarray = None, +) -> np.ndarray: + """Calculate the 2nd moment for the layers. Args: - arr: The array to be stratified. - strata: The reference strata vector. + X: The first layer to be used. + Y: The second layer to be used. + W: The connectivity graph that will be used for moment calculations. + normalize_W: Whether to normalize W before calculation. Defaults to True. + center: Whether to correct the center. Defaults to False. + mX: The moment matrix to correct the center. Defaults to None. + mY: The moment matrix to correct the center. Defaults to None. Returns: - A list containing the strata from the array, with each element of the list to be the components with line index - corresponding to the reference strata vector's unique elements' index. + The second moment of the layers. """ + if normalize_W: + if type(W) == np.ndarray: + d = np.sum(W, 1).flatten() + else: + d = W.sum(1).A.flatten() + W = diags(1 / d) @ W if issparse(W) else np.diag(1 / d) @ W - s = np.unique(strata) - return [arr[strata == s[i]] for i in range(len(s))] + XY = W @ elem_prod(Y, X) + + if center: + mX = calc_1nd_moment(X, W, False) if mX is None else mX + mY = calc_1nd_moment(Y, W, False) if mY is None else mY + XY = XY - elem_prod(mX, mY) + return XY -def strat_mom(arr: Union[np.ndarray, csr_matrix], strata: np.ndarray, fcn_mom: Callable) -> np.ndarray: - """Stratify the mRNA expression data and calculate its momentum. + +def calc_12_mom_labeling( + data: np.ndarray, t: np.ndarray, calculate_2_mom: bool = True +) -> Union[Tuple[np.ndarray, np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]: + """Calculate 1st and 2nd momentum for given data. Args: - arr: The mRNA expression data. - strata: The time stamp array used to stratify `arr`. - fcn_mom: The function used to calculate the momentum. + data: The normalized mRNA expression data. + t: The time stamp array. + calculate_2_mom: Whether to calculate 2nd momentum. Defaults to True. Returns: - The momentum for each stratum. + A tuple (m, [v], t_uniq) where `m` is the first momentum, `v` is the second momentum which would be returned + only if `calculate_2_mom` is true, and `t_uniq` is the unique time stamps. """ - arr = arr.A if issparse(arr) else arr - x = stratify(arr, strata) - return np.array([fcn_mom(y) for y in x]) + t_uniq = np.unique(t) + + m = np.zeros((data.shape[0], len(t_uniq))) + if calculate_2_mom: + v = np.zeros((data.shape[0], len(t_uniq))) + + for i in range(data.shape[0]): + data_ = ( + np.array(data[i].A.flatten(), dtype=float) if issparse(data) else np.array(data[i], dtype=float) + ) # consider using the `adata.obs_vector`, `adata.var_vector` methods or accessing the array directly. + m[i] = strat_mom(data_, t, np.nanmean) + if calculate_2_mom: + v[i] = strat_mom(data_, t, np.nanvar) + + return (m, v, t_uniq) if calculate_2_mom else (m, t_uniq) def calc_mom_all_genes( @@ -1116,6 +1177,39 @@ def calc_mom_all_genes( return Mn, Mo, Mt, Mr +def strat_mom(arr: Union[np.ndarray, csr_matrix], strata: np.ndarray, fcn_mom: Callable) -> np.ndarray: + """Stratify the mRNA expression data and calculate its momentum. + + Args: + arr: The mRNA expression data. + strata: The time stamp array used to stratify `arr`. + fcn_mom: The function used to calculate the momentum. + + Returns: + The momentum for each stratum. + """ + + arr = arr.A if issparse(arr) else arr + x = stratify(arr, strata) + return np.array([fcn_mom(y) for y in x]) + + +def stratify(arr: np.ndarray, strata: np.ndarray) -> List[np.ndarray]: + """Stratify the given array with the given reference strata. + + Args: + arr: The array to be stratified. + strata: The reference strata vector. + + Returns: + A list containing the strata from the array, with each element of the list to be the components with line index + corresponding to the reference strata vector's unique elements' index. + """ + + s = np.unique(strata) + return [arr[strata == s[i]] for i in range(len(s))] + + def gaussian_kernel( X: np.ndarray, nbr_idx: np.ndarray, sigma: int, k: Optional[int] = None, dists: Optional[np.ndarray] = None ) -> csr_matrix: @@ -1143,99 +1237,3 @@ def gaussian_kernel( W[i, nbr_idx[i][:k]] = np.exp(-s2_inv * dists[i][:k] ** 2) return csr_matrix(W) - - -def calc_12_mom_labeling( - data: np.ndarray, t: np.ndarray, calculate_2_mom: bool = True -) -> Union[Tuple[np.ndarray, np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]: - """Calculate 1st and 2nd momentum for given data. - - Args: - data: The normalized mRNA expression data. - t: The time stamp array. - calculate_2_mom: Whether to calculate 2nd momentum. Defaults to True. - - Returns: - A tuple (m, [v], t_uniq) where `m` is the first momentum, `v` is the second momentum which would be returned - only if `calculate_2_mom` is true, and `t_uniq` is the unique time stamps. - """ - - t_uniq = np.unique(t) - - m = np.zeros((data.shape[0], len(t_uniq))) - if calculate_2_mom: - v = np.zeros((data.shape[0], len(t_uniq))) - - for i in range(data.shape[0]): - data_ = ( - np.array(data[i].A.flatten(), dtype=float) if issparse(data) else np.array(data[i], dtype=float) - ) # consider using the `adata.obs_vector`, `adata.var_vector` methods or accessing the array directly. - m[i] = strat_mom(data_, t, np.nanmean) - if calculate_2_mom: - v[i] = strat_mom(data_, t, np.nanvar) - - return (m, v, t_uniq) if calculate_2_mom else (m, t_uniq) - - -def calc_1nd_moment( - X: np.ndarray, W: np.ndarray, normalize_W: bool = True -) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]: - """Calculate first moment for the layers. - - Args: - X: The layer to calculate the moment. - W: The connectivity graph that will be used for moment calculations. - normalize_W: Whether to normalize W before calculation. Defaults to True. - - Returns: - The first moment of the layer. - """ - if normalize_W: - if type(W) == np.ndarray: - d = np.sum(W, 1).flatten() - else: - d = np.sum(W, 1).A.flatten() - W = diags(1 / d) @ W if issparse(W) else np.diag(1 / d) @ W - return W @ X, W - else: - return W @ X - - -def calc_2nd_moment( - X: np.ndarray, - Y: np.ndarray, - W: np.ndarray, - normalize_W: bool = True, - center: bool = False, - mX: np.ndarray = None, - mY: np.ndarray = None, -) -> np.ndarray: - """Calculate the 2nd moment for the layers. - - Args: - X: The first layer to be used. - Y: The second layer to be used. - W: The connectivity graph that will be used for moment calculations. - normalize_W: Whether to normalize W before calculation. Defaults to True. - center: Whether to correct the center. Defaults to False. - mX: The moment matrix to correct the center. Defaults to None. - mY: The moment matrix to correct the center. Defaults to None. - - Returns: - The second moment of the layers. - """ - if normalize_W: - if type(W) == np.ndarray: - d = np.sum(W, 1).flatten() - else: - d = W.sum(1).A.flatten() - W = diags(1 / d) @ W if issparse(W) else np.diag(1 / d) @ W - - XY = W @ elem_prod(Y, X) - - if center: - mX = calc_1nd_moment(X, W, False) if mX is None else mX - mY = calc_1nd_moment(Y, W, False) if mY is None else mY - XY = XY - elem_prod(mX, mY) - - return XY diff --git a/dynamo/tools/pseudotime.py b/dynamo/tools/pseudotime.py index 00f643990..355b4a4c3 100755 --- a/dynamo/tools/pseudotime.py +++ b/dynamo/tools/pseudotime.py @@ -12,6 +12,135 @@ from ..dynamo_logger import main_info, main_info_insert_adata_obs +def order_cells( + adata: anndata.AnnData, + layer: str = "X", + basis: Optional[str] = None, + root_state: Optional[int] = None, + init_cells: Optional[Union[List, np.ndarray, pd.Index]] = None, + reverse: bool = False, + maxIter: int = 10, + sigma: float = 0.001, + gamma: float = 10.0, + eps: int = 0, + dim: int = 2, + Lambda: Optional[float] = None, + ncenter: Optional[int] = None, + **kwargs, +) -> anndata.AnnData: + """Order the cells based on the calculated pseudotime derived from the principal graph. + + Learns a "trajectory" describing the biological process the cells are going through, and calculates where each cell + falls within that trajectory. The trajectory will be composed of segments. The cells from a segment will share the + same value of state. One of these segments will be selected as the root of the trajectory. The most distal cell on + that segment will be chosen as the "first" cell in the trajectory, and will have a pseudotime value of zero. Then + the function will then "walk" along the trajectory, and as it encounters additional cells, it will assign them + increasingly large values of pseudotime based on distance. + + Args: + adata: The anndata object. + layer: The layer used to order the cells. + basis: The basis that indicates the data after dimension reduction. + root_state: The specific state for selecting the root cell. + init_cells: The index to search for root cells. If provided, root_state will be ignored. + reverse: Whether to reverse the selection of the root cell. + maxIter: The max number of iterations. + sigma: The bandwidth parameter. + gamma: Regularization parameter for k-means. + eps: The threshold of convergency to stop the iteration. Defaults to 0. + dim: The number of dimensions reduced to. Defaults to 2. + Lambda: Regularization parameter for inverse praph embedding. Defaults to 1.0. + ncenter: The number of center genes to be considered. If None, all genes would be considered. Defaults to None. + kwargs: Additional keyword arguments. + + Returns: + The anndata object updated with pseudotime, cell order state and other necessary information. + """ + import igraph as ig + + main_info("Ordering cells based on pseudotime...") + if basis is None: + X = adata.layers["X_" + layer].T if layer != "X" else adata.X.T + X = log1p_(adata, X) + else: + X = adata.obsm["X_" + basis] + + if "cell_order" not in adata.uns.keys(): + adata.uns["cell_order"] = {} + adata.uns["cell_order"]["root_cell"] = None + + DDRTree_kwargs = { + "maxIter": maxIter, + "sigma": sigma, + "gamma": gamma, + "eps": eps, + "dim": dim, + "Lambda": Lambda if Lambda else 5 * X.shape[1], + "ncenter": ncenter if ncenter else _cal_ncenter(X.shape[1]), + } + DDRTree_kwargs.update(kwargs) + + Z, Y, stree, R, W, Q, C, objs = DDRTree(X, **DDRTree_kwargs) + + principal_graph = stree + dp = distance.squareform(distance.pdist(Y.T)) + mst = minimum_spanning_tree(principal_graph) + + adata.uns["cell_order"]["cell_order_method"] = "DDRTree" + adata.uns["cell_order"]["Z"] = Z + adata.uns["cell_order"]["Y"] = Y + adata.uns["cell_order"]["stree"] = stree + adata.uns["cell_order"]["R"] = R + adata.uns["cell_order"]["W"] = W + adata.uns["cell_order"]["minSpanningTree"] = mst + adata.uns["cell_order"]["centers_minSpanningTree"] = mst + + root_cell = select_root_cell(adata, Z=Z, root_state=root_state, init_cells=init_cells, reverse=reverse) + cc_ordering = get_order_from_DDRTree(dp=dp, mst=mst, root_cell=root_cell) + + ( + cellPairwiseDistances, + pr_graph_cell_proj_dist, + pr_graph_cell_proj_closest_vertex, + pr_graph_cell_proj_tree + ) = project2MST(mst, Z, Y, project_point_to_line_segment) + + adata.uns["cell_order"]["root_cell"] = root_cell + adata.uns["cell_order"]["centers_order"] = cc_ordering["orders"].values + adata.uns["cell_order"]["centers_parent"] = cc_ordering["parent"].values + adata.uns["cell_order"]["minSpanningTree"] = pr_graph_cell_proj_tree + adata.uns["cell_order"]["pr_graph_cell_proj_closest_vertex"] = pr_graph_cell_proj_closest_vertex + + cells_mapped_to_graph_root = np.where(pr_graph_cell_proj_closest_vertex == root_cell)[0] + # avoid the issue of multiple cells projected to the same point on the principal graph + if len(cells_mapped_to_graph_root) == 0: + cells_mapped_to_graph_root = [root_cell] + + pr_graph_cell_proj_tree_graph = ig.Graph.Weighted_Adjacency(matrix=pr_graph_cell_proj_tree) + tip_leaves = [v.index for v in pr_graph_cell_proj_tree_graph.vs.select(_degree=1)] + root_cell_candidates = np.intersect1d(cells_mapped_to_graph_root, tip_leaves) + + if len(root_cell_candidates) == 0: + root_cell = select_root_cell(adata, Z=Z, root_state=root_state, init_cells=init_cells, reverse=reverse, map_to_tree=False) + else: + root_cell = root_cell_candidates[0] + + cc_ordering_new_pseudotime = get_order_from_DDRTree(dp=cellPairwiseDistances, mst=pr_graph_cell_proj_tree, root_cell=root_cell) # re-calculate the pseudotime again + + adata.uns["cell_order"]["root_cell"] = root_cell + adata.obs["Pseudotime"] = cc_ordering_new_pseudotime["pseudo_time"].values + adata.uns["cell_order"]["parent"] = cc_ordering_new_pseudotime["parent"] + adata.uns["cell_order"]["branch_points"] = np.array(pr_graph_cell_proj_tree_graph.vs.select(_degree_gt=2)) + main_info_insert_adata_obs("Pseudotime") + + if root_state is None: + closest_vertex = pr_graph_cell_proj_closest_vertex + adata.obs["cell_pseudo_state"] = cc_ordering.loc[closest_vertex, "cell_pseudo_state"].values + main_info_insert_adata_obs("cell_pseudo_state") + + return adata + + def get_order_from_DDRTree(dp: np.ndarray, mst: np.ndarray, root_cell: int) -> pd.DataFrame: """Calculates the order of cells based on a minimum spanning tree and a distance matrix. @@ -278,135 +407,6 @@ def select_root_cell( return root_cell -def order_cells( - adata: anndata.AnnData, - layer: str = "X", - basis: Optional[str] = None, - root_state: Optional[int] = None, - init_cells: Optional[Union[List, np.ndarray, pd.Index]] = None, - reverse: bool = False, - maxIter: int = 10, - sigma: float = 0.001, - gamma: float = 10.0, - eps: int = 0, - dim: int = 2, - Lambda: Optional[float] = None, - ncenter: Optional[int] = None, - **kwargs, -) -> anndata.AnnData: - """Order the cells based on the calculated pseudotime derived from the principal graph. - - Learns a "trajectory" describing the biological process the cells are going through, and calculates where each cell - falls within that trajectory. The trajectory will be composed of segments. The cells from a segment will share the - same value of state. One of these segments will be selected as the root of the trajectory. The most distal cell on - that segment will be chosen as the "first" cell in the trajectory, and will have a pseudotime value of zero. Then - the function will then "walk" along the trajectory, and as it encounters additional cells, it will assign them - increasingly large values of pseudotime based on distance. - - Args: - adata: The anndata object. - layer: The layer used to order the cells. - basis: The basis that indicates the data after dimension reduction. - root_state: The specific state for selecting the root cell. - init_cells: The index to search for root cells. If provided, root_state will be ignored. - reverse: Whether to reverse the selection of the root cell. - maxIter: The max number of iterations. - sigma: The bandwidth parameter. - gamma: Regularization parameter for k-means. - eps: The threshold of convergency to stop the iteration. Defaults to 0. - dim: The number of dimensions reduced to. Defaults to 2. - Lambda: Regularization parameter for inverse praph embedding. Defaults to 1.0. - ncenter: The number of center genes to be considered. If None, all genes would be considered. Defaults to None. - kwargs: Additional keyword arguments. - - Returns: - The anndata object updated with pseudotime, cell order state and other necessary information. - """ - import igraph as ig - - main_info("Ordering cells based on pseudotime...") - if basis is None: - X = adata.layers["X_" + layer].T if layer != "X" else adata.X.T - X = log1p_(adata, X) - else: - X = adata.obsm["X_" + basis] - - if "cell_order" not in adata.uns.keys(): - adata.uns["cell_order"] = {} - adata.uns["cell_order"]["root_cell"] = None - - DDRTree_kwargs = { - "maxIter": maxIter, - "sigma": sigma, - "gamma": gamma, - "eps": eps, - "dim": dim, - "Lambda": Lambda if Lambda else 5 * X.shape[1], - "ncenter": ncenter if ncenter else _cal_ncenter(X.shape[1]), - } - DDRTree_kwargs.update(kwargs) - - Z, Y, stree, R, W, Q, C, objs = DDRTree(X, **DDRTree_kwargs) - - principal_graph = stree - dp = distance.squareform(distance.pdist(Y.T)) - mst = minimum_spanning_tree(principal_graph) - - adata.uns["cell_order"]["cell_order_method"] = "DDRTree" - adata.uns["cell_order"]["Z"] = Z - adata.uns["cell_order"]["Y"] = Y - adata.uns["cell_order"]["stree"] = stree - adata.uns["cell_order"]["R"] = R - adata.uns["cell_order"]["W"] = W - adata.uns["cell_order"]["minSpanningTree"] = mst - adata.uns["cell_order"]["centers_minSpanningTree"] = mst - - root_cell = select_root_cell(adata, Z=Z, root_state=root_state, init_cells=init_cells, reverse=reverse) - cc_ordering = get_order_from_DDRTree(dp=dp, mst=mst, root_cell=root_cell) - - ( - cellPairwiseDistances, - pr_graph_cell_proj_dist, - pr_graph_cell_proj_closest_vertex, - pr_graph_cell_proj_tree - ) = project2MST(mst, Z, Y, project_point_to_line_segment) - - adata.uns["cell_order"]["root_cell"] = root_cell - adata.uns["cell_order"]["centers_order"] = cc_ordering["orders"].values - adata.uns["cell_order"]["centers_parent"] = cc_ordering["parent"].values - adata.uns["cell_order"]["minSpanningTree"] = pr_graph_cell_proj_tree - adata.uns["cell_order"]["pr_graph_cell_proj_closest_vertex"] = pr_graph_cell_proj_closest_vertex - - cells_mapped_to_graph_root = np.where(pr_graph_cell_proj_closest_vertex == root_cell)[0] - # avoid the issue of multiple cells projected to the same point on the principal graph - if len(cells_mapped_to_graph_root) == 0: - cells_mapped_to_graph_root = [root_cell] - - pr_graph_cell_proj_tree_graph = ig.Graph.Weighted_Adjacency(matrix=pr_graph_cell_proj_tree) - tip_leaves = [v.index for v in pr_graph_cell_proj_tree_graph.vs.select(_degree=1)] - root_cell_candidates = np.intersect1d(cells_mapped_to_graph_root, tip_leaves) - - if len(root_cell_candidates) == 0: - root_cell = select_root_cell(adata, Z=Z, root_state=root_state, init_cells=init_cells, reverse=reverse, map_to_tree=False) - else: - root_cell = root_cell_candidates[0] - - cc_ordering_new_pseudotime = get_order_from_DDRTree(dp=cellPairwiseDistances, mst=pr_graph_cell_proj_tree, root_cell=root_cell) # re-calculate the pseudotime again - - adata.uns["cell_order"]["root_cell"] = root_cell - adata.obs["Pseudotime"] = cc_ordering_new_pseudotime["pseudo_time"].values - adata.uns["cell_order"]["parent"] = cc_ordering_new_pseudotime["parent"] - adata.uns["cell_order"]["branch_points"] = np.array(pr_graph_cell_proj_tree_graph.vs.select(_degree_gt=2)) - main_info_insert_adata_obs("Pseudotime") - - if root_state is None: - closest_vertex = pr_graph_cell_proj_closest_vertex - adata.obs["cell_pseudo_state"] = cc_ordering.loc[closest_vertex, "cell_pseudo_state"].values - main_info_insert_adata_obs("cell_pseudo_state") - - return adata - - def _cal_ncenter(ncells, ncells_limit=100): """Calculate the number of centers genes to be considered. diff --git a/dynamo/tools/pseudotime_velocity.py b/dynamo/tools/pseudotime_velocity.py index 35bafb841..7b18b6311 100644 --- a/dynamo/tools/pseudotime_velocity.py +++ b/dynamo/tools/pseudotime_velocity.py @@ -15,89 +15,6 @@ from .utils import projection_with_transition_matrix -def gradient(E: Union[csr_matrix, np.ndarray], f: np.ndarray, tol: float = 1e-5) -> csr_matrix: - """Calculate the graph's gradient. - - Args: - E: The adjacency matrix of the graph. - f: The pseudotime matrix. - tol: The tolerance of considering a value to be non-zero. Defaults to 1e-5. - - Returns: - The gradient of the graph. - """ - if issparse(E): - row, col = E.nonzero() - val = E.data - else: - row, col = np.where(E != 0) - val = E[E != 0] - - G_i, G_j, G_val = np.zeros_like(row), np.zeros_like(col), np.zeros_like(val) - - for ind, i, j, k in zip(np.arange(len(row)), list(row), list(col), list(val)): - if i != j and np.abs(k) > tol: - G_i[ind], G_j[ind] = i, j - G_val[ind] = f[j] - f[i] - - valid_ind = G_val != 0 - G = csr_matrix((G_val[valid_ind], (G_i[valid_ind], G_j[valid_ind])), shape=E.shape) - G.eliminate_zeros() - - return G - - -def laplacian(E: Union[csr_matrix, np.ndarray], convention: Literal["graph", "diffusion"] = "graph") -> csr_matrix: - """Calculate the laplacian of the given graph (here the adjacency matrix). - - Args: - E: The adjacency matrix. - convention: The convention of results. Could be either "graph" or "diffusion". If "diffusion" is specified, the - negative of graph laplacian would be returned. Defaults to "graph". - - Returns: - The laplacian matrix. - - Raises: - NotImplementedError: invalid `convention`. - """ - if issparse(E): - A = E.copy() - A.data = np.ones_like(A.data) - L = diags(A.sum(0).A1, 0) - A - else: - A = np.sign(E) - L = np.diag(np.sum(A, 0)) - A - if convention == "graph": - pass - elif convention == "diffusion": - L = -L - else: - raise NotImplementedError("The convention is not implemented. ") - - L = csr_matrix(L) - - return L - - -def pseudotime_transition(E: np.ndarray, pseudotime: np.ndarray, laplace_weight: float = 10) -> csr_matrix: - """Calculate the transition graph with pseudotime gradient. - - Args: - E: The adjacency matrix. - pseudotime: The pseudo time value matrix. - laplace_weight: The weight of adding laplacian to gradient during calculation of transition graph. Defaults to - 10. - - Returns: - The pseudo-based transition matrix. - """ - grad = gradient(E, pseudotime) - lap = laplacian(E, convention="diffusion") - T = grad + laplace_weight * lap - return T - - def pseudotime_velocity( adata: anndata.AnnData, pseudotime: str = "pseudotime", @@ -237,3 +154,86 @@ def pseudotime_velocity( logger.info_insert_adata("gamma", "var", indent_level=2) adata.varm["pseudotime_vel_params"] = np.zeros((adata.n_vars, 2)) adata.uns["pseudotime_vel_params_names"] = ["gamma", "gamma_b"] + + +def pseudotime_transition(E: np.ndarray, pseudotime: np.ndarray, laplace_weight: float = 10) -> csr_matrix: + """Calculate the transition graph with pseudotime gradient. + + Args: + E: The adjacency matrix. + pseudotime: The pseudo time value matrix. + laplace_weight: The weight of adding laplacian to gradient during calculation of transition graph. Defaults to + 10. + + Returns: + The pseudo-based transition matrix. + """ + grad = gradient(E, pseudotime) + lap = laplacian(E, convention="diffusion") + T = grad + laplace_weight * lap + return T + + +def gradient(E: Union[csr_matrix, np.ndarray], f: np.ndarray, tol: float = 1e-5) -> csr_matrix: + """Calculate the graph's gradient. + + Args: + E: The adjacency matrix of the graph. + f: The pseudotime matrix. + tol: The tolerance of considering a value to be non-zero. Defaults to 1e-5. + + Returns: + The gradient of the graph. + """ + if issparse(E): + row, col = E.nonzero() + val = E.data + else: + row, col = np.where(E != 0) + val = E[E != 0] + + G_i, G_j, G_val = np.zeros_like(row), np.zeros_like(col), np.zeros_like(val) + + for ind, i, j, k in zip(np.arange(len(row)), list(row), list(col), list(val)): + if i != j and np.abs(k) > tol: + G_i[ind], G_j[ind] = i, j + G_val[ind] = f[j] - f[i] + + valid_ind = G_val != 0 + G = csr_matrix((G_val[valid_ind], (G_i[valid_ind], G_j[valid_ind])), shape=E.shape) + G.eliminate_zeros() + + return G + + +def laplacian(E: Union[csr_matrix, np.ndarray], convention: Literal["graph", "diffusion"] = "graph") -> csr_matrix: + """Calculate the laplacian of the given graph (here the adjacency matrix). + + Args: + E: The adjacency matrix. + convention: The convention of results. Could be either "graph" or "diffusion". If "diffusion" is specified, the + negative of graph laplacian would be returned. Defaults to "graph". + + Returns: + The laplacian matrix. + + Raises: + NotImplementedError: invalid `convention`. + """ + if issparse(E): + A = E.copy() + A.data = np.ones_like(A.data) + L = diags(A.sum(0).A1, 0) - A + else: + A = np.sign(E) + L = np.diag(np.sum(A, 0)) - A + if convention == "graph": + pass + elif convention == "diffusion": + L = -L + else: + raise NotImplementedError("The convention is not implemented. ") + + L = csr_matrix(L) + + return L diff --git a/dynamo/tools/psl.py b/dynamo/tools/psl.py index 28e4def5e..94fc92328 100755 --- a/dynamo/tools/psl.py +++ b/dynamo/tools/psl.py @@ -14,22 +14,6 @@ # from scikits.sparse.cholmod import cholesky -def diag_mat(values: List[int]): - """Returns a diagonal matrix with the given values on the diagonal. - - Args: - values: A list of values to place on the diagonal of the matrix. - - Returns: - A diagonal matrix with the given values on the diagonal. - """ - - mat = np.zeros((len(values), len(values))) - np.fill_diagonal(mat, values) - - return mat - - def psl( Y: np.ndarray, sG: Optional[csr_matrix] = None, @@ -200,3 +184,19 @@ def psl( Z = np.dot(U, tmp) return (S, Z) + + +def diag_mat(values: List[int]): + """Returns a diagonal matrix with the given values on the diagonal. + + Args: + values: A list of values to place on the diagonal of the matrix. + + Returns: + A diagonal matrix with the given values on the diagonal. + """ + + mat = np.zeros((len(values), len(values))) + np.fill_diagonal(mat, values) + + return mat diff --git a/dynamo/tools/recipes.py b/dynamo/tools/recipes.py index 7bc25aecf..5ebe839b3 100644 --- a/dynamo/tools/recipes.py +++ b/dynamo/tools/recipes.py @@ -20,7 +20,8 @@ # add recipe_csc_data() -def recipe_kin_data( +# support using just spliced/unspliced/new/total 4 layers, as well as uu, ul, su, sl layers +def recipe_one_shot_data( adata: AnnData, tkey: Optional[str] = None, reset_X: bool = True, @@ -30,22 +31,23 @@ def recipe_kin_data( keep_filtered_cells: Optional[bool] = None, keep_filtered_genes: Optional[bool] = None, keep_raw_layers: Optional[bool] = None, + one_shot_method: str = "sci-fate", del_2nd_moments: Optional[bool] = None, ekey: str = "M_t", vkey: str = "velocity_T", basis: str = "umap", - rm_kwargs: Dict["str", Any] = {}, + rm_kwargs: Dict[str, Any] = {}, ) -> AnnData: - """An analysis recipe that properly pre-processes different layers for an kinetics experiment with both labeling and - splicing or only labeling data. + """An analysis recipe that properly pre-processes different layers for a one-shot experiment with both labeling and + splicing data. Args: - adata: An AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four + adata: AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four different layers. tkey: The column key for the labeling time of cells in .obs. Used for labeling based scRNA-seq data (will also - support for conventional scRNA-seq data). Note that `tkey` will be saved to adata.uns['pp']['tkey'] and used - in `dyn.tl.dynamics` in which when `group` is None, `tkey` will also be used for calculating 1st/2st moment - or covariance. We recommend to use hour as the unit of `time`. Defaults to None. + support for conventional scRNA-seq data). Note that `tkey` will be saved to `adata.uns['pp']['tkey']` and + used in `dyn.tl.dynamics` in which when `group` is None, `tkey` will also be used for calculating 1st/2nd + moment or covariance. We recommend to use hour as the unit of `time`. Defaults to None. reset_X: Whether do you want to let dynamo reset `adata.X` data based on layers stored in your experiment. One critical functionality of dynamo is about visualizing RNA velocity vector flows which requires proper data into which the high dimensional RNA velocity vectors will be projected. @@ -64,6 +66,8 @@ def recipe_kin_data( `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. keep_raw_layers: Whether to keep layers with raw measurements in the returned adata object. Used in `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. + one_shot_method: The method to use for calculate the absolute labeling and splicing velocity for the one-shot + data of use. Defaults to "sci-fate". del_2nd_moments: Whether to remove second moments or covariances. Argument used for `dynamics` function. If None, would be set according to `DynamoAdataConfig`. Defaults to None. ekey: The dictionary key that corresponds to the gene expression in the layer attribute. ekey and vkey will be @@ -75,7 +79,7 @@ def recipe_kin_data( rm_kwargs: Other kwargs passed into the pp.recipe_monocle function. Defaults to {}. Raises: - Exception: The recipe is only applicable to kinetics experiment datasets with labeling data. + Exception: the recipe is only applicable to kinetics experiment datasets with labeling data. Returns: An updated adata object that went through a proper and typical time-resolved RNA velocity analysis. @@ -111,7 +115,7 @@ def recipe_kin_data( ) # Preprocessing - preprocessor = Preprocessor(cell_cycle_score_enable=True) + preprocessor = Preprocessor() preprocessor.config_monocle_recipe(adata, n_top_genes=n_top_genes) preprocessor.size_factor_kwargs.update( { @@ -131,15 +135,14 @@ def recipe_kin_data( preprocessor.select_genes_kwargs["keep_filtered"] = keep_filtered_genes if reset_X: - reset_adata_X(adata, experiment_type="kin", has_labeling=has_labeling, has_splicing=has_splicing) - preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="kin") + reset_adata_X(adata, experiment_type="one-shot", has_labeling=has_labeling, has_splicing=has_splicing) + preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="one-shot") if not keep_raw_layers: del_raw_layers(adata) if has_splicing and has_labeling: # new, total (and uu, ul, su, sl if existed) layers will be normalized with size factor calculated with total # layers spliced / unspliced layers will be normalized independently. - tkey = adata.uns["pp"]["tkey"] # first calculate moments for labeling data relevant layers using total based connectivity graph moments(adata, group=tkey, layers=layers) @@ -161,20 +164,30 @@ def recipe_kin_data( moments(adata, conn=conn, layers=["X_spliced", "X_unspliced"]) # then perform kinetic estimations with properly preprocessed layers for either the labeling or the splicing # data - dynamics(adata, model="deterministic", est_method="twostep", del_2nd_moments=del_2nd_moments) + dynamics( + adata, + model="deterministic", + one_shot_method=one_shot_method, + del_2nd_moments=del_2nd_moments, + ) # then perform dimension reduction reduceDimension(adata, reduction_method=basis) # lastly, project RNA velocity to low dimensional embedding. cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) else: - dynamics(adata, model="deterministic", est_method="twostep", del_2nd_moments=del_2nd_moments) + dynamics( + adata, + model="deterministic", + one_shot_method=one_shot_method, + del_2nd_moments=del_2nd_moments, + ) reduceDimension(adata, reduction_method=basis) - cell_velocities(adata, basis=basis) + cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) return adata -def recipe_deg_data( +def recipe_kin_data( adata: AnnData, tkey: Optional[str] = None, reset_X: bool = True, @@ -184,15 +197,14 @@ def recipe_deg_data( keep_filtered_cells: Optional[bool] = None, keep_filtered_genes: Optional[bool] = None, keep_raw_layers: Optional[bool] = None, - del_2nd_moments: Optional[bool] = True, - fraction_for_deg: bool = False, - ekey: str = "M_s", - vkey: str = "velocity_S", + del_2nd_moments: Optional[bool] = None, + ekey: str = "M_t", + vkey: str = "velocity_T", basis: str = "umap", - rm_kwargs: Dict[str, Any] = {}, -): - """An analysis recipe that properly pre-processes different layers for a degradation experiment with both - labeling and splicing data or only labeling. Functions need to be updated. + rm_kwargs: Dict["str", Any] = {}, +) -> AnnData: + """An analysis recipe that properly pre-processes different layers for an kinetics experiment with both labeling and + splicing or only labeling data. Args: adata: An AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four @@ -221,12 +233,10 @@ def recipe_deg_data( `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. del_2nd_moments: Whether to remove second moments or covariances. Argument used for `dynamics` function. If None, would be set according to `DynamoAdataConfig`. Defaults to None. - fraction_for_deg: Whether to use the fraction of labeled RNA instead of the raw labeled RNA to estimate the - degradation parameter. Defaults to False. ekey: The dictionary key that corresponds to the gene expression in the layer attribute. ekey and vkey will be - automatically detected from the adata object. Parameters required by `cell_velocities`. Defaults to "M_s". + automatically detected from the adata object. Parameters required by `cell_velocities`. Defaults to "M_t". vkey: The dictionary key that corresponds to the estimated velocity values in the layers attribute. Parameters - required by `cell_velocities` Defaults to "velocity_S". + required by `cell_velocities` Defaults to "velocity_T". basis: The dictionary key that corresponds to the reduced dimension in `.obsm` attribute. Can be `X_spliced_umap` or `X_total_umap`, etc. Parameters required by `cell_velocities`. Defaults to "umap". rm_kwargs: Other kwargs passed into the pp.recipe_monocle function. Defaults to {}. @@ -249,6 +259,9 @@ def recipe_deg_data( keep_raw_layers = DynamoAdataConfig.use_default_var_if_none( keep_raw_layers, DynamoAdataConfig.RECIPE_KEEP_RAW_LAYERS_KEY ) + del_2nd_moments = DynamoAdataConfig.use_default_var_if_none( + del_2nd_moments, DynamoAdataConfig.RECIPE_DEL_2ND_MOMENTS_KEY + ) has_splicing, has_labeling, splicing_labeling, _ = detect_experiment_datatype(adata) @@ -264,7 +277,8 @@ def recipe_deg_data( "layers." ) - preprocessor = Preprocessor() + # Preprocessing + preprocessor = Preprocessor(cell_cycle_score_enable=True) preprocessor.config_monocle_recipe(adata, n_top_genes=n_top_genes) preprocessor.size_factor_kwargs.update( { @@ -284,65 +298,50 @@ def recipe_deg_data( preprocessor.select_genes_kwargs["keep_filtered"] = keep_filtered_genes if reset_X: - reset_adata_X(adata, experiment_type="deg", has_labeling=has_labeling, has_splicing=has_splicing) - preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="deg") + reset_adata_X(adata, experiment_type="kin", has_labeling=has_labeling, has_splicing=has_splicing) + preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="kin") if not keep_raw_layers: del_raw_layers(adata) if has_splicing and has_labeling: # new, total (and uu, ul, su, sl if existed) layers will be normalized with size factor calculated with total # layers spliced / unspliced layers will be normalized independently. + tkey = adata.uns["pp"]["tkey"] - # first calculate moments for spliced related layers using spliced based connectivity graph - moments(adata, layers=["X_spliced", "X_unspliced"]) + # first calculate moments for labeling data relevant layers using total based connectivity graph + moments(adata, group=tkey, layers=layers) - # then calculate moments for labeling data relevant layers using total based connectivity graph - # first get X_total based pca embedding - CM = np.log1p(adata[:, adata.var.use_for_pca].layers["X_total"].A) + # then we want to calculate moments for spliced and unspliced layers based on connectivity graph from spliced + # data. + # first get X_spliced based pca embedding + CM = np.log1p(adata[:, adata.var.use_for_pca].layers["X_spliced"].A) cm_genesums = CM.sum(axis=0) valid_ind = np.logical_and(np.isfinite(cm_genesums), cm_genesums != 0) valid_ind = np.array(valid_ind).flatten() - pca(adata, CM[:, valid_ind], pca_key="X_total_pca") + + pca(adata, CM[:, valid_ind], pca_key="X_spliced_pca") # then get neighbors graph based on X_spliced_pca - neighbors(adata, X_data=adata.obsm["X_total_pca"], layer="X_total") + neighbors(adata, X_data=adata.obsm["X_spliced_pca"], layer="X_spliced") # then normalize neighbors graph so that each row sums up to be 1 conn = normalize_knn_graph(adata.obsp["connectivities"] > 0) - moments(adata, conn=conn, group=tkey, layers=layers) - + # then calculate moments for spliced related layers using spliced based connectivity graph + moments(adata, conn=conn, layers=["X_spliced", "X_unspliced"]) # then perform kinetic estimations with properly preprocessed layers for either the labeling or the splicing # data - dynamics( - adata, - model="deterministic", - est_method="twostep", - del_2nd_moments=del_2nd_moments, - fraction_for_deg=fraction_for_deg, - ) + dynamics(adata, model="deterministic", est_method="twostep", del_2nd_moments=del_2nd_moments) # then perform dimension reduction reduceDimension(adata, reduction_method=basis) # lastly, project RNA velocity to low dimensional embedding. - try: - set_transition_genes(adata) - cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) - except BaseException: - vel_params_df = get_vel_params(adata) - cell_velocities( - adata, - min_r2=vel_params_df.gamma_r2.min(), - enforce=True, - vkey=vkey, - ekey=ekey, - basis=basis, - ) - + cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) else: - dynamics(adata, model="deterministic", del_2nd_moments=del_2nd_moments, fraction_for_deg=fraction_for_deg) + dynamics(adata, model="deterministic", est_method="twostep", del_2nd_moments=del_2nd_moments) reduceDimension(adata, reduction_method=basis) + cell_velocities(adata, basis=basis) return adata -def recipe_mix_kin_deg_data( +def recipe_deg_data( adata: AnnData, tkey: Optional[str] = None, reset_X: bool = True, @@ -352,22 +351,23 @@ def recipe_mix_kin_deg_data( keep_filtered_cells: Optional[bool] = None, keep_filtered_genes: Optional[bool] = None, keep_raw_layers: Optional[bool] = None, - del_2nd_moments: Optional[bool] = None, - ekey: str = "M_t", - vkey: str = "velocity_T", + del_2nd_moments: Optional[bool] = True, + fraction_for_deg: bool = False, + ekey: str = "M_s", + vkey: str = "velocity_S", basis: str = "umap", rm_kwargs: Dict[str, Any] = {}, ): - """An analysis recipe that properly pre-processes different layers for a mixture kinetics and degradation - experiment with both labeling and splicing or only labeling data. + """An analysis recipe that properly pre-processes different layers for a degradation experiment with both + labeling and splicing data or only labeling. Functions need to be updated. Args: adata: An AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four different layers. tkey: The column key for the labeling time of cells in .obs. Used for labeling based scRNA-seq data (will also - support for conventional scRNA-seq data). Note that `tkey` will be saved to `adata.uns['pp']['tkey']` and - used in `dyn.tl.dynamics` in which when `group` is None, `tkey` will also be used for calculating 1st/2nd - moment or covariance. We recommend to use hour as the unit of `time`. Defaults to None. + support for conventional scRNA-seq data). Note that `tkey` will be saved to adata.uns['pp']['tkey'] and used + in `dyn.tl.dynamics` in which when `group` is None, `tkey` will also be used for calculating 1st/2st moment + or covariance. We recommend to use hour as the unit of `time`. Defaults to None. reset_X: Whether do you want to let dynamo reset `adata.X` data based on layers stored in your experiment. One critical functionality of dynamo is about visualizing RNA velocity vector flows which requires proper data into which the high dimensional RNA velocity vectors will be projected. @@ -388,16 +388,18 @@ def recipe_mix_kin_deg_data( `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. del_2nd_moments: Whether to remove second moments or covariances. Argument used for `dynamics` function. If None, would be set according to `DynamoAdataConfig`. Defaults to None. + fraction_for_deg: Whether to use the fraction of labeled RNA instead of the raw labeled RNA to estimate the + degradation parameter. Defaults to False. ekey: The dictionary key that corresponds to the gene expression in the layer attribute. ekey and vkey will be - automatically detected from the adata object. Parameters required by `cell_velocities`. Defaults to "M_t". + automatically detected from the adata object. Parameters required by `cell_velocities`. Defaults to "M_s". vkey: The dictionary key that corresponds to the estimated velocity values in the layers attribute. Parameters - required by `cell_velocities` Defaults to "velocity_T". + required by `cell_velocities` Defaults to "velocity_S". basis: The dictionary key that corresponds to the reduced dimension in `.obsm` attribute. Can be `X_spliced_umap` or `X_total_umap`, etc. Parameters required by `cell_velocities`. Defaults to "umap". rm_kwargs: Other kwargs passed into the pp.recipe_monocle function. Defaults to {}. Raises: - Exception: the recipe is only applicable to kinetics experiment datasets with labeling data. + Exception: The recipe is only applicable to kinetics experiment datasets with labeling data. Returns: An updated adata object that went through a proper and typical time-resolved RNA velocity analysis. @@ -414,9 +416,6 @@ def recipe_mix_kin_deg_data( keep_raw_layers = DynamoAdataConfig.use_default_var_if_none( keep_raw_layers, DynamoAdataConfig.RECIPE_KEEP_RAW_LAYERS_KEY ) - del_2nd_moments = DynamoAdataConfig.use_default_var_if_none( - del_2nd_moments, DynamoAdataConfig.RECIPE_DEL_2ND_MOMENTS_KEY - ) has_splicing, has_labeling, splicing_labeling, _ = detect_experiment_datatype(adata) @@ -432,7 +431,6 @@ def recipe_mix_kin_deg_data( "layers." ) - # Preprocessing preprocessor = Preprocessor() preprocessor.config_monocle_recipe(adata, n_top_genes=n_top_genes) preprocessor.size_factor_kwargs.update( @@ -453,8 +451,8 @@ def recipe_mix_kin_deg_data( preprocessor.select_genes_kwargs["keep_filtered"] = keep_filtered_genes if reset_X: - reset_adata_X(adata, experiment_type="mix_pulse_chase", has_labeling=has_labeling, has_splicing=has_splicing) - preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="mix_pulse_chase") + reset_adata_X(adata, experiment_type="deg", has_labeling=has_labeling, has_splicing=has_splicing) + preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="deg") if not keep_raw_layers: del_raw_layers(adata) @@ -462,24 +460,22 @@ def recipe_mix_kin_deg_data( # new, total (and uu, ul, su, sl if existed) layers will be normalized with size factor calculated with total # layers spliced / unspliced layers will be normalized independently. tkey = adata.uns["pp"]["tkey"] - # first calculate moments for labeling data relevant layers using total based connectivity graph - moments(adata, group=tkey, layers=layers) + # first calculate moments for spliced related layers using spliced based connectivity graph + moments(adata, layers=["X_spliced", "X_unspliced"]) - # then we want to calculate moments for spliced and unspliced layers based on connectivity graph from spliced - # data. - # first get X_spliced based pca embedding - CM = np.log1p(adata[:, adata.var.use_for_pca].layers["X_spliced"].A) + # then calculate moments for labeling data relevant layers using total based connectivity graph + # first get X_total based pca embedding + CM = np.log1p(adata[:, adata.var.use_for_pca].layers["X_total"].A) cm_genesums = CM.sum(axis=0) valid_ind = np.logical_and(np.isfinite(cm_genesums), cm_genesums != 0) valid_ind = np.array(valid_ind).flatten() - - pca(adata, CM[:, valid_ind], pca_key="X_spliced_pca") + pca(adata, CM[:, valid_ind], pca_key="X_total_pca") # then get neighbors graph based on X_spliced_pca - neighbors(adata, X_data=adata.obsm["X_spliced_pca"], layer="X_spliced") + neighbors(adata, X_data=adata.obsm["X_total_pca"], layer="X_total") # then normalize neighbors graph so that each row sums up to be 1 conn = normalize_knn_graph(adata.obsp["connectivities"] > 0) - # then calculate moments for spliced related layers using spliced based connectivity graph - moments(adata, conn=conn, layers=["X_spliced", "X_unspliced"]) + moments(adata, conn=conn, group=tkey, layers=layers) + # then perform kinetic estimations with properly preprocessed layers for either the labeling or the splicing # data dynamics( @@ -487,26 +483,33 @@ def recipe_mix_kin_deg_data( model="deterministic", est_method="twostep", del_2nd_moments=del_2nd_moments, + fraction_for_deg=fraction_for_deg, ) # then perform dimension reduction reduceDimension(adata, reduction_method=basis) # lastly, project RNA velocity to low dimensional embedding. - cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) + try: + set_transition_genes(adata) + cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) + except BaseException: + vel_params_df = get_vel_params(adata) + cell_velocities( + adata, + min_r2=vel_params_df.gamma_r2.min(), + enforce=True, + vkey=vkey, + ekey=ekey, + basis=basis, + ) + else: - dynamics( - adata, - model="deterministic", - est_method="twostep", - del_2nd_moments=del_2nd_moments, - ) + dynamics(adata, model="deterministic", del_2nd_moments=del_2nd_moments, fraction_for_deg=fraction_for_deg) reduceDimension(adata, reduction_method=basis) - cell_velocities(adata, enforce=True, vkey=vkey, ekey=ekey, basis=basis) return adata -# support using just spliced/unspliced/new/total 4 layers, as well as uu, ul, su, sl layers -def recipe_one_shot_data( +def recipe_mix_kin_deg_data( adata: AnnData, tkey: Optional[str] = None, reset_X: bool = True, @@ -516,18 +519,17 @@ def recipe_one_shot_data( keep_filtered_cells: Optional[bool] = None, keep_filtered_genes: Optional[bool] = None, keep_raw_layers: Optional[bool] = None, - one_shot_method: str = "sci-fate", del_2nd_moments: Optional[bool] = None, ekey: str = "M_t", vkey: str = "velocity_T", basis: str = "umap", rm_kwargs: Dict[str, Any] = {}, -) -> AnnData: - """An analysis recipe that properly pre-processes different layers for a one-shot experiment with both labeling and - splicing data. +): + """An analysis recipe that properly pre-processes different layers for a mixture kinetics and degradation + experiment with both labeling and splicing or only labeling data. Args: - adata: AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four + adata: An AnnData object that stores data for the kinetics experiment, must include `uu, ul, su, sl` four different layers. tkey: The column key for the labeling time of cells in .obs. Used for labeling based scRNA-seq data (will also support for conventional scRNA-seq data). Note that `tkey` will be saved to `adata.uns['pp']['tkey']` and @@ -551,8 +553,6 @@ def recipe_one_shot_data( `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. keep_raw_layers: Whether to keep layers with raw measurements in the returned adata object. Used in `recipe_monocle`. If None, would be set according to `DynamoAdataConfig`. Defaults to None. - one_shot_method: The method to use for calculate the absolute labeling and splicing velocity for the one-shot - data of use. Defaults to "sci-fate". del_2nd_moments: Whether to remove second moments or covariances. Argument used for `dynamics` function. If None, would be set according to `DynamoAdataConfig`. Defaults to None. ekey: The dictionary key that corresponds to the gene expression in the layer attribute. ekey and vkey will be @@ -620,8 +620,8 @@ def recipe_one_shot_data( preprocessor.select_genes_kwargs["keep_filtered"] = keep_filtered_genes if reset_X: - reset_adata_X(adata, experiment_type="one-shot", has_labeling=has_labeling, has_splicing=has_splicing) - preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="one-shot") + reset_adata_X(adata, experiment_type="mix_pulse_chase", has_labeling=has_labeling, has_splicing=has_splicing) + preprocessor.preprocess_adata_monocle(adata=adata, tkey=tkey, experiment_type="mix_pulse_chase") if not keep_raw_layers: del_raw_layers(adata) @@ -652,7 +652,7 @@ def recipe_one_shot_data( dynamics( adata, model="deterministic", - one_shot_method=one_shot_method, + est_method="twostep", del_2nd_moments=del_2nd_moments, ) # then perform dimension reduction @@ -663,7 +663,7 @@ def recipe_one_shot_data( dynamics( adata, model="deterministic", - one_shot_method=one_shot_method, + est_method="twostep", del_2nd_moments=del_2nd_moments, ) reduceDimension(adata, reduction_method=basis) diff --git a/dynamo/tools/sampling.py b/dynamo/tools/sampling.py index 3a7ced727..cb75161c9 100644 --- a/dynamo/tools/sampling.py +++ b/dynamo/tools/sampling.py @@ -14,6 +14,51 @@ from .utils import nearest_neighbors, timeit +def sample( + arr: Union[list, np.ndarray], + n: int, + method: Literal["random", "velocity", "trn", "kmeans"] = "random", + X: Optional[np.ndarray] = None, + V: Optional[np.ndarray] = None, + seed: int = 19491001, + **kwargs, +) -> np.ndarray: + """A collection of various sampling methods. + + Args: + arr: The array to be sub-sampled. + n: The number of samples. + method: The method to be used. + "random": randomly choosing `n` elements from `arr`; + "velocity": Higher the velocity, higher the chance to be sampled; + "trn": Topology Representing Network based sampling; + "kmeans": `n` points that are closest to the kmeans centroids on `X` are chosen. + Defaults to "random". + X: Coordinates associated to each element in `arr`. Defaults to None. + V: Velocity associated to each element in `arr`. Defaults to None. + seed: The randomization seed. Defaults to 19491001. + + Raises: + NotImplementedError: `method` is invalid. + + Returns: + The sampled data array. + """ + + if method == "random": + np.random.seed(seed) + sub_arr = arr[np.random.choice(arr.shape[0], size=n, replace=False)] + elif method == "velocity" and V is not None: + sub_arr = arr[sample_by_velocity(V=V, n=n, seed=seed, **kwargs)] + elif method == "trn" and X is not None: + sub_arr = arr[trn(X=X, n=n, return_index=True, seed=seed, **kwargs)] + elif method == "kmeans": + sub_arr = arr[sample_by_kmeans(X, n, return_index=True)] + else: + raise NotImplementedError(f"The sampling method {method} is not implemented or relevant data are not provided.") + return sub_arr + + class TRNET: """Class for topology representing network sampling. @@ -255,48 +300,3 @@ def lhsclassic( H[:, i] = H[:, i] * (bounds[i][1] - bounds[i][0]) + bounds[i][0] return H - - -def sample( - arr: Union[list, np.ndarray], - n: int, - method: Literal["random", "velocity", "trn", "kmeans"] = "random", - X: Optional[np.ndarray] = None, - V: Optional[np.ndarray] = None, - seed: int = 19491001, - **kwargs, -) -> np.ndarray: - """A collection of various sampling methods. - - Args: - arr: The array to be sub-sampled. - n: The number of samples. - method: The method to be used. - "random": randomly choosing `n` elements from `arr`; - "velocity": Higher the velocity, higher the chance to be sampled; - "trn": Topology Representing Network based sampling; - "kmeans": `n` points that are closest to the kmeans centroids on `X` are chosen. - Defaults to "random". - X: Coordinates associated to each element in `arr`. Defaults to None. - V: Velocity associated to each element in `arr`. Defaults to None. - seed: The randomization seed. Defaults to 19491001. - - Raises: - NotImplementedError: `method` is invalid. - - Returns: - The sampled data array. - """ - - if method == "random": - np.random.seed(seed) - sub_arr = arr[np.random.choice(arr.shape[0], size=n, replace=False)] - elif method == "velocity" and V is not None: - sub_arr = arr[sample_by_velocity(V=V, n=n, seed=seed, **kwargs)] - elif method == "trn" and X is not None: - sub_arr = arr[trn(X=X, n=n, return_index=True, seed=seed, **kwargs)] - elif method == "kmeans": - sub_arr = arr[sample_by_kmeans(X, n, return_index=True)] - else: - raise NotImplementedError(f"The sampling method {method} is not implemented or relevant data are not provided.") - return sub_arr From 6f38e7a313d1978809b5fe05de43050e5491e69d Mon Sep 17 00:00:00 2001 From: sichao Date: Tue, 21 Nov 2023 16:01:25 -0500 Subject: [PATCH 13/14] change the location of func gradient --- dynamo/tools/graph_calculus.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dynamo/tools/graph_calculus.py b/dynamo/tools/graph_calculus.py index 075e2c077..b916248b0 100644 --- a/dynamo/tools/graph_calculus.py +++ b/dynamo/tools/graph_calculus.py @@ -606,6 +606,19 @@ def divop(W: Union[sp.csr_matrix, np.ndarray]) -> np.ndarray: return -0.5 * gradop(W).T +def gradient(E: Union[sp.csr_matrix, np.ndarray], p: np.ndarray) -> np.ndarray: + """Calculate gradient of a weighted graph. + + Args: + E: The length of the edges of the graph. + p: The potential of the graph. + + Returns: + The gradient of the weighted graph. + """ + return gradop(E).dot(p) + + def gradop(adj: Union[sp.csr_matrix, np.ndarray]) -> sp.csr_matrix: """Return the gradient operator of a weighted graph in matrix form. @@ -624,19 +637,6 @@ def gradop(adj: Union[sp.csr_matrix, np.ndarray]) -> sp.csr_matrix: return sp.csr_matrix((x, (i, j)), shape=(ne, nv)) -def gradient(E: Union[sp.csr_matrix, np.ndarray], p: np.ndarray) -> np.ndarray: - """Calculate gradient of a weighted graph. - - Args: - E: The length of the edges of the graph. - p: The potential of the graph. - - Returns: - The gradient of the weighted graph. - """ - return gradop(E).dot(p) - - class GraphVectorField: """An object representing a graph vector field, storing its edges, edge lengths, and edge weights. From 960dff78c183bdbb2e8e0c661ca8898463cae04c Mon Sep 17 00:00:00 2001 From: sichao Date: Thu, 30 Nov 2023 15:31:57 -0500 Subject: [PATCH 14/14] update the import in tests --- tests/test_neighbors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_neighbors.py b/tests/test_neighbors.py index f4c7a426e..7ed2fa097 100644 --- a/tests/test_neighbors.py +++ b/tests/test_neighbors.py @@ -6,7 +6,7 @@ import dynamo as dyn from dynamo.tools.connectivity import ( - _gen_neighbor_keys, + generate_neighbor_keys, check_and_recompute_neighbors, check_neighbors_completeness, ) @@ -23,7 +23,7 @@ def test_neighbors_subset(): # check obsp keys subsetting by AnnData Obj neighbor_result_prefix = "" - conn_key, dist_key, neighbor_key = _gen_neighbor_keys(neighbor_result_prefix) + conn_key, dist_key, neighbor_key = generate_neighbor_keys(neighbor_result_prefix) check_and_recompute_neighbors(adata, result_prefix=neighbor_result_prefix) expected_conn_mat = adata.obsp[conn_key][indices][:, indices] expected_dist_mat = adata.obsp[dist_key][indices][:, indices]