diff --git a/src/infercnvpy/tl/_infercnv.py b/src/infercnvpy/tl/_infercnv.py
index deb0b9d..b4762b3 100644
--- a/src/infercnvpy/tl/_infercnv.py
+++ b/src/infercnvpy/tl/_infercnv.py
@@ -97,15 +97,15 @@ def infercnv(
         logging.warning(f"Skipped {np.sum(var_mask)} genes because they don't have a genomic position annotated. ")  # type: ignore
     if exclude_chromosomes is not None:
         var_mask = var_mask | adata.var["chromosome"].isin(exclude_chromosomes)
+
     tmp_adata = adata[:, ~var_mask]
+    reference = _get_reference(adata, reference_key, reference_cat, reference)[:, ~var_mask]
 
     expr = tmp_adata.X if layer is None else tmp_adata.layers[layer]
 
     if scipy.sparse.issparse(expr):
         expr = expr.tocsr()
 
-    reference = _get_reference(tmp_adata, reference_key, reference_cat, reference)
-
     var = tmp_adata.var.loc[:, ["chromosome", "start", "end"]]  # type: ignore
 
     chr_pos, chunks = zip(
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 2513958..9f1decf 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -49,6 +49,10 @@ def test_infercnv(adata_oligodendroma, reference_key, reference_cat):
     cnv.tl.infercnv(adata_oligodendroma, reference_key=reference_key, reference_cat=reference_cat)
 
 
+def test_infercnv_manual_reference(adata_oligodendroma):
+    cnv.tl.infercnv(adata_oligodendroma, reference=np.ones(adata_oligodendroma.shape[1]))
+
+
 @pytest.mark.skip(
     reason="rpy2 segfaults on the CI. I don't know why and don't have the time for a painful debugging session."
 )