Integration dev

.github/workflows/tests.yml

@@ -31,25 +31,83 @@ jobs:
       SNAKEMAKE_CONDA_FRONTEND: mamba
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Create test environment
-        uses: mamba-org/setup-micromamba@v2
+        uses: mamba-org/setup-micromamba@v3
         with:
           environment-file: environment.yml
           cache-environment: true
           cache-downloads: true
 
       - name: Cache Snakemake rule environments
-        uses: actions/cache@v4
+        uses: actions/cache@v6
         with:
           path: .snakemake/conda
           key: snakemake-conda-${{ runner.os }}-${{ hashFiles('workflow/envs/*.yml') }}
           restore-keys: |
             snakemake-conda-${{ runner.os }}-
 
       - name: Run tests
-        run: pytest tests -q
+        run: pytest tests -q --run-downsample-scaling
+
+  downsample-rule:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    env:
+      SNAKEMAKE_CONDA_FRONTEND: mamba
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Create test environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-file: environment.yml
+          cache-environment: true
+          cache-downloads: true
+
+      - name: Cache Snakemake rule environments
+        uses: actions/cache@v6
+        with:
+          path: .snakemake/conda
+          key: snakemake-conda-${{ runner.os }}-${{ hashFiles('workflow/envs/*.yml') }}
+          restore-keys: |
+            snakemake-conda-${{ runner.os }}-
+
+      - name: Run downsampling rule smoke test
+        run: pytest tests/downsampling --run-downsample-rule -q
+
+  downsample-workflow:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    env:
+      SNAKEMAKE_CONDA_FRONTEND: mamba
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Create test environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-file: environment.yml
+          cache-environment: true
+          cache-downloads: true
+
+      - name: Cache Snakemake rule environments
+        uses: actions/cache@v6
+        with:
+          path: .snakemake/conda
+          key: snakemake-conda-${{ runner.os }}-${{ hashFiles('workflow/envs/*.yml') }}
+          restore-keys: |
+            snakemake-conda-${{ runner.os }}-
+
+      - name: Run full downsampling workflow test
+        run: pytest tests/downsampling --run-downsample-workflow -q
 
   conda-validation:
     if: github.event_name == 'workflow_dispatch' && inputs.run_conda_validation
@@ -60,6 +118,7 @@ jobs:
         env_name:
           - cellbender.yml
           - doubletfinder.yml
+          - downsample_clusters.yml
           - emptydrops.yml
           - posthocfilter.yml
           - scdblfinder.yml
@@ -72,10 +131,10 @@ jobs:
       SNAKEMAKE_CONDA_FRONTEND: mamba
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Create test environment
-        uses: mamba-org/setup-micromamba@v2
+        uses: mamba-org/setup-micromamba@v3
         with:
           environment-file: environment.yml
           cache-environment: true
@@ -92,10 +151,10 @@ jobs:
         shell: bash -el {0}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Create test environment
-        uses: mamba-org/setup-micromamba@v2
+        uses: mamba-org/setup-micromamba@v3
         with:
           environment-file: environment.yml
           cache-environment: true
@@ -117,10 +176,10 @@ jobs:
       GITHUB_PAT: ${{ github.token }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Create test environment
-        uses: mamba-org/setup-micromamba@v2
+        uses: mamba-org/setup-micromamba@v3
         with:
           environment-file: environment.yml
           cache-environment: true

.gitignore

@@ -13,9 +13,16 @@ scratch/
 *out
 results/
 testdata/results/
+testdata/downsampling/results/
 # Keep golden test outputs trackable even though results/ is ignored.
 !tests/reference_outputs/
 !tests/reference_outputs/**
 !tests/reference_outputs/testdata/
 !tests/reference_outputs/testdata/results/
+testdata/downsampling/results/
 !tests/reference_outputs/testdata/results/**
+!tests/downsampling/reference_outputs/
+!tests/downsampling/reference_outputs/**
+!tests/downsampling/reference_outputs/testdata/
+!tests/downsampling/reference_outputs/testdata/results/
+!tests/downsampling/reference_outputs/testdata/results/**

README.md

@@ -20,6 +20,17 @@ In order to assess different tools and options for pre-processing scRNA-seq data
 | Mouse | C57BL/6 | L8TX_190509_01_E09 | striatum, striatal amygdala | cells | Chromium 10x 3' Gene Expression v3 | NovaSeq6000 | 13,475 | 82,801 | 3,400 | [nemo](https://assets.nemoarchive.org/dat-qg7n1b0) | [run1](https://data.nemoarchive.org/biccn/grant/u19_zeng/zeng/transcriptome/scell/10x_v2/mouse/raw/STR/NW_TX0007-8_S01_L003.fastq.tar);[run2](https://data.nemoarchive.org/biccn/grant/u19_zeng/zeng/transcriptome/scell/10x_v2/mouse/raw/STR/NW_TX0010-7_S01_L003.fastq.tar) | 2 runs on same library |
 | Mouse | C57BL/6 | L8TX_210204_01_H05 | olfactory region: main and accessory olfactory bulbs | cells | Chromium 10x 3' Gene Expression v3 | NovaSeq6000 | 10,895 | 136,593 | 3,971 | [nemo](https://assets.nemoarchive.org/dat-qg7n1b0) | [fastq]( https://data.nemoarchive.org/biccn/grant/u19_zeng/zeng/transcriptome/scell/10x_v3/mouse/raw/NW_TX0109-4_S01_L003-001.fastq.tar) | |
 
+
+## Workflow modes
+
+The main Snakemake entrypoint supports three `workflow_mode` values in `config/config.yaml` or via `--config`:
+
+- `preprocess`: run the preprocessing workflow only. This is the default and preserves the original behavior.
+- `preprocess_and_downsample`: run preprocessing and then downsample the generated Seurat `.rds` outputs with `workflow/rules/downsample_clusters.smk`.
+- `downsample_only`: skip preprocessing and run downsampling on existing Seurat `.rds` files from `downsampleSeuratObjectDir`.
+
+Downsampling outputs are written to `downsampleResultsDir`, defaulting to `results/downsampling`. Use `downsampleTargets` to restrict downsampling to selected Seurat object basenames, or leave it as `all` to use every available input for the selected mode.
+
 ## Tests
 For information on how to run the test suite, or run the workflow in test mode, see tests/README.md.
 

config/config.yaml

@@ -1,6 +1,7 @@
 conda-channel-priority: "strict"
 
 sampleTable: "samplesheet.tsv"
+workflow_mode: "preprocess_and_downsample" # {"preprocess", "preprocess_and_downsample", "downsample_only"}
 workflow_seed: 12345
 
 emptydrop_removal_methods: ["tenx","emptydrops"]
@@ -10,3 +11,9 @@ posthoc_methods: ["threshold", "mad"]
 min_nfeature: 200
 min_ncount: 500
 max_mtdna: 5
+
+downsampleSeuratObjectDir: "seurat_objects"
+downsampleResultsDir: "results/downsampling"
+downsampleTargets: "all"
+nDownsampleReplicates: 100
+downsampleRate: 0.8

profiles/slurm/config.yaml

@@ -7,6 +7,8 @@ set-threads:
     soupx: 1
     scdblfinder: 1
     emptydrops: 1
+    downsample_cluster_replicate: 1
+    downsample_clusters: 1
 
 set-resources:
     tenx2seuratrds:
@@ -43,3 +45,13 @@ set-resources:
         nodes: 1
         tasks: 1
         cpus_per_task: 1
+
+    downsample_cluster_replicate:
+        nodes: 1
+        tasks: 1
+        cpus_per_task: 1
+
+    downsample_clusters:
+        nodes: 1
+        tasks: 1
+        cpus_per_task: 1

scrnaseq_preprocess_slurmrunner.sh

@@ -35,4 +35,4 @@ PATH_TO_MY_CONDA_ENVS=$1
 
 snakemake --unlock --snakefile workflow/Snakefile --configfile config/config.yaml --use-conda --workflow-profile profiles/slurm --profile cannon
 
-snakemake --conda-prefix $PATH_TO_MY_CONDA_ENVS --snakefile workflow/Snakefile --rerun-incomplete --retries 2 --jobs 200 --latency-wait 120 --configfile config/config.yaml --use-conda --workflow-profile profiles/slurm --profile cannon
+snakemake --conda-prefix $PATH_TO_MY_CONDA_ENVS --snakefile workflow/Snakefile --rerun-incomplete --retries 2 --keep-going --jobs 1500 --max-jobs-per-timespan "10/1s" --max-status-checks-per-second 5 --latency-wait 120 --configfile config/config.yaml --use-conda --workflow-profile profiles/slurm --profile cannon

tests/README.md

@@ -22,6 +22,8 @@ The dry-run DAG test uses the small CellRanger-style input data in `testdata/` a
 
 GitHub Actions runs the default test suite via `.github/workflows/tests.yml`, using the same top-level `environment.yml` test runner environment and caching Snakemake-created rule environments under `.snakemake/conda`.
 
+The default suite also includes the downsampling fixture/reference checks and a dry-run of `workflow_mode=downsample_only`. Optional downsampling execution tests can be run with `pytest tests/downsampling --run-downsample-rule -q` or `pytest tests/downsampling --run-downsample-workflow -q`, and both are available as manual GitHub Actions dispatch jobs.
+
 The default test suite also includes a focused local rule-execution smoke test. It runs the real `tenx2seuratrds`, `find_markers`, and `combine_markers` rule chain against `testdata/`, using Snakemake's `--use-conda` support and writing outputs under pytest's temporary directory. This catches broken R package imports, script argument drift, invalid Seurat object creation, and marker CSV schema changes without submitting to SLURM.
 
 The R output validator also checks that the Seurat object has at least 100 features and 100 cells; metadata rows match the cell count; barcode row names are present, unique, and nonempty; `orig.ident`, `nCount_RNA`, `nFeature_RNA`, `percent.mt`, and `seurat_clusters` metadata columns exist; RNA count and feature-count metadata values are finite and positive; mitochondrial percentages are finite and within `[0, 100]`; at least two clusters are present; PCA and UMAP reductions exist; the marker table is nonempty and has the expected columns; marker gene symbols are present and nonempty; marker numeric columns are finite; marker p-value and percent columns are within `[0, 1]`; marker clusters are present in the Seurat metadata; markers are reported for at least two clusters; and the marker `workflow` column matches the expected test workflow label. The test runner environment is defined in the repository-level `environment.yml`; the rule-specific R environment is still created by Snakemake from `workflow/envs/tenx2seuratrds.yml`. A separate lightweight checkpoint-expansion test uses a fake `Rscript` to materialize the `marker_manifest` checkpoint, verify that dynamic `find_markers` jobs are generated for each cluster id, and confirm that `combine_markers` receives the expected marker chunks.
@@ -48,7 +50,7 @@ The default tests build and inspect the DAG and run a focused local R-rule smoke
 pytest tests --run-workflow
 ```
 
-The full-run test calls `tests/run_test_workflow.sh`, which uses `testdata/samplesheet_test.tsv` and overrides the workflow output directory with `resultsDir=testdata/results`. The manifest in `tests/test_sample_rule_output_files.txt` is therefore written with paths under `testdata/results/`.
+The full-run test calls `tests/run_test_workflow.sh`, which uses `testdata/samplesheet_test.tsv` and overrides the workflow output directories with `resultsDir=testdata/results` and `downsampleResultsDir=testdata/results/downsampling`. The manifest in `tests/test_sample_rule_output_files.txt` is therefore written with paths under `testdata/results/`.
 
 For testing, omit `--snakemake-conda-prefix` so Snakemake uses its default `.snakemake/conda` location under the repository root. The runner assumes that the current environment already provides `snakemake` on `PATH`.
 

tests/conftest.py

@@ -10,6 +10,24 @@ def pytest_addoption(parser):
         default=None,
         help="optional value to pass to Snakemake --conda-prefix for --run-workflow",
     )
+    parser.addoption(
+        "--run-downsample-rule",
+        action="store_true",
+        default=False,
+        help="execute the downsample_clusters Snakemake rule on one downsampling test fixture",
+    )
+    parser.addoption(
+        "--run-downsample-workflow",
+        action="store_true",
+        default=False,
+        help="execute the full downsampling Snakemake workflow on downsampling testdata",
+    )
+    parser.addoption(
+        "--run-downsample-scaling",
+        action="store_true",
+        default=False,
+        help="run downsample_clusters on small and medium fixtures and check runtime scales reasonably",
+    )
     parser.addoption(
         "--run-conda-validation",
         action="store_true",

tests/downsampling/README.md

@@ -0,0 +1,53 @@
+# Downsampling Tests
+
+The default pytest suite uses the Seurat RDS fixtures in `testdata/downsampling/seurat_objects/`, validates the saved reference TSVs, and runs a Snakemake dry run in `workflow_mode=downsample_only` against `testdata/downsampling/results/`:
+
+```bash
+pytest tests/downsampling -q
+```
+
+The dry-run test checks that the DAG includes the `downsample_clusters` rule (which loops over all replicates for a target in a single job) and expected testdata outputs without executing R code.
+
+To execute the single workflow rule on one test fixture, run the optional rule test:
+
+```bash
+pytest tests/downsampling --run-downsample-rule -q
+```
+
+To execute the full downsampling test workflow and write outputs under `testdata/downsampling/results/`, run:
+
+```bash
+pytest tests/downsampling --run-downsample-workflow -q
+```
+
+The full-workflow test removes `testdata/downsampling/results/` before running so that both expected test fixture outputs are regenerated from scratch. It validates each output table and compares it to the reference snapshot under `tests/downsampling/reference_outputs/testdata/results/`.
+
+All three of `--run-downsample-rule`, `--run-downsample-workflow`, and `--run-downsample-scaling` run automatically in CI on every push/PR (as separate jobs in `.github/workflows/tests.yml`) rather than being opt-in-only — these flags mainly exist so they can also be run manually/locally without the rest of the suite.
+
+```bash
+pytest tests/downsampling --run-downsample-scaling -q
+```
+
+It runs `downsample_clusters` once on the small fixture (`filtered_seurat_tenx_test`, 300 cells) and once on a synthetic mid-scale fixture (`filtered_seurat_tenx_test_medium`, 2500 cells, generated by stratified-resampling the small fixture via `tests/downsampling/make_medium_fixture.R`), then asserts the wall-clock ratio between the two doesn't wildly exceed the ~8.3x cell-count ratio. This is the kind of check that would have caught the `do.call(SCTransform, ...)` performance regression (see `workflow/scripts/downsample_clusters.R`) — that bug produced no incorrect output and no error, only a data-size-dependent slowdown invisible to the tiny, single-size fixtures the other tests use.
+
+By default Snakemake creates rule conda environments under `.snakemake/conda` in the repository root. To use an explicit Snakemake conda prefix:
+
+```bash
+pytest tests/downsampling --run-downsample-workflow --snakemake-conda-prefix /path/to/snakemake-conda-envs -q
+```
+
+Snakemake uses the explicit Linux lock file `workflow/envs/downsample_clusters.linux-64.pin.txt` when creating the rule environment on Linux. The workflow test uses two downsampling replicates, a 50% downsampling rate, and `workflowSeed=12345`.
+
+Reference outputs are compared by `(bootstrap_number, clusterid)`. The `max_jaccard` column is numeric-compared with `DOWNSAMPLE_REFERENCE_REL_TOLERANCE` defaulting to `1e-6` and `DOWNSAMPLE_REFERENCE_ABS_TOLERANCE` defaulting to `1e-8`.
+
+To refresh the reference outputs after an intentional workflow change, first run:
+
+```bash
+pytest tests/downsampling --run-downsample-workflow -q
+```
+
+Then copy the regenerated outputs into `tests/downsampling/reference_outputs/`:
+
+```bash
+python tests/downsampling/update_reference_outputs.py
+```

tests/downsampling/conftest.py

@@ -0,0 +1 @@
+# Downsampling-specific options are registered in tests/conftest.py.

tests/downsampling/make_medium_fixture.R

@@ -0,0 +1,58 @@
+# Generates a mid-scale downsampling test fixture by stratified resampling
+# (with light Poisson jitter) of the existing tiny fixture, preserving
+# per-cluster proportions. This exists purely to give the scaling test two
+# data points of meaningfully different size, so quadratic-or-worse
+# performance regressions (e.g. the do.call()/SCTransform hang) are
+# detectable in CI without needing production-scale data.
+#
+# Usage: Rscript make_medium_fixture.R <input_rds> <output_rds> <target_ncells> [seed]
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 3) {
+  stop("Usage: make_medium_fixture.R <input_rds> <output_rds> <target_ncells> [seed]", call. = FALSE)
+}
+
+input_rds <- args[1]
+output_rds <- args[2]
+target_ncells <- as.integer(args[3])
+seed <- if (length(args) >= 4) as.integer(args[4]) else 12345L
+
+suppressPackageStartupMessages(library(Seurat))
+set.seed(seed)
+
+obj <- readRDS(input_rds)
+counts <- obj[["RNA"]]$counts
+clusters <- obj$seurat_clusters
+
+cluster_props <- table(clusters) / length(clusters)
+per_cluster_target <- round(cluster_props * target_ncells)
+
+sampled_source_idx <- unlist(lapply(names(per_cluster_target), function(cl) {
+  pool <- which(clusters == cl)
+  sample(pool, size = per_cluster_target[[cl]], replace = TRUE)
+}))
+
+sampled_counts <- counts[, sampled_source_idx, drop = FALSE]
+sampled_clusters <- unname(clusters[sampled_source_idx])
+
+jitter <- matrix(
+  rpois(length(sampled_counts), lambda = 0.02),
+  nrow = nrow(sampled_counts),
+  ncol = ncol(sampled_counts)
+)
+sampled_counts <- sampled_counts + jitter
+colnames(sampled_counts) <- paste0("cell_", seq_len(ncol(sampled_counts)))
+
+new_obj <- CreateSeuratObject(counts = sampled_counts)
+pct_mt <- PercentageFeatureSet(new_obj, pattern = "^MT-")
+new_obj$percent.mt <- if (is.data.frame(pct_mt)) pct_mt[[1]] else unname(pct_mt)
+new_obj$seurat_clusters <- factor(sampled_clusters, levels = levels(clusters))
+Idents(new_obj) <- new_obj$seurat_clusters
+
+dir.create(dirname(output_rds), showWarnings = FALSE, recursive = TRUE)
+saveRDS(new_obj, output_rds)
+
+message(sprintf(
+  "Wrote %s: %d cells x %d genes (%d clusters)",
+  output_rds, ncol(new_obj), nrow(new_obj), length(levels(new_obj$seurat_clusters))
+))

tests/downsampling/reference_outputs/testdata/results/filtered_seurat_emptydrops_test_clusterdownsampling.tsv

@@ -0,0 +1,9 @@
+clusterid	max_jaccard	bootstrap_number
+3	0.8421052631578947	1
+2	0.29896907216494845	1
+0	0.3431372549019608	1
+1	0.32989690721649484	1
+3	0.9393939393939394	2
+2	0.64	2
+1	0.5263157894736842	2
+0	0.3384615384615385	2

tests/downsampling/reference_outputs/testdata/results/filtered_seurat_tenx_test_clusterdownsampling.tsv

@@ -0,0 +1,7 @@
+clusterid	max_jaccard	bootstrap_number
+0	0.5966386554621849	1
+1	0.4492753623188406	1
+2	0.0847457627118644	1
+0	0.5957446808510638	2
+1	0.6666666666666666	2
+2	0.13157894736842105	2