Snakemake7

.github/workflows/ci.yaml

@@ -1,17 +1,22 @@
 name: CI
 on:
   pull_request:
+    paths-ignore:
+        - "docs/**"
+        - "**.md"
     branches:
     - main
+
 
 jobs:
   Testing:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Test whole workflow
-      uses: snakemake/snakemake-github-action@v1.24.0
+      uses: snakemake/snakemake-github-action@v1.25.1
       with:
         directory: .test/ci/
         snakefile: workflow/Snakefile
         args: "--use-conda --show-failed-logs -j 1 --conda-cleanup-pkgs cache --conda-frontend mamba"
+        stagein: "conda config --set channel_priority strict"

.github/workflows/main.yaml

@@ -9,22 +9,32 @@ on:
 
 jobs:
 
-  Testing:
+  Testing_local-fastq:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Test workflow (Local Fastq > VCF)
-      uses: snakemake/snakemake-github-action@v1.24.0
+      uses: cademirch/snakemake-github-action@v1
       with:
         directory: .test/ecoli/
         snakefile: workflow/Snakefile
         args: "--use-conda --show-failed-logs -j 10 --conda-cleanup-pkgs cache --conda-frontend mamba"
+  Testing_local-fastq_and_sra:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Test workflow (Local Fastq > VCF)
+      uses: cademirch/snakemake-github-action@v1
+      with:
+        directory: .test/ecoli/
+        snakefile: workflow/Snakefile
+        args: "--config samples='config/local_and_sra.csv' --use-conda --show-failed-logs -j 10 --conda-cleanup-pkgs cache --conda-frontend mamba"
   Testing_QC:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Test workflow (QC rules)
-      uses: snakemake/snakemake-github-action@v1.24.0
+      uses: cademirch/snakemake-github-action@v1
       with:
         snakefile: workflow/modules/qc/Snakefile
         directory: .test/qc/
@@ -34,7 +44,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Test workflow (Postprocess)
-      uses: snakemake/snakemake-github-action@v1.24.0
+      uses: cademirch/snakemake-github-action@v1
       with:
         snakefile: workflow/modules/postprocess/Snakefile
         directory: .test/postprocess/
@@ -44,8 +54,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Test workflow (Trackhubs)
-      uses: snakemake/snakemake-github-action@v1.24.0
+      uses: cademirch/snakemake-github-action@v1
       with:
         snakefile: workflow/modules/trackhub/Snakefile
         directory: .test/trackhub/
-        args: "--use-conda --show-failed-logs -j 10 --conda-cleanup-pkgs cache --conda-frontend mamba"
+        args: "--use-conda --show-failed-logs -j 10 --conda-cleanup-pkgs cache --conda-frontend mamba"

.gitignore

@@ -42,3 +42,6 @@ tmp/
 .test/ci/results
 .test/ci/benchmarks
 .test/ci/logs
+.vscode
+.test/trackhub/*.sizes
+.test/trackhub/out.log

.readthedocs.yaml

@@ -0,0 +1,15 @@
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+
+# Build from the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Explicitly set the version of Python and its requirements
+python:
+  install:
+    - requirements: docs/requirements.txt

.test/ci/config/config.yaml

@@ -22,7 +22,7 @@ trackhub_email: "hi@website.com"
 minNmer: 500 # the minimum Nmer used to split up the genome; e.g. a value of 200 means only Nmers 200 or greater are used to define the boundaries of intervals. The minimum is 50.
 num_gvcf_intervals: 50 # The maximum number of intervals to create for GVCF generation. Note: the actual number of intervals may be less than the specified value if the reference genome has very few gaps.
 db_scatter_factor: 0.15 # Scatter factor for calculating number of intervals to create for genomics db generation. (scatter_factor * num_samples * num_gvcf_intervals) gives us number of db intervals to create. Reccomend <1
-
+ploidy: 2 # Ploidy for HaplotypeCaller and Sentieon Haplotyper
 ## Coverage options ##
 ## default pipeline is optimized for low coverage data - if using high coverage data (> 10x), uncomment high coverage options and comment out low coverage options
 

.test/ci/config/resources.yaml

@@ -73,7 +73,10 @@ sentieon_combine_gvcf:
 process_ref:
   mem: 15000
 # custom python algo to create intervals
-create_intervals:
+create_db_intervals:
+  mem: 5000
+
+create_gvcf_intervals:
   mem: 5000
 
 ## Callable sites workflow

.test/ecoli/config/config.yaml

@@ -9,7 +9,7 @@ intervals: True #Set to True if you want to perform variant calling using interv
 sentieon: False #set to True if you want to use sentieon, False if you want GATK
 sentieon_lic: ".lic" #set to path of sentieon license
 remote_reads: False # set if you want reads to be on google cloud storage remote
-remote_reads_prefix: "" # set to google bucket name where reads live
+remote_reads_prefix: ""
 bigtmp: "" #Set to a path with lots of free space to use for commands that require large amounts of temp space; defaults to system tmpdir if empty
 cov_filter: True #set to True if you want to include coverage thresholds in the callable sites bed file (default uses mappability only)
 generate_trackhub: True #Set to true if you want to generate a Genome Browser Trackhub. Dependent on postprocessing module.
@@ -19,11 +19,14 @@ trackhub_email: "hi@website.com"
 # Variables you *might* need to change
 ##############################
 
+refGenome: 
+refPath:
+
 # Interval approach options, only applicable if intervals is True
 minNmer: 500 # the minimum Nmer used to split up the genome; e.g. a value of 200 means only Nmers 200 or greater are used to define the boundaries of intervals. The minimum is 50.
 num_gvcf_intervals: 3 # The maximum number of intervals to create for GVCF generation. Note: the actual number of intervals may be less than the specified value if the reference genome has very few gaps.
 db_scatter_factor: 0.15 # Scatter factor for calculating number of intervals to create for genomics db generation. (scatter_factor * num_samples * num_gvcf_intervals) gives us number of db intervals to create. Reccomend <1
-
+ploidy: 1 # Ploidy for HaplotypeCaller and Sentieon Haplotyper
 ## Coverage options ##
 ## default pipeline is optimized for low coverage data - if using high coverage data (> 10x), uncomment high coverage options and comment out low coverage options
 

.test/ecoli/config/ecoli_config_genome.csv

@@ -0,0 +1,3 @@
+BioSample,LibraryName,Run,Organism,BioProject,fq1,fq2
+SAMN12676327,EK7.12,SRR10058855,Escherichia coli,PRJNA563564,data/local_fastq/my_sample1_1.fastq.gz,data/local_fastq/my_sample1_2.fastq.gz
+SAMN12676342,EK7.30,SRR10058838,Escherichia coli,PRJNA563564,data/local_fastq/my_sample2_1.fastq.gz,data/local_fastq/my_sample2_2.fastq.gz

.test/ecoli/config/ecoli_samples.csv

@@ -1,3 +1,3 @@
 BioSample,LibraryName,refGenome,Run,Organism,BioProject,fq1,fq2,refPath
-SAMN12676327,EK7.12,GCA_000008865.2,SRR10058855,Escherichia coli,PRJNA563564,data/local_fastq/my_sample1_1.fastq.gz,data/local_fastq/my_sample1_2.fastq.gz,data/local_genome/local_genome.fna
+SAMN12676327,EK7.12,GCA_000008865.2,SRR10058855,Escherichia coli,PRJNA563564,data/local_fastq/my_sample1_1.fastq.gz,data/local_fastq/my_sample1_2.fastq.gz,data/local_genome/local_genome.fna.gz
 SAMN12676342,EK7.30,GCA_003018455.1,SRR10058838,Escherichia coli,PRJNA563564,data/local_fastq/my_sample2_1.fastq.gz,data/local_fastq/my_sample2_2.fastq.gz

.test/ecoli/config/local_and_sra.csv

@@ -0,0 +1,3 @@
+BioSample,LibraryName,refGenome,Run,Organism,BioProject,fq1,fq2,refPath
+SAMN12676327,EK7.12,GCA_000008865.2,SRR10058855,Escherichia coli,PRJNA563564,data/local_fastq/my_sample1_1.fastq.gz,data/local_fastq/my_sample1_2.fastq.gz,data/local_genome/local_genome.fna.gz
+SAMN12676342,EK7.30,GCA_003018455.1,SRR10058838,Escherichia coli,PRJNA563564

.test/ecoli/config/resources.yaml

@@ -50,8 +50,11 @@ bedtools:
 process_ref:
     mem: 4000
 # custom python algo to create intervals
-create_intervals:
-    mem: 4000
+create_db_intervals:
+  mem: 5000
+
+create_gvcf_intervals:
+  mem: 5000
 
 ## Callable sites workflow