From 06d85baf0a1f0e90977c29bd22da6c477e997958 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 31 Jan 2026 20:28:48 +0100 Subject: [PATCH 001/175] Create run_cutadapt.jl. --- .gitignore | 2 + cutadapt_primer_trimming_stats.txt | 129 ++++++++++++++++++++++ pipelinesteps.txt | 20 ++-- requirements.txt | 1 + src/main.jl | 14 +++ src/run_cutadapt.jl | 168 +++++++++++++++++++++++++++++ 6 files changed, 324 insertions(+), 10 deletions(-) create mode 100644 cutadapt_primer_trimming_stats.txt create mode 100644 requirements.txt create mode 100644 src/main.jl create mode 100644 src/run_cutadapt.jl diff --git a/.gitignore b/.gitignore index 97e509a..e5d0a29 100644 --- a/.gitignore +++ b/.gitignore @@ -254,3 +254,5 @@ rsconnect/ ### Other ### # Things with ambiguous intellectual property internal +inputs/fastq +cutadapt/output_* \ No newline at end of file diff --git a/cutadapt_primer_trimming_stats.txt b/cutadapt_primer_trimming_stats.txt new file mode 100644 index 0000000..1593477 --- /dev/null +++ b/cutadapt_primer_trimming_stats.txt @@ -0,0 +1,129 @@ +This is cutadapt 4.2 with Python 3.11.2 +Command line parameters: -g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G CYCCTACYYTMGYYCTKGA -m 200 --discard-untrimmed -o ./outputs/fastq/JIN-Nu-mul_R1_trimmed.fastq.gz -p ./outputs/fastq/JIN-Nu-mul_R2_trimmed.fastq.gz ./outputs/fastq/JIN-Nu-mul_*_L001_R1_001.fastq.gz ./outputs/fastq/JIN-Nu-mul_*_L001_R2_001.fastq.gz +Processing paired-end reads on 1 core ... +ERROR: [Errno 2] No such file or directory: './outputs/fastq/JIN-Nu-mul_*_L001_R1_001.fastq.gz' +This is cutadapt 4.2 with Python 3.11.2 +Command line parameters: -g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G CYCCTACYYTMGYYCTKGA -m 200 --discard-untrimmed -o ./outputs/fastq/JIN-Nu-mul_R1_trimmed.fastq.gz -p ./outputs/fastq/JIN-Nu-mul_R2_trimmed.fastq.gz ./inputs/fastq/JIN-Nu-mul_S54_L001_R1_001.fastq.gz ./inputs/fastq/JIN-Nu-mul_S54_L001_R2_001.fastq.gz +Processing paired-end reads on 1 core ... +Finished in 7.733 s (32.926 µs/read; 1.82 M reads/minute). + +=== Summary === + +Total read pairs processed: 234,851 + Read 1 with adapter: 229,487 (97.7%) + Read 2 with adapter: 177,592 (75.6%) + +== Read fate breakdown == +Pairs that were too short: 404 (0.2%) +Pairs discarded as untrimmed: 60,591 (25.8%) +Pairs written (passing filters): 173,856 (74.0%) + +Total basepairs processed: 117,895,202 bp + Read 1: 58,947,601 bp + Read 2: 58,947,601 bp +Total written (filtered): 80,551,787 bp (68.3%) + Read 1: 39,926,366 bp + Read 2: 40,625,421 bp + +=== First read: Adapter 1 === + +Sequence: CCAGCASCYGCGGTAATTCC; Type: regular 5'; Length: 20; Trimmed: 229487 times + +Minimum overlap: 3 +No. of allowed errors: +1-9 bp: 0; 10-19 bp: 1; 20 bp: 2 + +Overview of removed sequences +length count expect max.err error counts +3 604 3669.5 0 604 +4 100 917.4 0 100 +5 10 229.3 0 10 +9 1 0.9 0 1 +10 8 0.2 1 3 5 +11 4 0.1 1 3 1 +12 16 0.0 1 8 8 +13 10 0.0 1 2 8 +14 16 0.0 1 7 9 +15 10 0.0 1 4 6 +16 16 0.0 1 2 14 +17 28 0.0 1 13 15 +18 120 0.0 1 15 44 61 +19 1565 0.0 1 373 697 495 +20 51044 0.0 2 42814 4174 4056 +21 77925 0.0 2 66902 5338 5685 +22 49668 0.0 2 42264 3789 3615 +23 48268 0.0 2 42172 2777 3319 +24 71 0.0 2 14 50 7 +29 1 0.0 2 1 +68 1 0.0 2 1 +78 1 0.0 2 0 0 1 + + +=== Second read: Adapter 2 === + +Sequence: ACTTTCGTTCTTGATYRA; Type: regular 5'; Length: 18; Trimmed: 120098 times + +Minimum overlap: 3 +No. of allowed errors: +1-9 bp: 0; 10-18 bp: 1 + +Overview of removed sequences +length count expect max.err error counts +3 281 3669.5 0 281 +4 254 917.4 0 254 +5 2 229.3 0 2 +6 1 57.3 0 1 +7 3 14.3 0 3 +10 2 0.2 1 0 2 +11 4 0.1 1 3 1 +12 2 0.0 1 1 1 +13 6 0.0 1 1 5 +14 8 0.0 1 4 4 +15 19 0.0 1 10 9 +16 11 0.0 1 3 8 +17 443 0.0 1 50 393 +18 63922 0.0 1 32105 31817 +19 19015 0.0 1 13804 5211 +20 13766 0.0 1 10057 3709 +21 21942 0.0 1 15923 6019 +22 33 0.0 1 9 24 +60 2 0.0 1 0 2 +61 2 0.0 1 0 2 +184 2 0.0 1 0 2 +185 80 0.0 1 1 79 +186 197 0.0 1 169 28 +187 42 0.0 1 32 10 +188 37 0.0 1 22 15 +189 19 0.0 1 17 2 +190 2 0.0 1 0 2 +191 1 0.0 1 1 + + +=== Second read: Adapter 3 === + +Sequence: CYCCTACYYTMGYYCTKGA; Type: regular 5'; Length: 19; Trimmed: 57494 times + +Minimum overlap: 3 +No. of allowed errors: +1-9 bp: 0; 10-19 bp: 1 + +Overview of removed sequences +length count expect max.err error counts +3 351 3669.5 0 351 +4 103 917.4 0 103 +9 1 0.9 0 1 +10 5 0.2 1 2 3 +11 13 0.1 1 2 11 +12 19 0.0 1 4 15 +13 487 0.0 1 21 466 +14 51676 0.0 1 7573 44103 +15 2734 0.0 1 387 2347 +16 1438 0.0 1 352 1086 +17 647 0.0 1 129 518 +19 1 0.0 1 0 1 +47 1 0.0 1 0 1 +165 3 0.0 1 3 +182 7 0.0 1 0 7 +183 1 0.0 1 0 1 +184 5 0.0 1 2 3 +185 2 0.0 1 0 2 diff --git a/pipelinesteps.txt b/pipelinesteps.txt index cae66fd..c067831 100644 --- a/pipelinesteps.txt +++ b/pipelinesteps.txt @@ -1,13 +1,13 @@ sftp raw .fastq.gz files to local -bash fastqc on server `fastqc -t 20 *` -bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` -bash cutadapt to trim .fastq.gz files `python3 cutadapt_test.py` -R filter and taxonomic assignment -output .fasta and +S bash fastqc on server `fastqc -t 20 *` +S bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` +L bash cutadapt to trim .fastq.gz files `python3 cutadapt_test.sh` +L R filter and taxonomic assignment +L output .fasta and taxonomy.csv sftp .fasta to server -bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i input.fasta -o output.fasta` -bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global input.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` -output taxonomy.tsv +S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i input.fasta -o output.fasta` +S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global input.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` +S output taxonomy.tsv sftp taxonomy.tsv to local -python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` -python merge tables `python3 merge_tables.py` +L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` +L python merge tables `python3 merge_tables.py` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8ae48c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +cutadapt \ No newline at end of file diff --git a/src/main.jl b/src/main.jl new file mode 100644 index 0000000..f22ddb9 --- /dev/null +++ b/src/main.jl @@ -0,0 +1,14 @@ +include("run_cutadapt.jl") + +using .Cutadapt + +## Instantiate +primers_path = "./inputs/primers.yml" +primer_pairs = ["TarEuk", "Meta2"] +fastq_input_dir = "./inputs/fastq/" +cutadapt_dir = "./cutadapt/" +optional_args = "-m 200 --discard-untrimmed" + +## Main + +cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) \ No newline at end of file diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl new file mode 100644 index 0000000..217b99c --- /dev/null +++ b/src/run_cutadapt.jl @@ -0,0 +1,168 @@ +module Cutadapt + +export cutadapt + + using YAML + using TimeZones + using Logging + + # Prevent duplication of primers. Must be instantiated outside get_primers() loop. Global scope may be an issue. + used_forward = String[] + used_reverse = String[] + + # Get primers based on input. Will not run without used_forward and used_reverse arrays being defined first. + function get_primers(input, primers_path) + data = YAML.load_file(primers_path) + + for pair in data["Pairs"] + if haskey(pair, input) + pair_names = pair[input] + + forward = data["Forward"][pair_names[1]] + reverse = data["Reverse"][pair_names[2]] + + output_forward = in(forward, used_forward) ? "x" : forward + output_reverse = in(reverse, used_reverse) ? "x" : reverse + + if output_forward != "x" + push!(used_forward, output_forward) + end + if output_reverse != "x" + push!(used_reverse, output_reverse) + end + + return (output_forward, output_reverse) + end + end + + path = pwd() * "/" * primers_path + return "Invalid input. Please specify a valid primer pair in '$path'." + end + + # Formats cutadapt's arguments for the primer sets requested + function get_primer_args(primer_pairs, primers_path) + args = "" + + for pair in primer_pairs + pair_tuple = get_primers(pair, primers_path) + forward_primer = pair_tuple[1] + reverse_primer = pair_tuple[2] + + args *= join([flag * primer * " " for (flag, primer) in (("-g ", forward_primer), ("-G ", reverse_primer)) if primer != "x"], "") + end + + # Reset global used primers. + used_forward = String[] + used_reverse = String[] + + return chop(args) + end + + function run_cutadapt(primer_args, optional_args, fastq_in_dir, cutadapt_dir) + samples = [] + + # Messy filesystem stuff + time = chop("$(now(localzone()))", tail = 13) + fastq_out_dir = cutadapt_dir * "output_$time/" + log_dir = fastq_out_dir * "/logs/" + stats_path = joinpath(log_dir, "cutadapt_primer_trimming_stats.txt") + summary_path = joinpath(log_dir, "cutadapt_trimmed_percentage.txt") + stats_basename = basename(stats_path) + summary_basename = basename(summary_path) + + isdir(log_dir) || mkpath(log_dir) + + # Filter .fastq.gz files + for f in filter(x->occursin(r"_R1.*fastq\.gz$", x), readdir(fastq_in_dir)) + push!(samples, split(f, '_')[1]) + end + + @info("cutadapt running at $time with arguments: $primer_args $optional_args.") + + nsamples = length(samples) + for (i, sample) in enumerate(samples) + inputR1 = fastq_in_dir * sample * "_*_L001_R1_001.fastq.gz" + inputR2 = fastq_in_dir * sample * "_*_L001_R2_001.fastq.gz" + + outputR1 = fastq_out_dir * sample * "_R1_trimmed.fastq.gz" + outputR2 = fastq_out_dir * sample * "_R2_trimmed.fastq.gz" + + @info("On sample $i/$nsamples ($sample).") + cutadapt_cmd = "cutadapt $primer_args $optional_args -o $outputR1 -p $outputR2 $inputR1 $inputR2" + + open(stats_path, "a") do io + run(pipeline(`bash -lc $cutadapt_cmd`; stdout=io, stderr=io)) + end + + end + + samples_str = join(samples, " ") + cmd = "paste <(printf \"%s\\n\" $samples_str) " * + "<(grep \"passing\" $stats_basename | cut -f3 -d \"(\" | tr -d \")\") " * + "<(grep \"filtered\" $stats_basename | cut -f3 -d \"(\" | tr -d \")\") " * + "> $summary_basename" + + cd(log_dir) do + run(pipeline(`bash -lc $cmd`)) + end + + @info("cutadapt complete. Output available in $fastq_out_dir.") + end + + """ + function cutadapt(primer_pairs, primers_path, fastq_in_dir, fastq_out_dir, optional_args) + + Requires `cutadapt` installed and in PATH. Runs `cutadapt` command with as many primer pairs as necessary (useful for multiplex) and any optional parameters. Primer aguments are determined from YAML file in format: + ``` YAML + Forward: + PrimerF: "CCAGCASCYGCGGTAATTCC" + + Reverse: + Primer1R: "ACTTTCGTTCTTGATYRA" + Primer2R: "DCTKTCGTYCTTGATYRA" + + Pairs: + - PrimerPair1: + - PrimerF + - Primer1R + - PrimerPair2: + - PrimerF + - Primer2R + ``` + Where a primer pair contains the same forward or reverse as another specified pair, these are deduplicated by name rather than sequence. For example: + ```julia + primer_pairs = ["PrimerPair1", "PrimerPair2"] + ``` + This would execute `cutadapt` with arguments `-g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G DCTKTCGTYCTTGATYRA`. + + ## Arguments + - `primer_pairs` (default: [""]): Specify primers in string array format. + - `primers_path` (default: "./inputs/primers.yml"): Specify path of YAML file for primers. + - `fastq_in_dir` (default: "./inputs/fastq/"): Specify path of *_*_L001_R1_001.fastq.gz and *_*_L001_R2_001.fastq.gz files for cutadapt. + - `cutadapt_dir` (default: "./cutadapt/"): Specify an output directory for trimmed .fastq.gz and logs. + + ## Keyword Arguments + - `optional_args` (optional, default: "-m 200 --discard-untrimmed"): Specify additional arguments passed to `cutadapt` command. + + """ + function cutadapt( + primer_pairs = [""], + primers_path = "./inputs/primers.yml", + fastq_in_dir = "./inputs/fastq/", + cutadapt_dir = "./cutadapt/"; + optional_args = "-m 200 --discard-untrimmed" + ) + run_cutadapt( + get_primer_args(primer_pairs, primers_path), + optional_args, + fastq_in_dir, + cutadapt_dir + ) + end +end + +primers_path = "./inputs/primers.yml" +primer_pairs = ["TarEuk", "Meta2"] +fastq_input_dir = "./inputs/fastq/" +fastq_output_dir = "./cutadapt/fastq/" +optional_args = "-m 200 --discard-untrimmed" \ No newline at end of file From 4691a26302cd3b7ed60a543d084c69a2476d0d49 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 31 Jan 2026 20:33:29 +0100 Subject: [PATCH 002/175] Edit .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e5d0a29..5b4522f 100644 --- a/.gitignore +++ b/.gitignore @@ -254,5 +254,5 @@ rsconnect/ ### Other ### # Things with ambiguous intellectual property internal -inputs/fastq +inputs/fastq/ cutadapt/output_* \ No newline at end of file From 5ed72860d633ab763195316d57ad31a8ada972ca Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 31 Jan 2026 20:36:58 +0100 Subject: [PATCH 003/175] Remove ignored files. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5b4522f..e5d0a29 100644 --- a/.gitignore +++ b/.gitignore @@ -254,5 +254,5 @@ rsconnect/ ### Other ### # Things with ambiguous intellectual property internal -inputs/fastq/ +inputs/fastq cutadapt/output_* \ No newline at end of file From c61e7a1cc46af55af12658cea3ef4b8110c83212 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sun, 1 Feb 2026 23:36:16 +0100 Subject: [PATCH 004/175] Create conversion and merge functions for vsearch and dada2 outputs. --- .gitignore | 2 +- DADA2/.~lock.tax_counts_fasta.csv# | 1 + DADA2/tax_counts_fasta.csv | 199 ++++++++++++++++++++++++ protist_filtered.csv | 59 +++++++ src/dada2.r | 0 src/main.jl | 8 +- src/merge_and_filter_taxa.jl | 237 +++++++++++++++++++++++++++++ vsearch/taxonomy.tsv | 159 +++++++++++++++++++ 8 files changed, 662 insertions(+), 3 deletions(-) create mode 100644 DADA2/.~lock.tax_counts_fasta.csv# create mode 100644 DADA2/tax_counts_fasta.csv create mode 100644 protist_filtered.csv create mode 100644 src/dada2.r create mode 100644 src/merge_and_filter_taxa.jl create mode 100644 vsearch/taxonomy.tsv diff --git a/.gitignore b/.gitignore index e5d0a29..5b4522f 100644 --- a/.gitignore +++ b/.gitignore @@ -254,5 +254,5 @@ rsconnect/ ### Other ### # Things with ambiguous intellectual property internal -inputs/fastq +inputs/fastq/ cutadapt/output_* \ No newline at end of file diff --git a/DADA2/.~lock.tax_counts_fasta.csv# b/DADA2/.~lock.tax_counts_fasta.csv# new file mode 100644 index 0000000..51e5ef1 --- /dev/null +++ b/DADA2/.~lock.tax_counts_fasta.csv# @@ -0,0 +1 @@ +,joshua,parrot,01.02.2026 22:46,file:///home/joshua/.config/libreoffice/4; \ No newline at end of file diff --git a/DADA2/tax_counts_fasta.csv b/DADA2/tax_counts_fasta.csv new file mode 100644 index 0000000..0a151d2 --- /dev/null +++ b/DADA2/tax_counts_fasta.csv @@ -0,0 +1,199 @@ +sequence,tax.Domain,tax.Supergroup,tax.Division,tax.Subdivision,tax.Class,tax.Order,tax.Family,tax.Genus,tax.Species,boot.Domain,boot.Supergroup,boot.Division,boot.Subdivision,boot.Class,boot.Order,boot.Family,boot.Genus,boot.Species,SeqName,SeqName.y,JIN-Nu-mul,JIN-Nu-mul2 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCCGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,100,100,100,seq1,seq1,45046,45046 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,76,seq2,seq2,11240,11240 +AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGAACGTGCACGGACGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,77,seq3,seq3,10288,10288 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,99,99,seq4,seq4,8606,8606 +TAGGTAGCGAGCGTTATCCGGATTTACTGGGTGTAAAGGGCGCGTAGGCGGGCTGACAAGTCAGGAGTGAAAACTATGGGCTTAACCCATAGCCTGCTTTTGAAACTGTGAGTCTTGAGTATCGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAAGAACACCAGTGGCGAAGGCGGATTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGGAGTTAACACAATAAGTATTCCACCTGGGGAGTACGGCCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_NK4A214_group,Ruminococcaceae_NK4A214_group_sp.,100,100,100,100,100,100,100,44,44,seq5,seq5,8346,8346 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,62,62,seq6,seq6,6751,6751 +TAGGTTGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGGCGGAGAAGCAAGTTGGGAGTGAAATCCATGGGCTCAACCCATGAACTGCTCTCAAAACTGTTTCCCTTGAGTATCGGAGAGGCAAGCGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGTTAACACAATAAGTATTCCACCTGGGGAGTACGACCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_UCG005,Ruminococcaceae_UCG005_sp.,100,100,100,100,100,100,100,95,95,seq7,seq7,1400,1400 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTTGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGACATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGTGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,65,65,seq8,seq8,1073,1073 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCTGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,37,37,37,seq9,seq9,1000,1000 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTTTGGTGTGCACTGGCATGGGCTCGCCTCGCTGGCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGCCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTACGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,100,100,100,100,67,seq10,seq10,920,920 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCCGGTTGGCCGGTCCGATTTTTTCGTGTACTGGATTTCCAACGGGGCCTTTCCTTCTGGCTAACCTTGAGTCCTTGTGGCTCTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Candida,Candida_wounanorum,100,100,100,100,100,100,100,100,100,seq11,seq11,909,909 +AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGATCGACGACGGTCGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,75,seq12,seq12,614,614 +AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGGTCGACGACGGTCGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,70,seq13,seq13,499,499 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGTTGGGCCGATCGGTCCGCCTTCTGGTGTGTACCGGTCGTCTCGTCCCTTCTGCTGGCGATGCGCTCCTGGCCTTAACTGGCCGGGTCGTTCCTCCAGCACTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Plantago,Plantago_lanceolata,100,100,100,100,100,100,100,99,99,seq14,seq14,421,421 +AGCTTTCCAAGTGCATAAAATGATTGTTGTGGTTAAAAAGCTCGTAGTTGGATTATAAAGATTGTATAATGAGCATCTTGGATGTTTTTCATTATCATCTTACTTTTTTATTATATTAGTAATAATATAATAACTGTTACTTTGAATAAATCAGAGGGTTTAAACCAGGCATTATATGCTTGTATGGTCTAGCATGGAATAACACTATAGGAAAAGTTAGTGTGGTTTCACTAATCTTTTTCATGATTAATAGGAACAAACGGGGGCATTCGTATCGCTACGTTAGAGGTGAAATTCTTGGACCGTAGCGAGACGTCCTACTGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Chromadorea,Chromadorea_X,Strongyloides,Strongyloides_myopotami,100,100,100,100,100,100,100,100,73,seq15,seq15,369,369 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCCGGTTGGCCGGTCCGATTTTTTCGTGTACTGGATTTCCAACGGGGCCTTTCCTTCTGGCTAACCTTGAGTCCTTGTGGCTCTTGGCGAACCGGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Saccharomyces,Saccharomyces_cerevisiae,100,100,100,100,100,100,100,63,63,seq16,seq16,361,361 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCATTGTGCTTATGTGTCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,86,86,83,seq17,seq17,339,339 +GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGGCGTCTAAGTCAGCGGTGAAAGGTTCCGGCTCAACCGGGACAGTGCCGATGATACTGGCTGCCTTGAATGCGGTCAAGGCCGGCGGAATGTGGCGTGTAGCGGTGAAATGCATAGATATGCCACAGAACACCGATAGCGAAGGCAGCTGGCCGGGCCTGCATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGGACACTCGTCGTCGGCGACAGACAGCCGGCGGCCAAGCGAAAGTGATAAGTGTCCCACCTGGGGAGTACGGTCGCAAGGCTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Sphingobacteriales,Sphingobacteriaceae,Parapedobacter,Parapedobacter_sp.,99,99,99,99,99,46,31,25,25,seq18,seq18,336,336 +GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCGGGCCGTCAAGTCAGCGGTAAAATTGCGGGGCTCAACCCCGTCGAGCCGTTGAAACTGGCAGCCTTGAGTGGGCGAGAAGTATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACTCCGATTGCGAAGGCAGCATGCCGGCGCCCAACTGACGCTGAAGCACGAAAGCGTGGGTATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGAGCGCTAATTGTTTGCGGAGAATGATCCGTGAGTGATACAGCGAAAGCGTTAAGCGCTCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,71,71,71,seq19,seq19,316,316 +GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGAGAGTCAAGCCGGCGGTCAAATTGCGGGGCCCAACCCCGTACCGCCGTCGGAACTGGCTCCCTTGAGTGGACGAGAAGTAAGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCTTACCGGTGTCCAACTGACGCTCAGGCACGAAAGCGTGGGGATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGGATACTAGCTGTCCGGGTCGAGTGAGACCTGGGGGGCACAGCGAAAGCGTTAAGTATCCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,97,43,43,43,seq20,seq20,314,314 +TAGGTAGCGAGCGTTATCCGGATTTACTGGGTGTAAAGGGCGCGTAGGCGGGCTGACAAGTCAGGAGTGAAAACTATGGGCTTAACCCATAGCCTGCTTTTGAAACTGTGAGTCTTGAGTATCGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAAGAACACCAGTGGCGAAGGCGGATTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAACAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGGAGTTAACACAATAAGTATTCCACCTGGGGAGTACGGCCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_NK4A214_group,Ruminococcaceae_NK4A214_group_sp.,100,100,100,100,100,100,100,46,46,seq21,seq21,287,287 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTCGCAAGAGGCGAGAGTGCCATTAGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTTTACCTACAGGTAAGATCAATGAGAGCCACCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,98,98,98,98,90,seq22,seq22,275,275 +AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGAACGTGCACGGACGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTATTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,67,seq23,seq23,273,273 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCTTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGAGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCATTGCTAGAGGTGAAATTCTTAGATTTATGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,99,99,99,67,67,seq24,seq24,252,252 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,94,seq25,seq25,238,238 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGATGGGACGATCGGTCCGCCTTTAGGTGAGCACCGGTCGTCCGGTCTCTTACGCCGGCGATGCGCTCCTAGCCTTAGTTGGCCGGGTCGTGCCTCCGGCACAGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATTATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Spinacia,Spinacia_oleracea,99,99,99,99,99,99,99,7,7,seq26,seq26,236,236 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCGGCCGGCATCGCGCTCCTAGCCTTAATTGGCCGGGTCGTGTTTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Aegilops,Aegilops_tauschii,100,100,100,100,100,100,100,82,82,seq27,seq27,220,220 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGACCGGTCCGCCTATGGTGTGCACCGGTCGGCTCGTCCCTTCTACCGGCGATACGCTCCTGGTCTTAATTGGCCGGGTCGTGCCTCCGGTGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTCTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Prunus,Prunus_persica,100,100,100,100,100,100,100,73,73,seq28,seq28,216,216 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTATTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,100,100,seq29,seq29,209,209 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTATTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,68,68,seq30,seq30,188,188 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,61,seq31,seq31,188,188 +GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGAGAGTCAAGCCGGCGGTCAAATCGCGGGGCCCAACCCCGTGCCGCCGTCGGAACTGGCTCCCTTGAGTGGGCGAGAAGTATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCTTACCGGCGCCCAACTGACGCTCAGGCACGAAAGCGTGGGGATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGGATGCTAGCTGTCCGGGGGGAATGGCCCCTGGGCGGCACAGCGAAAGCGTTAAGCATCCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,83,83,83,seq32,seq32,154,154 +TAGGTGGCAAGCGTTATCCGGATTTATTGGGTGTAAAGGGCGTGTAGGCGGGACTGCAAGTCAGATGTGAAAACTATGGGCTCAACCCATAGCCTGCATTTGAAACTGTAGTTCTTGAGTGCTGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAGGAACACCAGTGGCGAAGGCGGATTGCTGGACAGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGGATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGCAAACGCAATAAGTATCCCACCTGGGGAGTACGATCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Flavonifractor,Flavonifractor_sp.,100,100,100,100,100,100,100,98,98,seq33,seq33,124,124 +TATGGAGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGCATGGCAAGTCTGATGTGAAAGGCCCGGGCCCAACCCGGGAACTGCATTGGAAACTGTCAGGCTGGAGTGCAGGAGAGGTAAGTGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTACTGGACTGTAACTGACGTTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTCGGGGGTTTAAGGACCTCCGGTGCCGCAGCAAACGCAATAAGTATTCCACCTGGGGAGTACGTTC,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Lachnospiraceae,Lachnospiraceae_X,Lachnospiraceae_X_sp.,100,100,100,100,100,100,100,96,96,seq34,seq34,121,121 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTGGCGTATGTGCCTTAGGGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,80,80,80,80,79,seq35,seq35,120,120 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAACTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,94,94,seq36,seq36,117,117 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCATTGCTAGAGGTGAAATTCTTAGATTTATGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,92,91,seq37,seq37,117,117 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTCGGCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTCTACTTCCAAGTAAAATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAAGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,81,81,78,78,75,seq38,seq38,107,107 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTTGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCAACGGGGACTTACCTCCTGGTGAACTGCAATGTCCTTTACTGGGTGTTGTAGCGAACTAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Leucosporidium,Leucosporidium_fragarium,100,100,100,100,100,100,100,98,49,seq39,seq39,103,103 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGCCGGCCGGTCCGCCTTTCGGTGTGCACCGGTCGTCTCGTCCCTTCTGCCGGCGATGCGCTCCTGGTCTTAACTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Pogostemon,Pogostemon_cablin,100,100,100,100,100,100,100,82,82,seq40,seq40,91,91 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTACGGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq41,seq41,91,91 +GAGGATGCAAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCTGTTCAGCAAGTCAGAGGTGAAATACTTGAGCTTAACTCGGGAACTGCCTTTGATACTGTTGAGCTGGAATACGGATGCCGTGGGAGGAATGAGTAGTGTAGCGGTGAAATGCATAGATATTACTCAGAACACCGATTGCGAAGGCATCTCACGAATCCGTCATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATAACTAACCGTCGGCGATAGACAGTCGGTGGCCAAGCGAAAGCGATAAGTTATCCACCTGGGGAGTACGTTC,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Rikenellaceae,Rikenellaceae_RC9_gut_group,Rikenellaceae_RC9_gut_group_sp.,100,100,100,100,100,100,100,99,99,seq42,seq42,91,91 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCGGAATGACTCAGCGCAGTATGATATCTTTACCTCGAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,63,63,62,62,61,seq43,seq43,78,78 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTTCAGGTGTGCACCGGTTTACTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Cirsium,Cirsium_pendulum,100,100,100,100,100,100,100,92,92,seq44,seq44,64,64 +GAACGGATTAGGCTATGAATTTTCCATGCGTGAAGTCGCAGACCGCATGGTCGGTTTCTGGGCGGACGCGCTTGAAGAGGAAGGCATCATCGAAACGCCTGAACAGAAGCAGATTTTCTACGATGAAGTCGTCTATGCGCTGCTGGCCCAGATGTGGGCGCCGAACTCGCCGCAGTGGTTCAACACCGGACTGAAGCGCAGCTACGGGATCGCCGGAGACAAGGACGATCTGTATTACTATGATGAAAAGACGGGGGAAGTCGTCGAATCGGAAGACCGCTACACCCGCACTCAGGCTTCCGCCTGCTTTATTCTC,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,88,24,12,12,12,12,12,12,12,seq45,seq45,62,62 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,66,66,seq46,seq46,58,58 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCCGCCGGTCCGCCTCTGGTGTGCACTGGCGTGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,94,65,seq47,seq47,58,58 +AACTCCAAGAGTGTCTATGGTGGATGCTGCAGTTAAAGGGTCCGTAGTCGTGAATGCAATTAAATGTCGTTGTTCAATAGCGATGAGTTTGCTAATGTTTGCGGAACGGATAGGGAGTGTAGTATAGACTGGCGAAGAATGAAATCTCAAGACCCAGTTTGGACTAACGGAGGCGAAGGCGACACTCTTAGACGTATCTGAGGA,Eukaryota,Obazoa,Opisthokonta,Fungi,Opisthosporidia,Microsporida,Microsporida_X,Enterocytozoon,Enterocytozoon_bieneusi,100,100,100,100,100,100,100,100,100,seq48,seq48,53,53 +AGCTCCAATAGCGTATATTAAAGTTGTTGACGTTAAAAAGCTCGTAGTCGAACTTCGGCCTCTGGCAGTTGGTCCGCCTTTTGGTGTGTACTGATTTGTTGGAGGCTTACCTCTTGGTGAACTTCAATGCACTTTACTGGGTGTTGGAGGGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCTTATGCCTGAATACATTAGCATGGAATAATAAAATAGGACGTGTGATTCTATTTTGTTGGTTTCTAGGATTACCGTAATGATGAATAGGGTCAGTTGGGGGCATTTGTATTACATCGTCAGAGGTGAAATTCTTGGATTGATGTAAGACAAACTACTGCGAAAGCATCTGCCAAGGATGACTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Pucciniomycetes,Aecidium,Aecidium_kalanchoe,100,100,100,100,100,100,100,100,100,seq49,seq49,53,53 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTTGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCAACGGGGACTTACCTCCTGGTGAACTGCGATGTCCTTTACTGGGTGTCGTAGCGAACTAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Leucosporidium,Leucosporidium_scottii,100,100,100,100,100,100,100,100,100,seq50,seq50,53,53 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCTTGTCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,83,83,82,82,79,seq51,seq51,49,49 +GAGGATGCAAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCTGTTTATCAAGTCAGGGGTGAAATACCGGGGCTCAACTCCGGAATTGCCTCTGATACTGATAGGCTTGAATACTGTTGCCGTGGGAGGAATGAGTAGTGTAGCGGTGAAATGCATAGATATTACTCAGAACACCGATTGCGAAGGCATCTCACGAAACAGGGATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATGACTAACCGTCGGCGATATACAGTCGGTGGCCAAGCGAAAGCGATAAGTCATCCACCTGGGGAGTACGACCGCAAGGTTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Rikenellaceae,Rikenellaceae_RC9_gut_group,Rikenellaceae_RC9_gut_group_sp.,100,100,100,100,100,100,100,100,100,seq52,seq52,49,49 +AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTCGAGACGCAGCCAGGCTCAAGGGCCGATACTGCGGATTGGGACCATCCTCGAGAAGAACATATCTGTCATTGAGTTGATGGGTATGGGACTCTCGTCTTTTACTGTGAGCAAAATAGAGTGTTCAAAGCAGGCTTACGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACTTCGGTCTATTTTGTTGGTTATACTCCGAAGTAATGATTAATAGGGACAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,100,100,100,100,100,100,70,70,70,seq53,seq53,48,48 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTATGTACCTCCAAGTACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,88,88,88,seq54,seq54,47,47 +CCACATGCTCATGCGCGCCCGAAGCAAAGTTAAACAGCGAGGTCTTGCCGCAGTTCGGGTTGCCAACCAACGCCACGTTGATGGTCTTACCCTTGTTGAGCGCCATCGCGCGCCACTCCTCATCAGAGGGGAGGGTATACTCATTCGCCGTCTGTGCGCTCCCCGCCGTGTGCAACAGTTTCTCCTCCTCAAACTCAGCCGCGCTGACCACT,Eukaryota,Excavata,Discoba,Euglenozoa,Euglenida,Aphagea,Rhabdomonadales,Rhabdomonas,Rhabdomonas_intermedia,89,14,13,13,12,10,8,7,5,seq55,seq55,47,47 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,99,99,seq56,seq56,44,44 +TAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGCGAAGCAAGTCTGAAGTGAAAACCCAAGGCTCAACCATGGGAGTGCTTTGGAAACTGTATTGCTGGAGTGCAGGAGAGGTAAGTGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTACTGGACTGTAACTGACGTTGAGGCTCGAAGGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGATCACTAGGTGTCTGTGGTTTAGAACCATAGGTGCCGCAGCAAACGCAGTAAGTGATCCACCTGGGGAGTACGTTC,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Lachnospiraceae,Lachnospiraceae_X,Lachnospiraceae_X_sp.,100,100,100,100,100,100,100,58,58,seq57,seq57,42,42 +GAACGGATTAGGCTATGAATTTTCCATGCGTGAAGTCGCAGACCGCATGGTCGGTTTCTGGGCGGATGCGCTTGAAGAAGAAGGCATCATCGAAACGCCTGAACAGAAGCAGATTTTCTACGATGAAGTCGTCTATGCGCTGCTGGCCCAGATGTGGGCGCCGAACTCGCCGCAGTGGTTCAACACCGGACTGAAGCGCAGCTATGGAATCGCCGGAGACAAGGACGATCTGTATTACTATGATGAAAAGACGGGGGAAGTCGTCGAATCGGAAGACCGCTACACCCGCACTCAGGCTTCCGCCTGCTTTATTCTC,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,76,13,9,9,9,9,9,9,9,seq58,seq58,41,41 +AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCAGCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,100,100,100,100,100,100,100,58,54,seq59,seq59,40,40 +ATCATTACAAGCGTATATTAAAATTGTTGCATTTAAAAAGCTCGTAGTTGAATAATAGATTTGAAGTTAGATTGACCTAGTCAAGATATTCTTCATTTCTTTTGTTATATTTTCGGATATAACCATTTACTGTGAAAAAATTAGAGTGTTTAAAGCAAATTGTAAATTTGAATATTATTAGCATGGAATAATAATATATGATTAATATTATATATTATGGTAATATAGTATTAATAATGATTAATAGGGATAGTTGTGGGTATTCATATTTCATAGTCAGAGGTGAAATTCAAGGATTTATGAAAGATGAACGAATGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,99,99,97,97,97,97,77,77,77,seq60,seq60,39,39 +AGCTCTCCTGGTGTATGCAAACGCTGCTGCAGTTAAAGCGCTCCTAGTTGGCGGTCGGGCGCGGCCGCGGCGCCGGGACACTGCGCGCCGTCGCCGCGCGCTCGGGGTTACCATGAGAAAACCGTGACGCTCAAGGTAGGCGACTGAGCGCCCTAGCATGGGATAGCGGCGGGCTCCACGTGCGCTGCTGGTCCGTTCGGCGGAGCGAGAGGAAGAGGGGCAGTCGGGGGCCTCAGTACGATGGCGCCAGAGGTGAAATTCCGAGACCGCCGTCAGACTGCCGGCAGCGAAAGCGCTGGCCAAGGATGCTTTCG,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,75,73,73,73,71,71,71,71,seq61,seq61,38,38 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCAAGGCTTATGTGCCTCACAGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGGATTTAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,72,72,70,70,65,seq62,seq62,38,38 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCGGAAGTGCCTCCAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,86,86,85,85,82,seq63,seq63,37,37 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCCGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGGTCCAAGAATTTCACCTCTAGCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,55,34,34,seq64,seq64,33,33 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGTGTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,98,98,seq65,seq65,33,33 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTACTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq66,seq66,33,33 +CCAGAAGCTAAAGAGAAAACAGACTTGGCGTTTTCTTTAATACAACGTGTCCATGTAATGAAAACTGTGCATCATTATCCATTCCTTTAGGAAGCTAAGTCATCCCCATGGTAGAAGTGATGATTGGTGCCTTTTGCCTTTCTGACGGTAAGCACATTCTCTGGATATGAGATCCAGCTCCACTCACTCTGAACATCCAGGAAGCGGGGCCAGCTGTTCCCATGCCAGGCCCAAGCACTATCAGGTGTCCTCTGTATCCATCTTTGGTGTTCTTCCCAGCATGCACCACTCCAAGCGCCTCCCCCTTCTCAATCAGGCTCGGGTCTGAAGTCAAAGCAACACACGTCACAGCCGTCAAAAGCATTCTTTGTGGTACAGGCCACACCCAAGGAGCCACAGTATGA,Eukaryota,Excavata,Discoba,Euglenozoa,Euglenida,Euglenophyceae,Euglenaceae,Trachelomonas,Trachelomonas_grandis,78,31,31,31,30,12,12,9,8,seq67,seq67,33,33 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCAGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,65,65,seq68,seq68,32,32 +AGCTCCAATAGCGTATATTTAAGTTGTGGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,62,62,seq69,seq69,31,31 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTCGAACTTCGGGCCTGGCGGGACGGTCCGCCTTACGGTGTGTACTGTCCGGCCGGGTCTTACCTCCTGGTGAGGCCGTATGCCCTTTACTGGGTGTGCGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCATATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCTTTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Papiliotrema,Papiliotrema_fonsecae,100,100,100,100,100,100,100,95,95,seq70,seq70,31,31 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCCTATGTGCCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCATGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,73,73,67,67,64,seq71,seq71,31,31 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCCTCTGCCGCCCGGTCCGCCTATTTGGGTGTGTACTGGAGCGGTGGAGGCTTACCTCGTGGTGAACGATCATGCACTTTATTGGGTGTGGTCGGGAACCATGACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTACGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTCCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Cystobasidiomycetes_X,Cystobasidiomycetes_X_sp.,100,100,100,100,100,100,100,40,40,seq72,seq72,30,30 +AGCTCTGCCAGTGCATAGAACTATTGCTGCGGTTAAAAAGCTCGTAGTTGGATCTCTGTTCGCGGCCGGGTCGCTCCTTCGGGGGTGTACTCGCGACGTGGACATCCAGTCGATTCGTCCTCTGCTCGGGTTCGCCCTTGTGATTGGCGGTCGGCGTGTTTACCTTGAGCAAATCAGGGTGCTCAGGACAGGCATTGCGCCTGAATGTTCTTGCATGGAATAATAGAAGAGGATTTCGGTTCTGTTTTGTTGGTTTTGAAGCCGAGATAATGGTCAATAGAGACAAACGGGGGCATCGGTATTTCTGCGTGAGAGGTGAAATTCTTGGACCGCAGGAGGACCAACAACAGCGAAGGCAGTTGCCAAGAATGTCTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Chromadorea,Chromadorea_X,Miculenchus,Miculenchus_muscus,100,100,100,100,97,97,97,86,39,seq73,seq73,30,30 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCACCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCATTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTAGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,82,82,seq74,seq74,28,28 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCAATCGGTGTGCACCGGTCGTCTCGTCCCTTCTGCCGGCGATACGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Veronica,Veronica_anagallis-aquatica,100,100,100,100,100,100,100,64,64,seq75,seq75,28,28 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGATGGGACGATCGGTCCGCCTTTAGGTGAGCACCGGTCGTCCGGTCTCTTACGCCGGCGATGCGCTCCTAGCCTTAGTTGGCCGGGTCGTGCCTCCGGCACAGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAGGCCTACGCTCTGTATACATTAGCATGGGATAACATTATAGGATTCCGGTCCTATTGTGTTGGCCTTTGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Spinacia,Spinacia_oleracea,99,99,99,99,99,99,99,8,8,seq76,seq76,27,27 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCGTTTGGTGTGCACCGGTCGCCTCGTCCCTTCTGCCGGCGATGCGCTCCTGTCCTTAATTGGCCGGGTCGTGCCTCCGGCGTTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Apium,Apium_graveolens,100,100,100,100,100,100,100,72,72,seq77,seq77,27,27 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCCAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,98,98,seq78,seq78,26,26 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCCGGGCCTTACCTCTTGGTGAGACCTCATGCACTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq79,seq79,24,24 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCTGGACGGTCCGCCTTACGGTGTGCACTGTCCGGCCGGGCCTTACCTCCTGGTGAGGCCTCATGCCCTTTACTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGAGTCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTGACGGAAGACTAACAACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_dimennae,100,100,100,100,100,100,100,97,81,seq80,seq80,23,23 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGATCGGTCCGCCTCTGGTGTGCACCGGTCGGCTCGTCCCTTCTGCCGGCGATGCGCTCCTGGTCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACACCACAGGATTCTGATCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Embryophyceae_XXX,Embryophyceae_XXX_sp.,100,100,100,100,100,100,100,40,40,seq81,seq81,22,22 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGTGATGCGCTCCTAGCCTTAATTGGCCAGGTCGTGCCTCCGACATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,84,84,seq82,seq82,22,22 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTTCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,69,69,seq83,seq83,22,22 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTCGAACCTCGGGTCCGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCTTGCTGGATCTTACCTCTTGGTGAAGCCTTATGCCCTTTACTGGGTGTAGGGTCGAACCAGGAATTTTACTTTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Dioszegia,Dioszegia_crocea,100,100,100,100,100,100,100,95,60,seq84,seq84,22,22 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTTTTGGCTTATGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,85,85,73,seq85,seq85,22,22 +GCTCCTCCTCATACTGATGGCTCAGCGCCTCGTTGCGCTCGGTAAGCTCATCGTTGATCTCCTCCAGCTCCTCTTGCAGGCGCAGAAGCTCGGAGAGGTCCGCCGTCCACAGCACATGGCCGCCGGGGACGGGCATTCCCTGAAGCTGCACGCCGCCCGGAAGCATAACGGGGCCTGATTCCGTTTGGCGCATGACCGAT,Eukaryota,Obazoa,Opisthokonta,Metazoa,Mollusca,Cephalopoda,Cephalopoda_X,Spirula,Spirula_spirula,98,66,66,55,12,10,10,2,2,seq86,seq86,22,22 +TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGACGGCTTATTAAGTCTAGAATCAAAGCCCGGAGCTTAACTCCGGTTCGTTCTAGAAACTGGTAGGCTTGAGTATAGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCTATTACTGACGT,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,75,44,25,25,25,13,13,13,13,seq87,seq87,22,22 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCTGGTTGGCCAGTCTGGGTTTTTCCACGTACTGGGATGCAACCGGGCCTTTCCTTCTGGCTAACTGTGTGCTCCTTGTGGGTGCGCAGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGGATATATTAGCATGGAATAATGGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Kazachstania,Kazachstania_telluris,100,100,100,100,100,100,100,100,100,seq88,seq88,21,21 +TCCGACGACTACCACCCCGGCAAGAACATCTCCAGTCTGGTCCATACCATGGCACAGGCCTATCAGGGCACCGAGTCCATCTTGTATGCCCCCATGTTCGTGGCCAGCGATATCGTGCGCGGCGAGCTGATGAAGGAGCATATCATCCAAAACGCCATGAGCGAGATCGAGCAGGTCAATTGGATCCTCACCGGGATCGCAGACGTTTCTCAGGGGATGCCCAACTCCTGGGCTGGCTACATGACAGATGAGATACGCAACGAGCTGACAGCAAAGGGAGCCGTCGGTTATATCTGCGGGTATTTCTTCGACAGGAACGGACGGCTGCTTCAAAATCCCATCAATCGGAGCCTGATAGGCGTGTCTTTTCAGCAGA,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,87,21,21,21,15,14,14,14,14,seq89,seq89,21,21 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCTTGCGAGTCGGTCCGCCTTCTTGGTGTGTACTTACTTCGCGGGGACTTACCTCCTGGTGAACTGCAATGTCCTTTACTGGGTGTTGTAGCGAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Sphacelotheca,Sphacelotheca_koordersiana,100,100,100,100,100,100,97,60,60,seq90,seq90,20,20 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCGTTTGGTGTGCACCGGTCGCCTCGTCCCTTCTGCCGGCGATGCGCTCCTGTCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Apium,Apium_graveolens,100,100,100,100,100,100,100,84,84,seq91,seq91,20,20 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGTGCGACGACGCGGTCTGCCTCTGGTATGTACTGCGCTCGGCGCACCTTTCTGCCGGGGACGGGCTCCTGGGCTTTATTGTCTGGGACTCGGAGTCGGCGAGGTGACCTTGAGCAAACGAGAGTGTTCAAAGCAAGCCTACGCTCTGAATCATTTAGCATGGAATCACGTGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGAACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,100,100,100,100,100,100,100,100,100,seq92,seq92,20,20 +AGCTCTGCGAGTTTGCTCCCGTATTGTTGCAGTTAAAACGCCTGTAGCCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,66,seq93,seq93,20,20 +TGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGTGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,72,72,seq94,seq94,20,20 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGACCTGGCTGGGTGGTCCGCTTAACGGCGTGTACTGCCTGGCTGGGCCTTACCTCTTGGTGAGCCGGCGTGCCCTTTATTGGGGTGCGTCGGGGAACCAGGACTTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCCTATGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGAGTCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTGGTATTGAGTCGCTAGAGGTGAAATTCTTGGATTGACTCAAGACCGACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Agaricomycetes,Coprinellus,Coprinellus_congregatus,100,100,100,100,100,100,100,31,30,seq95,seq95,19,19 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,60,60,seq96,seq96,19,19 +TCCCAAGGCACTTACAACAACGACGACATCCTCGTCCTGCTCTTCCACAATACCCTTCACAGTCAGCATGCTTGCCGCCGTTCCCACCGAAGTGCCGCCAAATTTTAATACTTTCATTCGCTTATTCAGGTCATTTATGCGCAAATTTACACATATTTGGTCATATTTGTTACATTTAATTGAATTTTTTGTACTTTTGTGTTGGCAAACACATATCAAACACCAAAGAATTATGAAAAGACTACTTATTATACTCTCAGTACTTGTGACAACGCTTTCCTCAGCGACGGGACAAACCTATTACAAA,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,85,60,48,47,36,28,28,28,28,seq97,seq97,19,19 +AGCTCCAATAGCGTATATTAATGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGACTTGGCGCACCTGGCCCGCCACGGTTACGTGTGTGAGTGCCGGGATGCGCCTGTCACTTTTCTAGTAAACTATTGTGCTCTTCATTGAGTGTGATAGGTAGCTAGATAATTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCGTTTGCTATGAATACATTAGCATGGAATAATAACTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,100,100,100,100,100,100,100,93,93,seq98,seq98,18,18 +AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGGCGTTCGGGTGGTACTGGGAGGCCGGGCCCTAGGCTCTGTGCTTCCTGGAGCCACCTTCTGAGTGAGGCAACTCACTTCGGTTACCATGAGAAAAGTGTAGCGCTCAAAGCAAGCTAAGCTGAGCATTTAAGCATGGGATAACAGGCTATGACTCCATAGACGCTGTTGGTCCGTTTAGCGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCCAAGACCGCCGTCAGACTAACTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,98,98,seq99,seq99,18,18 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCTTGTGAGTCGGTCCGCCTTCTTGGTGTGTACTTACTTCACGGGGACTTACCTCTTGGTGAATTACCATGTCCTTTACTGGGTGTGGTAACGAACCAAGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_hordea,100,100,100,100,100,100,80,80,78,seq100,seq100,17,17 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCATAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCTCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,84,84,seq101,seq101,17,17 +AGCTCCAATAGCGTATATTAAAATTGTTGGCGTTAAAAAGCTCGTAGTCGAACTTCGGTGGCCGTCAGCCGGTCCGCTTTTAACGAGTGTGTACTGGATCTGATGGTTACTTTACCTCCTGGTGAACTAGCATGTCGTTTATTCGGCGTGTTAGGGAACCCGGACATTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCATGTTTGCCCGAATATCTTAGCATGGAATAATAGAATAGGACGTGCACGCCTATTGTGTTGGTCTCTAGGTGTGCCGTAATGATGAATAGGGGCGGTTGGGGGCATTTGTATTCAATTGCTAGAGGTGAAATTCTTGGATTTATTGAAGACAAACTACTGCGAAAGCATTTGCCAAGGACGCTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Agaricostilbomycetes,Bensingtonia,Bensingtonia_yuccicola,100,100,100,100,100,100,100,100,100,seq102,seq102,16,16 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTCTGCTAGCGAGAATAGGTCATCTCTTTGAGTATGTACTTGTTGTCGTTGGCATTAATCCGATTCATCTAACAGTTAAACCAAACTGTTGGATATTCGGAGCTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCCTTGAATACTCCAGCATGGAATAACAAGTAAGGACTCAAGTTCTTCTTGTTGGTTTAAGAGCCTGAGTAATGATTAAGAGGAACAGTTGGGGGCATTCGTACTTAGTAGTCAGAGGTGAAATTCTTAGATTTACTAAAGACGAACTACTGCGAAGGCATCTGCCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,75,75,seq103,seq103,16,16 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCTCACGGTGAGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCTGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,100,100,100,100,100,100,100,48,48,seq104,seq104,16,16 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGACTCGGTTGGGTGGTCTGCCTTACGGTATGTACTGCTCGACTGAGTCTTACCTCCTGGTGAGCCTGCATGTCCTTTACGGGGTGTGTAGGGGAACCAGGAATTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCATATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTTGGGGGCATTAGTATTCCGGTGCTAGAGGTGAAATTCTTAGATTGCCGGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Holtermannia,Holtermannia_corniformis,100,100,100,100,100,100,100,94,94,seq105,seq105,15,15 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCCGCCGGTCCGCCTCACGGCAAGCACCGACCAACTCGACCCTTTAGCCGGCGATGCGCTCCTAGCCTTGATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,88,88,seq106,seq106,15,15 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGTCGGGGGGAGCGGTCCGCCCCTCGTGGGTGTGCACTGGTCCACCCGACCTTTCTGCCGGGGACGCGCTCCTGGCCTTCGCTGGTCGGGACGCGGAGTCGGCGATGTTACTTTGAAAAAATTAGAGTGCTCAAAGCAAGCCTATGCTCTGAATACATTAGCATGGAATAACGTGATAGGACTCTGGTCCTGTTGTGTTGGTCTTCGGGACCGGAGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Sanionia,Sanionia_uncinata,100,100,100,100,100,100,100,100,100,seq107,seq107,15,15 +AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTTGGATCTCGGTCTGCCTCAAACGAGGTATGTACCAGGGATCTGAGACCATCCTCGAAGAAAACATGTCTGTCATTAAGTTAATGGGCATGGGATCTTCGTCATTTACTGTGAGCAAAATAGGGTGTTCAAAGCAGGCTTATGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACCTTGGTCTATTTTGTTGGTTTGTACTCCAAGGTAATGATTAATAGGGATAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,100,100,100,100,100,100,100,100,100,seq108,seq108,14,14 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTAGGCAAGGCCTGCTGGTCTGCCGACAGGCATGACTGGCTGTGCTGCGCCTCCTTTCGGGGAACGCCCCGGCTTAGCGGTCGGGGTTGGAACCGAGTTTTACTTTGAAGAAATTAGAGTGTTTAAGGCAGGCGTTTGCTTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTCTTATTTTGTTGGTTTGAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTAGTATTCAGTAGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTAGTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Rozellomycota,Rozellomycota_X,Rozellomycota_XX,Rozella,Rozella_sp.,100,79,79,69,42,42,42,28,28,seq109,seq109,14,14 +AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGATGATCGAGCATAGGTGAGGCACCGGGCTTTTTAGCTCTGAGTACCTTAGCGATGCCCTAAGGATGTTTAATCATCCCGGTTACCATGAGAAAAATGTAGCGCTCCAAGCAAGCTTTGCTGAGCATTTTAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCATAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,seq110,seq110,14,14 +GAGTGTTTTTCATTCCTCCACATCCTTTCCAAGGCGACCGTCCACCAGGAACAGGATGCCGATGATGACCACCATAAAGAGGGCCAGCACCAGCGCTGCGGAGGAGAGCTTTTGATAGTCCATGCTCTCAAAGGTATTGTTCATGAAATGCTGCAAAAGATACAGCCGGTCGAAGGGATATTTTCCCGTCAGCAGGTAGACCTCCCGAAATATCTTTAGAGAA,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,76,16,15,14,13,11,11,11,11,seq111,seq111,14,14 +TAGGTGGCAAGCGTTATCCGGATTTATTGGGTGTAAAGGGCGTGTAGGCGGGACTGCAAGTCAGATGTGAAAACTCAGGGCTCAACCCTGAGCCTGCATTTGAAACTGTAGTTCTTGAGTGCTGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAGGAACACCAGTGGCGAAGGCGGATTGCTGGACAGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGGATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGCAAACGCAATAAGTATCCCACCTGGGGAGTACGATCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Flavonifractor,Flavonifractor_sp.,100,100,100,100,100,100,100,99,99,seq112,seq112,14,14 +AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTCGGATCTCGTCCCCTTAATAAGGAAGCGGAGATTTTTTCTAATCTCTATGCGCCTTATTGTCTGGACTACGGTGACTCAAGGCAACTTGAATGTCACCCTGTTACTTTGAGCAAATTGGAGTGCTCCAACCAAGCCTAAGCTTGTACAGCTCAGCATGGAATAACGAGATAGGACTTTGATTCTTCTTGTTGGTGTCACGAATCGATAGTAATGATTGATAAGGAAATTCGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCTGCAAAGACAAACGAATGCGAAAGCATTTGCCCAGTATCTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,100,98,98,98,98,98,98,41,26,seq113,seq113,13,13 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGCCGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCGGCGGGGACTTACCTCCTGGTGAGCTGCGCTGCCCTTTACTGGGTGGCGTAGGGAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_yarrowii,100,100,100,100,100,100,71,63,63,seq114,seq114,13,13 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Stichococcus,Stichococcus_bacillaris,100,100,100,100,100,99,99,54,54,seq115,seq115,13,13 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTGCCGTCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCATATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACCGCTGCGAAAGCTTTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,100,100,100,100,100,99,99,99,99,seq116,seq116,13,13 +TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTATAGAATTAAACTTCGGGGCTTAACCCCGTCTCGTTCTATAAACTGATAGACTAGAGTGTGGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCCATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,73,47,24,24,24,14,14,14,14,seq117,seq117,13,13 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGCTCAAGAATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTTCTGTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGATTGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCTGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,98,seq118,seq118,12,12 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGACCGGTCCGCCTTTTGGTGTGCACCGGTCGGCTCGTCCCTTCTACCGGCGATACGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGTGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTCTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Prunus,Prunus_persica,100,100,100,100,100,100,100,85,85,seq119,seq119,12,12 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTCACGGTGTGCACCGGTTTACTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCTGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Artemisia,Artemisia_annua,100,100,100,100,100,100,100,91,91,seq120,seq120,12,12 +AGCTCTCCTAGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGACGATCAAGTACAAATGGATTGCCGGGCATTGAGCTCTGAGCAGTCTAGATGTACTTTACAAACCTGAAAAGGTTTCTGTTACCATGAGAAAATTGTAGCGCTCAAAGCAAGCTTAGCTGAGCATTATAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGACAGAGGTGAAATTCCGAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,seq121,seq121,12,12 +TGGATTTTGTCAACCGGCCGTCACCTCAAAACCACCTAAACAACTATATATCAATTGATTAAGAAAATATTTATGTTTTTATATCGGAAAGTAGTATTCATAAATGCAGAATGTCTGTTTATGTAAAGTTTACACAATAACTTGAAATGTTTGAAACGCTTAGGCACGAATTGCCGTGAATTCAGATTCGAATTTTATGCGATTTTTATTCCATCGGGATGCGGAAAGCGCTTTACCTTTGCACCGTCGAAAG,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,81,42,32,29,21,17,13,13,13,seq122,seq122,12,12 +AGCAGCTGCGGTGTGTACATGTACTACGACGCCGAGGGGACGGTGATATATGTCGGTAAGGCCAAAAATCTCAAGCGGCGCGTCTCGTCGTATTTCAACCGCACGCACGTCTCTACCCGTACAAATCTGCTCGTGCGCGCGATTGCCGATATGACCTACATCGTGGTCCCCACAGAGCAGGACGCGCTCAATCTTGAGAACTCCATGATTAAGGAATACCAGCCGCGCTACAACGTG,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Hexapoda,Insecta,Mengenilla,Mengenilla_chobauti,95,62,62,57,41,16,14,4,4,seq123,seq123,11,11 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGCCGGGTCGGCCGGTCCGCCTCACGGTGTGCACCGACCTACCCGACCCTTTTGTCGGCGATGCGTGCCGGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCCACGCTCTGCATACATTAGCATGGGATAACATCACAGGATTTCGGTCCTATTTTGTTGGCCTTCGGGATCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Juncus,Juncus_effusus,100,100,100,100,100,100,100,89,89,seq124,seq124,11,11 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTATCGGCGTAAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCAGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,71,71,68,68,64,seq125,seq125,11,11 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAATCTCGTAGTTGGATCTCAGGTCCAGGCTCGCGGTTCGTTTCGCGACGATACTGCCCGTCCTGACCTACCTCCCGGTTCTCCCTCGGTGCCCTTCGTTGAGTGCCCTGGGTAGCCGGAACGTTTACTTTGAAAAAATTAGAGTGCTCAAAGCAGGCAGTCTGCCTGAATAACCGCGCATGGAATAATGGAATAGGACCTCGGTTCTATTTTGTTGGTTTTCGGAACTCGAGGTAATGATTAAGAGAGACAGACGGGGGCATTCGTATTACGGTGTTAGAGGTGAAATTCTTGGATCGCCGTAAGACGAACTACTGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Annelida,Annelida_X,Annelida_XX,Achaeta,Achaeta_bifollicula,100,100,100,100,100,100,100,99,49,seq126,seq126,10,10 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAGTTGTGGCAGTAATAGTGGGTCATCTTTAACGAGCATGCACTTATTGTTATTGCCATTATTCTGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,100,99,99,99,99,98,66,66,seq127,seq127,10,10 +TCTATAAGAAGTTTTTCGCGGAAGACTTTGAGCAGATGATGAAATATGCTGACTGCTTGATTGTTCATAACGATTCCATGAAGCAGTTCTTTATCAACCGGGGCGTTGCACCAGAGAAGCTGGTTACCTTGGGGATTTTTGACTACTTGATCCCGGATGGCGAGATCAACCAGGCCAAGTTTGAACGGGCTGTTTCTGTCGCCGGTAACCTGGATGTCCGTAAGACCCAGTATTTGAACGATATCGGCAAAATCGATGCCAAGTTTAACCTTTACGGGCTGAATTTCACTTTGGACGCCTACAAGAATGTCGAATACCATGGGGCCTTTCCAGCCGATGAAATTCCCAAACAGCTGAATTCCGGCTTTGGC,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,90,23,21,19,19,17,17,17,17,seq128,seq128,10,10 +AGCTCCAAGAGCGTATCTTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGTCTGGCTGGCCGGTCCGCTTTTTTGCGAGTACTGGACCCAGCCGGGCCTTTCCTTCTGGCTAGCCTTTTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGAACGGTTGGGGACATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTACTGCGAAAGCATTTGTCAAAGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Hyphopichia,Hyphopichia_pseudoburtonii,100,100,100,100,100,100,100,95,93,seq129,seq129,9,9 +AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTTCGGCGCACTTGGCCCGTCTCGGTTTACGGGATTGTGTGCCGGTGTGCGCCATCCATCCTTCGAGAGAACACTTCTACCCTTCACTGGGTCGGGAGTGCTATCTCGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,100,100,100,100,100,100,100,100,91,seq130,seq130,9,9 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGTCGCTGGATGGCCCCCTGCCTCACGGCAGCTGGTTTGGCTGACTCCTAGCGTCCCATCCTCGGGTGGGTCCTGCTTGGCATTAGGTTGTTGGGCAGGGGAAGCCCGTCTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTAGGCCGTTGAATACATTAGCATGGAATAATGAGATAGGGCCTTGATGGATTCTTCTATTTTGTTGGTTTGCACGCCAAGGCAATGATTAACAGGGACAGTTGGGGGTATTCGTATTCAAATGTCAGAGGTGAAATTCTTGGATTTTTTGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,100,100,100,100,100,100,100,89,89,seq131,seq131,9,9 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGTCGGTCCGCCGTTTCGGTGTGCACTGGCGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACCCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Pseudostichococcus,Pseudostichococcus_monallantoides,100,100,100,100,100,100,100,99,99,seq132,seq132,9,9 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGTGCTGCCGGTCCGCCCTTTGGGTGTGCACCGGTTGCGCCCGTCCTGCTGCCGGGGACGGGTGCCTGGGCTTCACTGTCCGGGTCCTGGAGTCGGTGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCCCGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,100,100,100,100,100,100,100,100,100,seq133,seq133,9,9 +AGCTCCAATAGTGTATGTTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGTGGTCATCCGGCTCCGCCCGTATGGGTGGGCGCCTGGTTTGCCCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTCACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,53,53,seq134,seq134,9,9 +AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGTGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCACCAAGTAAGGTCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,89,89,89,89,84,seq135,seq135,9,9 +GACAGAGATTGTTTATCAGGGGCAGACGATTTCGATTGCCCGCTCCTATATTTCTATCGAAGAAGGCGCTTTCAAAGGCAACACGCAACTTGAAACGATCGTGATTCCCTCTCACGTGAATATTATCGGCAAGGAAGCGTTCGCTCAGTGCACGGGGCTTAAGACGGTTGTCATTGAAGGCTCGACGGGCTTGCTTTCCAACTACAATGGTTTGGAGA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,7,7,7,7,6,6,6,6,6,seq136,seq136,9,9 +TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGACGGTTTGCTAAGTTTAGAATCAAACCCTGGAGCTTAACTCCAGTTCGTTCTAAAAACTGGCAGACTTGAGTGTAGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCTATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,81,45,23,23,23,13,13,13,13,seq137,seq137,9,9 +AGCACCTGCGGTCTGCAGGGCACAGAGATCATCACCCAGGCCGATGGCTCCAACAACGACGCTCTCGACAAGCTCCAGGGCAAGTCGACCCGCACGGGCTACTACATGCGCAAACTCCTGCGTCAGGACGTGAGCCTCGACCCCGTATCGGCCACCGACCAGTATCACTACACCCCGCGTATCCGCTACACCGAGATCTTCCTGGCCTACGCAGAGGCAGCCAACGAGGCTTACGGCCCCATGGGTAAAGGCGGCAACAGCTACTCGGCCTACGACGTGATCAAGGCTATCCGCGAGCGCGCCGGCATCAGCGGCGACGCCTATCTGGAG,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,89,22,8,8,8,8,8,8,8,seq138,seq138,8,8 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTCTTCCTTCCTGTTTTAACCTTGCGGGGCTAAAATGAGGTTGGACTCCGTGATTGAGGTTATCTTAATTCATGGTGTTACTTTGACAAAACTATAGTGTTCCTGGCCGATCTTTGTCAGAATAATGAAGCATGGAATAACAGTTTAAAGCCCGTTTTTAGCTGTGGTCTGCTAAATGTGGAGCTATGATGAAAAGGAACTGCTGGGGGTAATCGAATTCATGGGTCAGAGGTGAAATTCTTGGATTCTGTGAAGACGAACGACTGCGAAAGCATCTATCAAATATGCTTCCA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,100,12,5,5,5,5,5,5,5,seq139,seq139,8,8 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGGCTTGGTTGATTGGTCCGCCTCACGGTGTGTACTGTTCGACCGAGCCTTACCTCTTGGTGAGCCAGCATGCCGTTTATTCGGTGTGTTGGGGAACCAGGATTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTAGTATTCAGTCGCTAGAGGTGAAATTCTTGGATTGACTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Agaricomycetes,Mrakiella,Mrakiella_aquatica,100,100,100,100,100,100,100,93,93,seq140,seq140,8,8 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGGACTTCTGTCCGGGTGCTGTTCTCCGCCGTAAGGCGTGTAGATCAATACCTCGACATCCGTTCGTTTGAGCTTCTTGCTCTTTCGAACTCTTTACTTTGAGAAAATTAGAATGTTTCAAGTAGGCTTTCGCCTGAATACTACAGCATGGAATAATAAGATAGGACTCTGGTTCCTTCTTGTTGGTTCTTAGAACTAGAGTAATGGTTAATAGGGACAGTCGGGGGCATTCGTATTCTACCGTTAGAGGTGAAATTCTTGGATCGGTTGAAGACGAACAACTGCGAAAGCATCTGCCAAGGATGTACTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Branchiopoda,Caenestheria,Caenestheria_lutraria,98,55,55,55,11,5,2,1,1,seq141,seq141,8,8 +AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCGTTTCGGTGTGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,63,63,seq142,seq142,8,8 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTTTTGGTGTGCACCGGTTTACTCGTCCCTTTTGTCGGCGATACGCTCCTGGCCTTAGTTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Taraxacum,Taraxacum_kok-saghyz,100,100,100,100,100,100,100,64,64,seq143,seq143,8,8 +AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTTGAATTTCGTTCAAATTGAATAGTGCCGAGTTTTCCTCGTGTTAATTCGAGTACTTTTTGATTAGAACTTGGGTGAGATGTACTCGTTTCGACGAGTTCGTTTCGCTCCGTTACTTTGAGCAAATTGGAGTGCTCCAACCAGGCTTAAGCTTGAACAGCTCAGCATGGAATAACAAGATAAGACTTTAGTTCTTCTTGTTGGTGACATGAACTAATAGTAATGGTTGATAAGGACATACGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACAAACAAGTGCGAAAGCATTTGCCCAGTATGTACCTGTTAA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,100,100,100,100,100,100,100,87,76,seq144,seq144,7,7 +AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTATGGCGCACTTGGCCCGCCGAGGTTTCTCGGTCGTGTGCCGGTGTGCGCCTGCCATCCTTCTGGGGAACGGCCTTGCCCTTCACTGGGTGGAGGTCGGTATCCAGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_jutlandica,100,100,100,100,100,100,100,49,21,seq145,seq145,7,7 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGACTGAAGCATTTTGCCGGCCGCGGTTTCGCGTGTTAGTGCTTGATGCTCTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGAATGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,100,seq146,seq146,7,7 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAACTTTGGTAGTAATAATGGGTTATCTCTTTGAGTATGTACCTATTGTTACTACCATTATTCCGATCTATACAAGGGGTAATTCCCTTGTGTTCTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTGATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAGCGCCTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAGCAGTCGGGGGTATTCGTATTCAGTCGTTAGAGGTGAAATTCTTAGATTGACTAAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,100,100,seq147,seq147,7,7 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTGTGGTAATAACAATGAATCATCTTTAACGAGCATGCACTTATTGTTGTTGCCATTATTCCGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,99,96,96,96,96,96,51,51,seq148,seq148,7,7 +AGGATGTCCGTCACCTCGCTGCGCTCCACCCGCAGCACCTCATAGCCGTACCGGGGGATGCTGCGCAGGATCCCGGTATGGCACAGCGCCACAAGGGCCTCCCGCACAGAGGTCTTGCTGCATCCGTACTCATCGATCAGCGCCCGCTCCGTCAGGATATCCCCGGCCTTGTACCGGGAATTGAAGATATCATTTATCACGGCGTTGTAGACCGTGGAGGACACGGAGTTCTTCATTCGGCATCGCCTCGCTTTCCATTGATCATACCGGATATATGCGGTCTTGTCAAACAACTCTTGCATATTCTTGCTTATGGTGGTATGATTACTGGTATGACCAGCAGCTTTTTACTCAGACAGCCATGCCGCCATGCGAAGAAAGGAGAGACCGACCATGTCCAAGA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,88,33,15,15,15,15,15,15,15,seq149,seq149,7,7 +CTCCGTAAATACAATAGGCGCAAAGACACAGATGGTGGTAAGAGTGGATGCCGCAATAGCTCCCGCCACCTGCTTCGCCCCCTCAATCGCGGCCTCACTCGGAGAAAGCCCCTCTTCACTTCTCATGCGGTAAATATTTTCAATTACAACGATAGAATTATCCACCAGCATACCCACGCCCAACGCCAGGCCGGACAGAGAGATGATATTCAGAGTAATCCCTGTAAAGTACATTGCCACCAGCGCCGTCAGGATACTGATAGGGATTGAACAGGCGATCACCAGTGTAGACCTGACACTGCGCAGGAATACTAAAAGAATCAGAATC,Eukaryota,TSAR,Rhizaria,Cercozoa,Sainouroidea,Sainouroidea_X,Guttulinopsidae,Rosculus,Rosculus_liberus,79,22,10,6,6,6,6,5,5,seq150,seq150,7,7 +CTGCAGAAAAAGAAAAAGGCCAGAAGCGATAGCTTTCTGACCTGGGTTTTTGGTGGTCGATGAGGGATTTGAACCCCCGACCTTGTCCTTGTAAGGGACCTGCGCTCCCGCTGCGCCAATCGACCGGATGAAGCGTGCTGCTTGCAACAGCGTTTCATTCTCCCATAATCATCGAGTTGACGCAAGGGTATATGCTGCCAAAGGTGTGAAGCG,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,91,26,16,16,7,7,7,7,7,seq151,seq151,7,7 +GAAATCTCAGTTAAGTCTAAGGTTGGCGACGAGCGCAAGTACTTAGTGCCACTCTCCAAGCAGATTCTGGTACAAGAGAACGACTACGTTCGTGCTGGCACAGCACTCTCCGACGGTGCTATCACTCCTGCCGACATCCTCAACATCATGGGTCCGACAGCTGTGCAAGAGTACATCGTGAATGAGGTGCAAGACGTGTACCGCATGCAGGGTGTGAAGATCAATGACAAGCACTTCGAGGTTATCGTACGTCAGATGATGCGCAAGGTTAACATCCTTGAGCCTGGCGATACTATCTTCCTCGAGAGCCAAATCGTTGACAAACGCGACTTCATGGAGGAGAACGACCGCATCTGGGGCAAGAAGTTTGTAA,Eukaryota,Excavata,Discoba,Discoba_X,Heterolobosea,Tetramitia_VI,Tetramitia_VI_X,Euplaesiobystra,Euplaesiobystra_salpumilio,83,37,37,9,9,9,9,9,9,seq152,seq152,7,7 +GAACTGCCGCGAAAGGAGAACAGATATGAAGCTTAAAAAATTCGGAGCTACGGAACAGCGCGTCTACGATCTGGTGAAGCCTGTCACGGATGAGCTGGGATATTATCTTTGGGACGTCTGCTTTGTCAAAGAGGGCGCTGTACGCTATCTTCGTATTTTCATCGACTGCGACGAGGGCATTTCAATAGAGGACTGCGAAAGAGTCACAGCGCCTGTTGATCGCCTGCTTGACGAGGCTGAC,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,87,18,14,14,11,11,11,11,11,seq153,seq153,7,7 +TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTCCAAAATTAAAGCCCGAAGCTTAACTTCGGTTCGTTTTGGAAACTGGTAGACTCGAGTGTGGTAGAGGCAAGTGGAACTTCTAGTGTAGCGGTAAAATGCGTAGATATTAGAAAGAACACCAGTGGCGAAGGCGACTTGCTGGGCCACCACTGACGG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,64,40,12,12,12,8,8,8,8,seq154,seq154,7,7 +TGCGCTGAGTGCCGGGAAATACAGTATGGAGATGGTCAAAGAGCCCTCTGTTGGCAATCCCGCCGCGGGTTCATTTTGAAATATGTCCGGTCATCATAATTGAAATTATTCTCAACAAAAACTTGTACTGCAGTATCAGTGGTGCTATAATATAGCCAATGACAGAGAGCGGATTCAAATTACCAATAAAGGGGCGCATACATATGGACATCACAAGAGATATTTACTACGTCGGCGTCAACGATCATCAGCTTGATCTGTTCGAAAGTCAATACATTGTCCCCAACGGTATGGCTTATAATTCCTAT,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,2,2,2,2,2,2,2,2,2,seq155,seq155,7,7 +AACCAAACCAGCAAGAGAAGCATTCAAACACATGGAAACATCTGGTTTGCCATATTTGATCCATGTGAAGATCATGCAGGCAAATGTGGCAACTGCTGGTGCAATAGTAGTGGTAACGAAAATAGAGCCAAGCTGGGAAACAGAAGTAGCAGCGGCACCATTAAAGCCGTACCAACCAAACCAGAGAATAAAGCAGCCCAATGCACCAATTGTCAAAGAGTGACCGGGAATGGCATTGACTTTGATTTTCCCATCTTCTCCCTTTGTGAATTTACCAATTCGAGGGCCAAGAATGGCTGCACCAA,Eukaryota,TSAR,Stramenopiles,Gyrista,Bacillariophyceae,Bacillariales,Bacillariaceae,Fragilariopsis,Fragilariopsis_kerguelensis,90,21,11,11,10,9,9,9,9,seq156,seq156,6,6 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGCTTGGTTGGCCGGTCCGCCTTTTTGGCGAGTACTGGACCCAACCGAGCCTTTCCTTCTGGCTAACCTTTCGCCCTTGTGGTGTTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTACCTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Debaryomyces,Debaryomyces_hansenii,100,100,100,100,100,100,100,100,100,seq157,seq157,6,6 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCGGAGCACCTTTTGCGAGTAACCGAAAGGTTATTACAATATAGGGGTGCGTCCTTTATCTGACCCATTGGTATGCCATTCATTTGGTGTGCCACTTTGGTTAGAAGTTTACCTTGAAAAAATTAGAGTGTTTAAAGCAAGTGAATAACAGCCTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTTCTATTTTGTTGGTTTCTAGGACCGAAGTAATGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Chytridiomycota,Olpidiales,Olpidiaceae,Olpidium,Olpidium_brassicae,100,94,94,86,37,16,16,16,16,seq158,seq158,6,6 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCAAGGGTCGCATCGGGGACAACCCGCATGCTCCCTACCAGTCTTAGACTGTTACTGTGAGAAAATTAGAGTGTTTCAAGCAGGCTGTTGCAGGAATACATTAGCATGGAATAACGAATGTGTCTAGAATCTTGGTTAATTCTAGATTACGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTGTTAAAGACTAACGTATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,100,100,100,100,100,98,98,78,78,seq159,seq159,6,6 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCGTCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,67,67,seq160,seq160,6,6 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,99,99,99,99,34,seq161,seq161,6,6 +CATAACTATCGGTACGATTAGCTGTGACAATCTGCTCACCAGGAAGATTATAAATTTGGTCAGGCACCCAGTCAGCAATCACTTGGATACCAGCAGCGTGAAGAGAACGAACGGCATCTAGCAAGTCATTGAAAGAACCATACTTGTTGTTTTTACTCATAGCAATATCATAGCGATCCTCAAAGGCGTAGCCGTTTCCGATGATAGCG,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,14,13,12,12,12,12,12,12,12,seq162,seq162,6,6 +CTACACCGTGCAGGGCACCGGCACGAAGGTTGTAACCCTGTATGTGGACGGCAGTCAGTACGAGACGGCGACAGTGACGAGGAGCGGCGTGACCAACGGCAGTTTCACGATAGCGATGAGCGGTCTGGGCGTAGGCCGCCACACGGTGCAGCTGGTGGCCGAGATGGAGGCCAGCGCAGACCTAACGCTGAGGAGCGAGAGCATCTACATGGACATCT,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Diplomonadida,Hexamitidae,Enteromonas,Enteromonas_hominis,85,13,5,4,4,4,4,1,1,seq163,seq163,6,6 +GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCGGGCTGCTAAGTCAGCGGTAAAATGTCGGGGCTCAACCCCGGCCGGCCGTTGAAACTGGTGGTCTTGAGTGGGCGAGAAGCATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCATGCCGGCGCCCGACTGACGCTGAAGCACGAAAGCGTGGGTATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGAATGCTAGGTGTCCGGGGCGAGCGAGTCCTGGGTGCCACAGCGAAAGCGTTAAGCATTCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,76,76,76,seq164,seq164,6,6 +ACATGGAAGTATCCCAAGAAGCCCTATCGTCTGAAATTCGCAAAAAAGACAGACATGCCCGGGTCTCTGCGCTCTAAGAATTTTGCATTGATAGCTAATTATATAGATTGCACACTGATGCGCAACGCCATTGCGTTTGAAGTCGGGCGTCTGTTGGGAATGCCATTTACCAACCATGCCGTCCCCGTGCGGGTTTATCTCAACGGACGTCTGAAGGGTGCGTATTTCCTGACCGAAAAGATTGGCATCAGTAGCAGCAGCGTGGATATCGATGAAAGCACTGGAGTTCTCCTTGAGCTCGACTCCAATTATGATGAGAAATATTGTTTCCGCTCTCCGATATACAATCTGCCGGTGATGA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,79,8,4,4,4,4,4,4,4,seq165,seq165,5,5 +AGCTCCAAAAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGCTTGGTTAGCCGGTCCGCCTTTTGGTGAGTACTGGATCTAACCGAGCCTTTCCTTCTGGGTAACCTTTCTTTCGGGGAAGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGACGGGGGTATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTACTGCGAAAGCATTTACCAAGCACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Candida,Candida_austromarina,100,100,100,100,100,100,100,75,37,seq166,seq166,5,5 +AGCTCCAATAGCGTATATTAAAGTTGCTGCTGTTAAAAAGCTCGTAGTTGGATTTCTGATATAATATTACTGTCCCGCTGTGGTTACACATGTGAGTGACGGATAATATTTATCTGTTTTGTGTTTTTTGTTTTGATAGTATGTAAATATTATCAAAAACATACACATCTTTTACTTTGAACAAATTAGAGTGTTTCAAGCAGGCATTTATGCCTTGAATACATGAGCATGGAATAATCATCGAGGACTTTTAGTTCTATGTTGATTGGTTCTAGAACTATAGTAATGATGGATAGGGATAGTTGGGGGTGCTAGTATTCCGAGGCCAGAGGTGAAATTCTTGGATTCTCGGAAGACTCACTTAGGCGAAAGCATTCACCAAGGATGTCTTCA,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,100,100,100,100,100,100,99,99,99,seq167,seq167,5,5 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGAGATGGGTCGGCCGGTCCGCCTTTTGGTGTGCACCGATCGTCTCGTCTCTTCTGCCGGCGATACGCTCCTGTACTTAATTGGACGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Panax,Panax_notoginseng,100,100,100,100,100,100,100,85,83,seq168,seq168,5,5 +CCAAAGGATAAATATGAACAGGAGGAAAAACAAAATGGTAAAACAGAAAATCGGAAAGAGAATTCTGAGCCTTGTAATGGTGCTTTGCCTTGCTGCTTCGGTATTTTCTATCCCGGTTGTTGCGAACGCTGTAACAACCGAAAACGTAACGGCACAGGCAACGGATTACGGTCTCGTTGACGATGTTCAGCAGGGACAGATACTGCAGTGCTGGAACTGGTCGTACAACGGCATAAAGAACAATATGCAGAAGATTGCCGAGCAGGGCTTCTCTGCTATCCAGACCTCGCCTATCCAG,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,85,19,14,14,14,14,14,14,14,seq169,seq169,5,5 +CGCTTTAGCTCCGATTACCGCCGTCACTGATTTGCGCGGCGTTAAAGTGCAGCTTTCCGTCGCGGTCAGCCCGATGCGCTTGTCTGCCTGTAAAAAGCGGATAAGCTCGCTCTGCACGGCTATGGGATAATCGCCGTAGCCGGGACTGTACCGCCATGTGAACCCGCAGCCTGCAAGTTCGGCGGAGGCTTTTATCTCCTTTTCGGCGTCGTCGCAGAAGCTTTCCGTAAGCGCGGAAGCCGCCGCG,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Malacostraca,Porcellionides,Porcellionides_sexfasciatus,93,69,69,65,44,24,24,4,4,seq170,seq170,5,5 +TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTCTAGAATAAAAGCCTGGAGCTTAACTCCAGTTCGTTCTAGAAACTGATATACTTGAGTGTAGTAGAGGCAAATGGAATTTCTAGTGTAGCGGTAAAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGATTTGCTAGGCTATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,64,43,21,21,21,6,6,6,6,seq171,seq171,5,5 +TGGAGCCGATTTATAAGATGCTCGCCCACTACACGGGCATTCTCCACCAGGTTTTCAGCCTCAACCACTTCAGCCACGGCGATAGCAGCAGCACATGCCAGGTGATTTCCGCCGAATGTGGTGCCGAGCATACCCTTTTTAGCCTCAAACTCAGGAGAGATCAGCACTGCTCCTACAGGGAAGCCGTTGGCTATACCTTTTGCCATAGTGATAAGGTCAGGACGGATATCGGCATACTGATGAGCGAAGAACTTGCCTGTGCGCCCATATCCGCTCTGTATCTCATCAAGGATAAGCATCACGCCATGCTTTTTCGTCACCTCACGCAGAGCGCGGAGGAAGTCGTCGCCGGGCATACGGATCCCTGCCACACCCTGTATGCCC,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,90,26,17,17,11,11,11,11,11,seq172,seq172,5,5 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTGTTCTGGCCATCCTTCCAATCGCTGCGTGCTTTTCTTCATTGATTAGTGCGTGGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,63,63,seq173,seq173,4,4 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTAGGCCTGGTTGGACGGTCTGCTCTAGGGTTTGTACTGTCCTGACCGGGTCTTACCTTCTGGTGAGCTGTCGTATTGTTTACTCAGTGCGGCAGGGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCATTCGCTTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTTCTATTTTGTTGGTTTCTAGGACCGAAGTAATGATTAATAGGGATAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Mucoromycota,Mortierellaceae,Mortierellaceae_X,Mortierella,Mortierella_hyalina,100,100,100,100,100,100,100,100,64,seq174,seq174,4,4 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTCGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCGACGGGGACTTACCTCCTGGTGAACTGCGATGTCCTTTACTGGGTGTCGTAGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_yarrowii,100,100,100,100,100,100,61,61,61,seq175,seq175,4,4 +AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGAAGTATATAATGAATTTCTAATTCAATTGTATATTGATTCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_sp.,100,99,99,96,96,96,96,89,70,seq176,seq176,4,4 +AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGATGGGCCGGCCGGTCCGCCCTAGGTGTGCACCGGTCGTCTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Nicotiana,Nicotiana_attenuata,100,100,100,100,100,100,100,84,84,seq177,seq177,4,4 +AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCGTCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAATTGCGTGTGTTGGTGTTTCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,31,30,seq178,seq178,4,4 +ATACTGGGCTGTACTTGGAGCTGGAAAGAAAAGGAAAAGATTAAATCAAAATCTAATTTGTGTCTCTCAGCATCATAACGACAGTCAAGAGGCAGCCAGGTGTACTGGGGAGCTCACAGGGTTAGGGTGGGACAGAATTAGGCTCACACCTGATTCTGCTCTGTAAGAATGTACAGCCCATTCAACTAACTTCTCCAAGCTTCAGAGCTTTAATTTTC,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,2,2,2,2,2,2,2,2,2,seq179,seq179,4,4 +ACAGGACTTCAAAGCCGTTTTTATGGTTATACGCGCGCGAGCTGTTTTTCGCGGCAGTTTTGGCGAGGTTTTCCGAGGCGGCAGGGAGTGATTTTTCTGATATTTTAGGTGTGGGATCTTTCTTTGATTTAGGCTGTTGGGGTTTCTTTTTTTTGCTCATTTTTTCGACTCCTATCTGCTTTATACTGCGAACATTATAGCACAAAAAATTATTTGTG,Eukaryota,TSAR,Rhizaria,Foraminifera,Globothalamea,Robertinida,Robertinidae,Robertina,Robertina_arctica,92,17,9,7,3,3,3,3,3,seq180,seq180,3,3 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTTGCCGGCCACGGTTTCGTGTGTTAGTGCTTGATGTTCTGGCCATCCTTCCAATCGCTATCGGCTGCTCTTCATTGAGCGGTCGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,99,88,85,seq181,seq181,3,3 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCAGCGCTGTTCTTGGCTCTCTGAGTCGGCTCAGTGTTTGGTCATCCGTATGGGAAGCTAGCTCGGCCTTCACTGGTCGGCTAGTGGATCATACACTTTACTTTGAAAAAATTAGAGTGTTTCAGGCAGGCAATTGCTTGGATACTTCAGCATGGAATAATGGAATAGGACTTTGACCTATTTGTTGGTTTCTTCGAGGTCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCATATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGATGAACTTATGCGAAAGCATTTGCCAAGGATGTTTTCATTAA,Eukaryota,TSAR,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Pseudoplatyophrya,Pseudoplatyophrya_nana,100,100,100,100,100,100,100,49,49,seq182,seq182,3,3 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTGGGGTGGGTTGACCGGTTTATCGAAAGATACATACTGGTCGACTGACTCTTTTCTTCTGGGATAGCTTCTGCTCTTTACTGAGTGGTTGTGATTTCCAGGATCTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCTTGTATACATTAGCATGGAATAATAGAATAGGACTTTTGATTCTATTTTGTTGGTTATAGAATCGAGTAATGATTAACAGGAACAGTCGTGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACTAACTAATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Chytridiomycota,Rhizophydiales,Gorgonomycetaceae,Gorgonomyces,Gorgonomyces_haynaldii,100,62,62,45,26,8,5,5,5,seq183,seq183,3,3 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCCTTTGCCGCCCGGTCCGCCTATTTGGGTGTGTACTGGAGCGGTGGAGGCTTACCTCGTGGTGAACGGCCATGTCCTTAACTGGGTGTGGTCGGGAACCATGACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTACGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTCCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Occultifur,Occultifur_externus,100,100,100,100,100,100,100,35,35,seq184,seq184,3,3 +AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCGCCTGATGGCCGGTCCGCCTTCTTGGTGTGTACTTGCTCATCGGGCGCTTACCTCCTGGTGAACTCTGATGTCCTTTATTGGGTGTCAGAGCAAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Sporobolomyces,Sporobolomyces_roseus,100,100,100,100,100,100,60,60,59,seq185,seq185,3,3 +AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGATTGTTATGATGAATCTATGTATTCGATTATAATATGAATCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,100,98,97,97,97,97,97,95,46,seq186,seq186,3,3 +AGCTCCAATAGCGTATATTAAGATTGTTGCAGTTGAAACGTCCGTAGTTGAATTTCACTCAGAAACAGAGTACTCTTGCATTTCGCTTGAGAACTTTGTTTCGAGCGGGAATTTGCGCAAGCATTTTCTCAACGTTACTTTGAGCAAACTGGAGGGCTCCAGCCGGGTATTATGCCTTAACAGCTCAGCATGGAATAACAAGCTAGGATTTTGGTCCATCTTGTTGGTAACATGGATTAAAATAATGGTTGATAAGGATGTACGGGGACATTCGGACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACGAACAAGTGCGAAAGCATTTGTCCAGTACTTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,98,84,81,81,80,80,78,47,39,seq187,seq187,3,3 +CGAAATGAACCCCGCAAGATAGAATTTGGCACGATCTCCCGCATCAATATATCTTTTTTTGACGGGAGAAAAATACCCCGAAATAAAGCTCTCATTGACGATTCTGTTAGCTTCTTTTATAAGTTGAGTTTCCGTTATTTTGTTTTTTTGCGCCGTTTCGCTCATGACAAAATAAGCCTCTTCAAAAAACTGCGAACCTGTATTTTTCAGCATAAGCATTCGTTTTTCCTGACTGTTCACAAATATCACCGCTCAAATTATCGACCGTGTTTTTTTGCAATATTCAAAAACGGCATAATTATTTCGGCGCAA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,36,36,36,36,36,36,36,36,36,seq188,seq188,3,3 +CGGCAAAATCAAAAGTTTTACCCCTGCCCTTAGCTCCGACACTATGGATTCCGTGACAATCTGCAAAATGTAACGCTAAGGCATCTGACAGATTCGTCTTGCCTGTTCCGCTTGAACCCTCAATATATATCAGAGTACGCGAACAAGGATTTTTTTCATAGAAACGCTTTACATATTGAAGCCAGAAGTCATAGGCACGGTCGTACATCGGCAGA,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,84,38,21,21,17,10,10,10,10,seq189,seq189,3,3 +TTCCCGGAGGCGGTGAACTACCTGCTGGACTTCCATGGCCGCGCCAGGGACTCCCCCAACGAGCGCCCCAAGCCCAGGGCCAAGCCGCCCCAGGACGAAAAGGTGCCCTTTGCCCTGCCCCCGCCCAACGCCGACCAGCGCCGGGTGTTTGCCTATCTGCGCAAGCGGGGGATCGCCGCCCAGGTGATCAACGGCTTCATCAACGCCGGCCTGCTCTACGAGGACGCCGAGCACCACAACTGCGTCTTTGTGGGCCGGGATACCGAGGGTGCGCCGGTGTTCGCCAACAAGCGCGGCACCTACGACCGGGACGGCAGCAGCT,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Diplomonadida,Hexamitidae,Giardia,Giardia_ardeae,97,21,8,8,8,8,8,8,6,seq190,seq190,3,3 +ACAACTGAACTCAATGCATTTACAAGGAATTCATTGCTCTTTTCCAATTTTTCCCTGCTTTCCATCAACTGTTTTGTCCGTTTTTCCAGCTTCTTCTCCATATTGAATCTGTTTTCAAACAGTTCAAGGATATTTAAAGTACGGCGTCTGATTACTTTAGCATCAAAAGGCTTATAAATAATATCGGATGCACCGTATTCATACGCTTTCTCATCTGTCTCTGCCGTTGCCTCTCCGGTAATCATAATGACGGGAATCTTATTCATATACTCCTTTTCCCGCATATGTTCCATTACTTCAAGACCTGATTTTTTCGGCATTACAAGA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,19,18,18,18,15,15,15,15,15,seq191,seq191,2,2 +AGCACCCGCGGTCAATGATGTATTTGCCAAAGCGGCTGCAAGTGATGCAAACGGCGAATATATGCGTGATGCAAATGGCTGTTCCTGGGCAGTATTCAATATCAAAGGCGGGAACGGACTTTCCGTAAAGGACGGGAGCGCCGAAGACGGGCTTTATACCGAACAGCATTTGAAGGACTTAGGGCTGTATGATTACTATACCGTTGAAAACCTGATCATCAATGCGGATTTAAAGCTGGCGCCCTCACAGCTCAGCTTTAAGATGCCGGATGATAAGA,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,87,5,1,1,0,0,0,0,0,seq192,seq192,2,2 +AGCAGCTGCCGATCGAATCGTGCGCGTGATTTTTCATCAGCTCTTTCCATATTGCCTCGATAAGCCCATGATGATATTTGAAGCCTAAGCTGTAAGCAATGGATGCGAGCGGTTCAAGAATATTTGTGATCTTATTCTCAATTCTTGTGTTCGCGGATTTTAGGTCAGCTCTTGAGGAATAGATGCTGCGGTGAACACGCATATATTTTCCGTCAAGGAACTCGCCCTCAATAGTATCGAGCGGAACCTTTTCCAGCTTGTCAAAAATTTCCTCATACCTTCCCATGCGCGTTTTTCTATCGGGGTACAGCTTT,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,9,8,8,8,8,8,8,8,8,seq193,seq193,2,2 +AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTAAGCCGGCCACGGTTTCGTGTGTTAGTGCTTGGTGTTCTGGCCATCCTTCCAATCGCTGCTGGCCTTTCTTCATTGATAGGTTGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,73,73,seq194,seq194,2,2 +AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTGGATCTGCTGCAGTGGTCAGCCTTCACGGTTTGTACTGCTTGCTGGTGGATCTTTTTACCTTCTAGGGAGCCATCATCCTCTTAGTTGAGTGTGATGGGGATCTAGAACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCATCTTACGCTTGAATACATTAGCATGGAATAATAAAATAGGACTTTGGTTTTATTTTGTTGGTTTCTAGGACCGAAGTAATGGTTAATAGGGATAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Blastocladiomycota,Blastocladiomycotina,Blastocladiomycetes,Physoderma,Physoderma_lycopi,100,100,100,100,100,100,100,100,99,seq195,seq195,2,2 +AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCTTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGAGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,90,90,90,88,87,seq196,seq196,2,2 +CGCTTCTGGGCGAAATTCCTGTACGACATCGGCGTGATCCAATGCAAGGAGCCTTATCGCAAGAGAACCGCTCACGGCATGATCCTCGGCGAGGACGGCGGCAAAATGTCCAAATCCCGCGGAAACGTGATCAATCCCGACGATATCGTAAGAGACTACGGTGCCGATTCTCTGCGTATGTACGAAATGTTTATCGGCGACTTCGAAAAGAGTGCGCCTTGGTCGCAGGCT,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Malacostraca,Bathymedon,Bathymedon_obtusifrons,90,50,50,45,36,29,28,6,6,seq197,seq197,2,2 +GTGTCACGTCACAACAATCTCCATTCTGCCGCAAAGCGGCAACATAACTCGTGATGAAATTCGGCGGTCACGCCGACAAAATGGGGTTGGGTCCCATTTTGAGCTGAATTGGGGGCGTGAAATAACGTGTTTCCCGTTTCCGCATACCTCTATTTTCACGCTATGCCTTCCCCGATGCCGGTGCCGTTTTTCCGAATGGCGGCCGACTGTTTTTCGTGCTCGGAGGCGGTCTTTGAGAAGTAAAAAAGACCGTCATCGTCGCTGACGAAGAAGAAATAACTGTGCTCCTCGGGA,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,96,19,17,17,16,16,16,16,16,seq198,seq198,2,2 diff --git a/protist_filtered.csv b/protist_filtered.csv new file mode 100644 index 0000000..8b6d7c7 --- /dev/null +++ b/protist_filtered.csv @@ -0,0 +1,59 @@ +SeqName,Pident,Accession,rRNA,Organellum,specimen,Domain,Supergroup,Division,Subdivision,Class,Order,Family,Genus,Species,sequence,tax.Domain,tax.Supergroup,tax.Division,tax.Subdivision,tax.Class,tax.Order,tax.Family,tax.Genus,tax.Species,boot.Domain,boot.Supergroup,boot.Division,boot.Subdivision,boot.Class,boot.Order,boot.Family,boot.Genus,boot.Species,JIN-Nu-mul,JIN-Nu-mul2 +seq2,99.2,AY321149.1.1455_U,18S_rRNA,nucleus,strain_T,Eukaryota,Metamonada,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,76,11240,11240 +seq10,99.5,MT991542.1.2393_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTTTGGTGTGCACTGGCATGGGCTCGCCTCGCTGGCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGCCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTACGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,100,100,100,100,67,920,920 +seq17,92.0,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCATTGTGCTTATGTGTCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,86,86,83,339,339 +seq22,95.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTCGCAAGAGGCGAGAGTGCCATTAGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTTTACCTACAGGTAAGATCAATGAGAGCCACCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,98,98,98,98,90,275,275 +seq25,100.0,EU091849.1.1430_U,18S_rRNA,nucleus,clone_P2-3m2,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Watanabea-Clade_XX,Watanabea-Clade_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,94,238,238 +seq31,100.0,MZ558750.1.2453_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_orientale,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,61,188,188 +seq35,91.2,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTGGCGTATGTGCCTTAGGGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,80,80,80,80,79,120,120 +seq38,90.9,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTCGGCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTCTACTTCCAAGTAAAATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAAGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,81,81,78,78,75,107,107 +seq43,90.8,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCGGAATGACTCAGCGCAGTATGATATCTTTACCTCGAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,63,63,62,62,61,78,78 +seq47,99.5,EU091849.1.1430_U,18S_rRNA,nucleus,clone_P2-3m2,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Watanabea-Clade_XX,Watanabea-Clade_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCCGCCGGTCCGCCTCTGGTGTGCACTGGCGTGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,94,65,58,58 +seq51,91.6,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCTTGTCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,83,83,82,82,79,49,49 +seq53,94.2,FJ355409.1.1048_U,18S_rRNA,nucleus,clone_051102_S1_W_T_SDP18_098,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTCGAGACGCAGCCAGGCTCAAGGGCCGATACTGCGGATTGGGACCATCCTCGAGAAGAACATATCTGTCATTGAGTTGATGGGTATGGGACTCTCGTCTTTTACTGTGAGCAAAATAGAGTGTTCAAAGCAGGCTTACGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACTTCGGTCTATTTTGTTGGTTATACTCCGAAGTAATGATTAATAGGGACAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,100,100,100,100,100,100,70,70,70,48,48 +seq54,90.5,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTATGTACCTCCAAGTACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,88,88,88,47,47 +seq59,97.4,KF648870.1.1484_U,18S_rRNA,nucleus,strain_MTZ1,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCAGCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,100,100,100,100,100,100,100,58,54,40,40 +seq60,88.5,NGBS01000218.13865.15598_U,18S_rRNA,nucleus,strain_LA,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,ATCATTACAAGCGTATATTAAAATTGTTGCATTTAAAAAGCTCGTAGTTGAATAATAGATTTGAAGTTAGATTGACCTAGTCAAGATATTCTTCATTTCTTTTGTTATATTTTCGGATATAACCATTTACTGTGAAAAAATTAGAGTGTTTAAAGCAAATTGTAAATTTGAATATTATTAGCATGGAATAATAATATATGATTAATATTATATATTATGGTAATATAGTATTAATAATGATTAATAGGGATAGTTGTGGGTATTCATATTTCATAGTCAGAGGTGAAATTCAAGGATTTATGAAAGATGAACGAATGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,99,99,97,97,97,97,77,77,77,39,39 +seq62,90.4,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCAAGGCTTATGTGCCTCACAGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGGATTTAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,72,72,70,70,65,38,38 +seq63,90.1,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCGGAAGTGCCTCCAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,86,86,85,85,82,37,37 +seq71,91.2,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCCTATGTGCCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCATGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,73,73,67,67,64,31,31 +seq85,92.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTTTTGGCTTATGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,85,85,73,22,22 +seq92,100.0,MZ067863.1.738_U,18S_rRNA,nucleus,strain_ACKU_309-4,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGTGCGACGACGCGGTCTGCCTCTGGTATGTACTGCGCTCGGCGCACCTTTCTGCCGGGGACGGGCTCCTGGGCTTTATTGTCTGGGACTCGGAGTCGGCGAGGTGACCTTGAGCAAACGAGAGTGTTCAAAGCAAGCCTACGCTCTGAATCATTTAGCATGGAATCACGTGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGAACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,100,100,100,100,100,100,100,100,100,20,20 +seq93,97.9,AY321149.1.1455_U,18S_rRNA,nucleus,strain_T,Eukaryota,Metamonada,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,AGCTCTGCGAGTTTGCTCCCGTATTGTTGCAGTTAAAACGCCTGTAGCCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,66,20,20 +seq96,100.0,FJ592322.1.1085_U,18S_rRNA,nucleus,clone_B06_SE1A,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,60,60,19,19 +seq98,100.0,AB695526.1.1759_U,18S_rRNA,nucleus,clone_MPE2-32,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,AGCTCCAATAGCGTATATTAATGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGACTTGGCGCACCTGGCCCGCCACGGTTACGTGTGTGAGTGCCGGGATGCGCCTGTCACTTTTCTAGTAAACTATTGTGCTCTTCATTGAGTGTGATAGGTAGCTAGATAATTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCGTTTGCTATGAATACATTAGCATGGAATAATAACTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,100,100,100,100,100,100,100,93,93,18,18 +seq99,84.9,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGGCGTTCGGGTGGTACTGGGAGGCCGGGCCCTAGGCTCTGTGCTTCCTGGAGCCACCTTCTGAGTGAGGCAACTCACTTCGGTTACCATGAGAAAAGTGTAGCGCTCAAAGCAAGCTAAGCTGAGCATTTAAGCATGGGATAACAGGCTATGACTCCATAGACGCTGTTGGTCCGTTTAGCGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCCAAGACCGCCGTCAGACTAACTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,98,98,18,18 +seq103,92.3,EF024471.1.1761_U,18S_rRNA,nucleus,clone_Elev_18S_1147,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTCTGCTAGCGAGAATAGGTCATCTCTTTGAGTATGTACTTGTTGTCGTTGGCATTAATCCGATTCATCTAACAGTTAAACCAAACTGTTGGATATTCGGAGCTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCCTTGAATACTCCAGCATGGAATAACAAGTAAGGACTCAAGTTCTTCTTGTTGGTTTAAGAGCCTGAGTAATGATTAAGAGGAACAGTTGGGGGCATTCGTACTTAGTAGTCAGAGGTGAAATTCTTAGATTTACTAAAGACGAACTACTGCGAAGGCATCTGCCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,75,75,16,16 +seq104,98.9,EF200530.1.1708_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCTCACGGTGAGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCTGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,100,100,100,100,100,100,100,48,48,16,16 +seq108,100.0,DQ388549.1.1646_U,18S_rRNA,nucleus,,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTTGGATCTCGGTCTGCCTCAAACGAGGTATGTACCAGGGATCTGAGACCATCCTCGAAGAAAACATGTCTGTCATTAAGTTAATGGGCATGGGATCTTCGTCATTTACTGTGAGCAAAATAGGGTGTTCAAAGCAGGCTTATGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACCTTGGTCTATTTTGTTGGTTTGTACTCCAAGGTAATGATTAATAGGGATAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,100,100,100,100,100,100,100,100,100,14,14 +seq110,87.8,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGATGATCGAGCATAGGTGAGGCACCGGGCTTTTTAGCTCTGAGTACCTTAGCGATGCCCTAAGGATGTTTAATCATCCCGGTTACCATGAGAAAAATGTAGCGCTCCAAGCAAGCTTTGCTGAGCATTTTAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCATAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,14,14 +seq113,84.5,EU087208.1.865_U,18S_rRNA,nucleus,clone_1_39,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Amoebogregarina,Amoebogregarina_sp.,AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTCGGATCTCGTCCCCTTAATAAGGAAGCGGAGATTTTTTCTAATCTCTATGCGCCTTATTGTCTGGACTACGGTGACTCAAGGCAACTTGAATGTCACCCTGTTACTTTGAGCAAATTGGAGTGCTCCAACCAAGCCTAAGCTTGTACAGCTCAGCATGGAATAACGAGATAGGACTTTGATTCTTCTTGTTGGTGTCACGAATCGATAGTAATGATTGATAAGGAAATTCGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCTGCAAAGACAAACGAATGCGAAAGCATTTGCCCAGTATCTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,100,98,98,98,98,98,98,41,26,13,13 +seq115,100.0,KM020169.1.1775_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Coccomyxaceae,Coccomyxa,Coccomyxa_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Stichococcus,Stichococcus_bacillaris,100,100,100,100,100,99,99,54,54,13,13 +seq116,100.0,X74000.1.2286_R,18S_rRNA,nucleus,strain_Andreyeva_748-I,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTGCCGTCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCATATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACCGCTGCGAAAGCTTTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,100,100,100,100,100,99,99,99,99,13,13 +seq118,100.0,EU709161.6.1494_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGCTCAAGAATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTTCTGTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGATTGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCTGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,98,12,12 +seq121,88.0,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTAGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGACGATCAAGTACAAATGGATTGCCGGGCATTGAGCTCTGAGCAGTCTAGATGTACTTTACAAACCTGAAAAGGTTTCTGTTACCATGAGAAAATTGTAGCGCTCAAAGCAAGCTTAGCTGAGCATTATAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGACAGAGGTGAAATTCCGAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,12,12 +seq125,90.5,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTATCGGCGTAAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCAGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,71,71,68,68,64,11,11 +seq127,90.8,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAGTTGTGGCAGTAATAGTGGGTCATCTTTAACGAGCATGCACTTATTGTTATTGCCATTATTCTGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,100,99,99,99,99,98,66,66,10,10 +seq130,100.0,AY884326.1.1078_U,18S_rRNA,nucleus,strain_CeS-2,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTTCGGCGCACTTGGCCCGTCTCGGTTTACGGGATTGTGTGCCGGTGTGCGCCATCCATCCTTCGAGAGAACACTTCTACCCTTCACTGGGTCGGGAGTGCTATCTCGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,100,100,100,100,100,100,100,100,91,9,9 +seq131,100.0,AJ579340.1.1732_U,18S_rRNA,nucleus,strain_SAG_30.83,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGTCGCTGGATGGCCCCCTGCCTCACGGCAGCTGGTTTGGCTGACTCCTAGCGTCCCATCCTCGGGTGGGTCCTGCTTGGCATTAGGTTGTTGGGCAGGGGAAGCCCGTCTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTAGGCCGTTGAATACATTAGCATGGAATAATGAGATAGGGCCTTGATGGATTCTTCTATTTTGTTGGTTTGCACGCCAAGGCAATGATTAACAGGGACAGTTGGGGGTATTCGTATTCAAATGTCAGAGGTGAAATTCTTGGATTTTTTGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,100,100,100,100,100,100,100,89,89,9,9 +seq132,100.0,JN573894.1.1681_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Diplosphaera,Diplosphaera_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGTCGGTCCGCCGTTTCGGTGTGCACTGGCGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACCCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Pseudostichococcus,Pseudostichococcus_monallantoides,100,100,100,100,100,100,100,99,99,9,9 +seq133,100.0,FR693368.1.1633_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGTGCTGCCGGTCCGCCCTTTGGGTGTGCACCGGTTGCGCCCGTCCTGCTGCCGGGGACGGGTGCCTGGGCTTCACTGTCCGGGTCCTGGAGTCGGTGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCCCGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,100,100,100,100,100,100,100,100,100,9,9 +seq134,99.2,JQ993661.1.1396_U,18S_rRNA,nucleus,,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,AGCTCCAATAGTGTATGTTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGTGGTCATCCGGCTCCGCCCGTATGGGTGGGCGCCTGGTTTGCCCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTCACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,53,53,9,9 +seq135,91.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGTGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCACCAAGTAAGGTCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,89,89,89,89,84,9,9 +seq141,76.9,AY919792.1.1726_U,18S_rRNA,nucleus,clone_LG32-03,Eukaryota,Alveolata,Alveolata,Chrompodellids,Colpodellidea,Colpodellida,Colpodellida_CHR1,Colpodellida_CHR1_X,Colpodellida_CHR1_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGGACTTCTGTCCGGGTGCTGTTCTCCGCCGTAAGGCGTGTAGATCAATACCTCGACATCCGTTCGTTTGAGCTTCTTGCTCTTTCGAACTCTTTACTTTGAGAAAATTAGAATGTTTCAAGTAGGCTTTCGCCTGAATACTACAGCATGGAATAATAAGATAGGACTCTGGTTCCTTCTTGTTGGTTCTTAGAACTAGAGTAATGGTTAATAGGGACAGTCGGGGGCATTCGTATTCTACCGTTAGAGGTGAAATTCTTGGATCGGTTGAAGACGAACAACTGCGAAAGCATCTGCCAAGGATGTACTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Branchiopoda,Caenestheria,Caenestheria_lutraria,98,55,55,55,11,5,2,1,1,8,8 +seq142,99.0,AJ431572.1.1716_U,18S_rRNA,nucleus,strain_SAG_35.83,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Desmococcus,Desmococcus_olivaceus,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCGTTTCGGTGTGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,63,63,8,8 +seq144,98.0,EF586127.1.796_U,18S_rRNA,nucleus,clone_121-O29,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTTGAATTTCGTTCAAATTGAATAGTGCCGAGTTTTCCTCGTGTTAATTCGAGTACTTTTTGATTAGAACTTGGGTGAGATGTACTCGTTTCGACGAGTTCGTTTCGCTCCGTTACTTTGAGCAAATTGGAGTGCTCCAACCAGGCTTAAGCTTGAACAGCTCAGCATGGAATAACAAGATAAGACTTTAGTTCTTCTTGTTGGTGACATGAACTAATAGTAATGGTTGATAAGGACATACGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACAAACAAGTGCGAAAGCATTTGCCCAGTATGTACCTGTTAA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,100,100,100,100,100,100,100,87,76,7,7 +seq145,97.2,MG775599.1.1357_U,18S_rRNA,nucleus,strain_SF33_R_Tri,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_sp.,AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTATGGCGCACTTGGCCCGCCGAGGTTTCTCGGTCGTGTGCCGGTGTGCGCCTGCCATCCTTCTGGGGAACGGCCTTGCCCTTCACTGGGTGGAGGTCGGTATCCAGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_jutlandica,100,100,100,100,100,100,100,49,21,7,7 +seq146,99.2,AM114805.1.2266_U,18S_rRNA,nucleus,clone_WIM47,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGACTGAAGCATTTTGCCGGCCGCGGTTTCGCGTGTTAGTGCTTGATGCTCTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGAATGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,100,7,7 +seq147,98.4,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAACTTTGGTAGTAATAATGGGTTATCTCTTTGAGTATGTACCTATTGTTACTACCATTATTCCGATCTATACAAGGGGTAATTCCCTTGTGTTCTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTGATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAGCGCCTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAGCAGTCGGGGGTATTCGTATTCAGTCGTTAGAGGTGAAATTCTTAGATTGACTAAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,100,100,7,7 +seq148,90.6,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTGTGGTAATAACAATGAATCATCTTTAACGAGCATGCACTTATTGTTGTTGCCATTATTCCGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,99,96,96,96,96,96,51,51,7,7 +seq159,98.7,AY124366.1.737_U,18S_rRNA,nucleus,clone_CSE28,Eukaryota,Alveolata,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCAAGGGTCGCATCGGGGACAACCCGCATGCTCCCTACCAGTCTTAGACTGTTACTGTGAGAAAATTAGAGTGTTTCAAGCAGGCTGTTGCAGGAATACATTAGCATGGAATAACGAATGTGTCTAGAATCTTGGTTAATTCTAGATTACGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTGTTAAAGACTAACGTATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,100,100,100,100,100,98,98,78,78,6,6 +seq161,99.2,MZ558750.1.2453_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_orientale,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,99,99,99,99,34,6,6 +seq167,93.9,JN635482.1.1281_U,18S_rRNA,nucleus,clone_hay20,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCTGTTAAAAAGCTCGTAGTTGGATTTCTGATATAATATTACTGTCCCGCTGTGGTTACACATGTGAGTGACGGATAATATTTATCTGTTTTGTGTTTTTTGTTTTGATAGTATGTAAATATTATCAAAAACATACACATCTTTTACTTTGAACAAATTAGAGTGTTTCAAGCAGGCATTTATGCCTTGAATACATGAGCATGGAATAATCATCGAGGACTTTTAGTTCTATGTTGATTGGTTCTAGAACTATAGTAATGATGGATAGGGATAGTTGGGGGTGCTAGTATTCCGAGGCCAGAGGTGAAATTCTTGGATTCTCGGAAGACTCACTTAGGCGAAAGCATTCACCAAGGATGTCTTCA,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,100,100,100,100,100,100,99,99,99,5,5 +seq173,100.0,EU709208.1.1193_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTGTTCTGGCCATCCTTCCAATCGCTGCGTGCTTTTCTTCATTGATTAGTGCGTGGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,63,63,4,4 +seq176,96.0,EF468654.1.1782_U,18S_rRNA,nucleus,strain_HC06-28,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGAAGTATATAATGAATTTCTAATTCAATTGTATATTGATTCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_sp.,100,99,99,96,96,96,96,89,70,4,4 +seq178,96.9,AB757862.1.1584_U,18S_rRNA,nucleus,,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCGTCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAATTGCGTGTGTTGGTGTTTCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,31,30,4,4 +seq181,98.7,EU709173.71.1479_U,18S_rRNA,nucleus,,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTTGCCGGCCACGGTTTCGTGTGTTAGTGCTTGATGTTCTGGCCATCCTTCCAATCGCTATCGGCTGCTCTTCATTGAGCGGTCGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,99,88,85,3,3 +seq182,98.9,JN635479.1.1258_U,18S_rRNA,nucleus,clone_hay11,Eukaryota,Alveolata,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Colpodida_X,Colpodida_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCAGCGCTGTTCTTGGCTCTCTGAGTCGGCTCAGTGTTTGGTCATCCGTATGGGAAGCTAGCTCGGCCTTCACTGGTCGGCTAGTGGATCATACACTTTACTTTGAAAAAATTAGAGTGTTTCAGGCAGGCAATTGCTTGGATACTTCAGCATGGAATAATGGAATAGGACTTTGACCTATTTGTTGGTTTCTTCGAGGTCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCATATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGATGAACTTATGCGAAAGCATTTGCCAAGGATGTTTTCATTAA,Eukaryota,TSAR,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Pseudoplatyophrya,Pseudoplatyophrya_nana,100,100,100,100,100,100,100,49,49,3,3 +seq186,100.0,EF468654.1.1782_U,18S_rRNA,nucleus,strain_HC06-28,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGATTGTTATGATGAATCTATGTATTCGATTATAATATGAATCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,100,98,97,97,97,97,97,95,46,3,3 +seq194,99.5,EU709205.65.1542_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTAAGCCGGCCACGGTTTCGTGTGTTAGTGCTTGGTGTTCTGGCCATCCTTCCAATCGCTGCTGGCCTTTCTTCATTGATAGGTTGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,73,73,2,2 diff --git a/src/dada2.r b/src/dada2.r new file mode 100644 index 0000000..e69de29 diff --git a/src/main.jl b/src/main.jl index f22ddb9..42ebf9e 100644 --- a/src/main.jl +++ b/src/main.jl @@ -1,6 +1,8 @@ include("run_cutadapt.jl") +include("merge_and_filter_taxa.jl") -using .Cutadapt +using CSV +using .Cutadapt, .TaxonomyTableTools ## Instantiate primers_path = "./inputs/primers.yml" @@ -11,4 +13,6 @@ optional_args = "-m 200 --discard-untrimmed" ## Main -cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) \ No newline at end of file +#cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) + +CSV.write("protist_filtered.csv", filter_table(merge_taxonomy_counts(),"./inputs/protist_filter.yml")) \ No newline at end of file diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl new file mode 100644 index 0000000..b968978 --- /dev/null +++ b/src/merge_and_filter_taxa.jl @@ -0,0 +1,237 @@ +module TaxonomyTableTools + +using CSV, DataFrames, Logging, YAML + +export merge_taxonomy_counts, filter_table + + # Import vsearch taxonomy + function import_vsearch(file::AbstractString) + rows = Vector{Vector{String}}() + open(file, "r") do io + for line in eachline(io) + clean = replace(chomp(line), "\r" => "") + isempty(clean) && continue + push!(rows, split(clean, '\t')) + end + end + return rows + end + + function vsearch_to_df(imported_list::Vector{Vector{String}}) + qseqid = String[] + pident = Float64[] + sseqtax = String[] + for rec in imported_list + length(rec) < 3 && continue + push!(qseqid, rec[1]) + push!(sseqtax, rec[2]) + push!(pident, parse(Float64, rec[3])) + end + return DataFrame(SeqName=qseqid, Pident=pident, sseqtax=sseqtax) + end + + function build_taxonomy_table_rows(df::DataFrame) + header = ["SeqName","Pident","Accession","rRNA","Organellum","specimen", + "Domain","Supergroup","Division","Subdivision","Class","Order", + "Family","Genus","Species"] + out_rows = Vector{Vector{String}}(undef, nrow(df)) + + for (i, r) in enumerate(eachrow(df)) + parts = split(r.sseqtax, '|') + # Always create full row with proper positioning + data = Vector{String}(undef, length(header)) + data[1] = string(r.SeqName) + data[2] = string(r.Pident) + + # Fill taxonomy fields (positions 3 to end) + for j in 1:min(length(parts), length(header)-2) + data[j+2] = parts[j] + end + + # Fill remaining fields with empty strings + for j in (length(parts)+3):length(header) + data[j] = "" + end + + out_rows[i] = data + end + return header, out_rows + end + + # Sorting helper + function seqnum(x) + if ismissing(x) + return typemax(Int) + end + m = match(r"seq(\d+)", String(x)) + return m === nothing ? typemax(Int) : parse(Int, m.captures[1]) + end + + """ + merge_taxonomy_counts(taxonomy_vsearch_path, counts_csv_path) + + Reads taxonomy TSV from vsearch and counts CSV from DADA2 and returns a merged DataFrame. + """ + function merge_taxonomy_counts( + taxonomy_vsearch_path::AbstractString = "./vsearch/taxonomy.tsv", # Fixed default paths + counts_csv_path::AbstractString = "./DADA2/tax_counts_fasta.csv") + + @info("Merging taxonomy counts.") + + # build taxonomy dataframe - FIXED VERSION + imported = import_vsearch(taxonomy_vsearch_path) + df_tax_raw = vsearch_to_df(imported) + header, rows = build_taxonomy_table_rows(df_tax_raw) + + # Create DataFrame properly by specifying all columns first + df_taxonomy = DataFrame([Symbol(h) => String[] for h in header]) + + # Append each row properly + for r in rows + push!(df_taxonomy, r) + end + + # read counts CSV + df_counts = CSV.read(counts_csv_path, DataFrame) + + # detect columns: seq id and optional sequence column + seq_id_col = nothing + sequence_col = nothing + + for col in names(df_counts) + sample = collect(skipmissing(df_counts[!, col])) + isempty(sample) && continue + first_val = string(first(sample)) + + if occursin(r"^seq\d+$", first_val) + seq_id_col = col + end + + if length(first_val) > 50 && occursin(r"^[ACGTN]+$", first_val) + sequence_col = col + end + end + + if isnothing(seq_id_col) + error("Cannot find a column with seq IDs (seq1, seq2, ...) in counts file!") + end + + df_counts_prepared = copy(df_counts) + + # If sequence column exists and isn't named "sequence", rename or drop accordingly + if !isnothing(sequence_col) && sequence_col != "sequence" + if "sequence" in names(df_counts_prepared) + select!(df_counts_prepared, Not(sequence_col)) + else + rename!(df_counts_prepared, sequence_col => "sequence") + end + end + + # Ensure SeqName column exists and refers to seq id column + if seq_id_col != "SeqName" + if "SeqName" in names(df_counts_prepared) + if isnothing(sequence_col) + rename!(df_counts_prepared, "SeqName" => "sequence") + else + select!(df_counts_prepared, Not("SeqName")) + end + end + rename!(df_counts_prepared, seq_id_col => "SeqName") + end + + # Left join instead of outerjoin to keep all taxonomy rows + merged_df = leftjoin(df_taxonomy, df_counts_prepared, on="SeqName") + sort!(merged_df, "SeqName", by=seqnum) + + return merged_df + end + + """ + filter_table(merged_df; filters_yaml_path, remove_empty_domain_override) + Applies protist filtering to a merged DataFrame produced by merge_taxonomy_counts. + + Applies protist filtering to a merged DataFrame using rules defined in a YAML file. + + ## Arguments: + - `merged_df`: Input DataFrame with taxonomy columns. + - `filters_yaml_path` (default: "./inputs/protist_filter.yml"): Path to YAML file defining filters and mappings. + - `remove_empty_domain_override` (default: nothing): Overrides YAML setting if provided. + + ## Expected YAML structure: + ``` YAML + mappings: { DivisionValue: SupergroupValue, ... } + filters: + - column: Domain + pattern: Bacteria + remove_empty_domain: true + ``` + """ + function filter_table( + merged_df::DataFrame, + filters_yaml_path::String; + remove_empty_domain_override::Union{Bool,Nothing} = nothing + ) + # Load YAML config + config = YAML.load_file(filters_yaml_path) + + df = deepcopy(merged_df) + + @info "Filtering table using configuration from $filters_yaml_path" + + # === Load mappings === + mapping_config = get(config, "mappings", Dict()) + if !isempty(mapping_config) + @assert mapping_config isa AbstractDict "mappings must be a dictionary in YAML" + end + + mapping = Dict(string(k) => string(v) for (k,v) in mapping_config) + + # Apply Supergroup mapping if both columns exist + if "Division" in names(df) && "Supergroup" in names(df) && !isempty(mapping) + df.Supergroup = [ + ismissing(d) ? s : (haskey(mapping, string(d)) ? mapping[string(d)] : s) + for (d, s) in zip(df.Division, df.Supergroup) + ] + end + + # === Load remove_empty_domain flag === + remove_empty = get(config, "remove_empty_domain", true) + if remove_empty_domain_override !== nothing + remove_empty = remove_empty_domain_override + end + + # Optionally remove blank/unassigned Domain entries + if remove_empty && ("Domain" in names(df)) + df = filter(row -> + !ismissing(row.Domain) && + !isempty(strip(string(row.Domain))) && + lowercase(strip(string(row.Domain))) != "blank", + df) + end + + # === Load and apply filters === + raw_filters = get(config, "filters", []) + @assert raw_filters isa Vector "filters must be a list in YAML" + + for item in raw_filters + @assert item isa Dict "Each filter must be a key-value map" + colname = get(item, "column", nothing) + pattern = get(item, "pattern", nothing) + @assert typeof(colname) <: AbstractString "filter column must be a string" + @assert typeof(pattern) <: AbstractString "filter pattern must be a string" + + if !(colname in names(df)) + @warn "Skipping filter: column '$colname' not found in data." + continue + end + + # Exclude rows where column contains the pattern (case-sensitive substring match) + df = filter(row -> ismissing(row[colname]) || !occursin(pattern, string(row[colname])), df) + end + + @info "Filtering complete. $(nrow(df))/$(nrow(merged_df)) rows retained." + + return df + end + +end \ No newline at end of file diff --git a/vsearch/taxonomy.tsv b/vsearch/taxonomy.tsv new file mode 100644 index 0000000..99ab532 --- /dev/null +++ b/vsearch/taxonomy.tsv @@ -0,0 +1,159 @@ +seq2 AY321149.1.1455_U|18S_rRNA|nucleus|strain_T|Eukaryota|Excavata|Metamonada|Parabasalia|Trichomonadea|Honigbergiellida|Hexamastigidae|Hexamastix|Hexamastix_kirbyi 99.2 243 2 0 1 243 1 1455 -1 0 +seq17 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 92.0 274 16 3 1 269 1 1489 -1 0 +seq22 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 95.3 275 11 2 1 275 1 1489 -1 0 +seq21 AF129862.1.1457_U|16S_rRNA||clone_CA26|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_NK4A214_group|Ruminococcaceae_NK4A214_group_sp. 90.1 374 37 0 1 374 1 1454 -1 0 +seq11 KC790327.1.507_U|18S_rRNA|nucleus|strain_318|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomyces|Saccharomyces_cerevisiae 100.0 379 0 0 1 379 1 507 -1 0 +seq19 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 89.8 374 37 1 1 374 1 1461 -1 0 +seq4 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 1143 -1 0 +seq15 AB453313.1.1235_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Chromadorea|Chromadorea_X|Strongyloides|Strongyloides_myopotami 100.0 350 0 0 1 350 1 1235 -1 0 +seq8 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.2 380 3 0 1 380 1 1745 -1 0 +seq12 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 96.6 386 12 1 1 386 1 1829 -1 0 +seq13 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 96.6 386 12 1 1 386 1 1829 -1 0 +seq6 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 100.0 380 0 0 1 380 1 1745 -1 0 +seq7 AY212772.1.1503_U|16S_rRNA||clone_wet70|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_UCG005|Ruminococcaceae_UCG005_sp. 94.7 374 20 0 1 374 1 1434 -1 0 +seq14 AJ236046.1.1766_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Plantago|Plantago_lanceolata 100.0 380 0 0 1 380 1 1745 -1 0 +seq25 EU091849.1.1430_U|18S_rRNA|nucleus|clone_P2-3m2|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Watanabea-Clade_XX|Watanabea-Clade_XX_sp. 100.0 379 0 0 1 379 1 1430 -1 0 +seq3 GQ352558.1.1816_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_vulpis 96.1 386 14 1 1 386 1 1816 -1 0 +seq1 RDQT01000029.10.1006_U|16S_rRNA||strain_LYJ002|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Bacilli|Bacillales|Staphylococcaceae|Staphylococcus|Staphylococcus_aureus 100.0 387 0 0 1 387 1 997 -1 0 +seq5 AF129862.1.1457_U|16S_rRNA||clone_CA26|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_NK4A214_group|Ruminococcaceae_NK4A214_group_sp. 90.4 374 36 0 1 374 1 1454 -1 0 +seq9 KC876030.20981.22849_U|18S_rRNA|nucleus|clone_LA13_165F6|Eukaryota|Obazoa|Opisthokonta|Metazoa|Craniata|Craniata_X|Mammalia|Homo|Homo_sapiens 100.0 387 0 0 1 387 1 1869 -1 0 +seq20 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 88.5 374 42 1 1 374 1 1461 -1 0 +seq38 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.9 275 21 1 1 275 1 1483 -1 0 +seq18 AY548930.1.1487_U|16S_rRNA||clone_1-1|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Sphingobacteriales|Lentimicrobiaceae|Lentimicrobiaceae_X|Lentimicrobiaceae_X_sp. 84.4 371 58 0 1 371 1 1444 -1 0 +seq43 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.8 273 25 0 1 273 1 1489 -1 0 +seq16 U02969.1.1039_U|18S_rRNA|nucleus|clone_YEpT7R1.0Xho|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomyces|Saccharomyces_cerevisiae 100.0 379 0 0 1 379 1 1039 -1 0 +seq23 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 95.6 386 16 1 1 386 1 1829 -1 0 +seq35 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.2 274 20 2 1 271 1 1489 -1 0 +seq24 AY821994.1.1627_U|18S_rRNA|nucleus|clone_CV1_B1_42|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 1627 -1 0 +seq29 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 +seq48 ABGB01001215.11.602_U|18S_rRNA|nucleus|strain_H348|Eukaryota|Obazoa|Opisthokonta|Fungi|Opisthosporidia|Microsporida|Microsporida_X|Enterocytozoon|Enterocytozoon_bieneusi 99.5 204 1 0 1 204 1 592 -1 0 +seq33 AY730662.1.1484_U|16S_rRNA||clone_|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Flavonifractor|Flavonifractor_sp. 97.1 374 11 0 1 374 1 1453 -1 0 +seq42 AY244968.1.1492_U|16S_rRNA||clone_BS14|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Rikenellaceae|Rikenellaceae_RC9_gut_group|Rikenellaceae_RC9_gut_group_sp. 92.4 357 27 0 1 357 1 1446 -1 0 +seq34 AF132250.1.1452_U|16S_rRNA||clone_adhufec236|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Lachnospiraceae|Lachnospiraceae_X|Lachnospiraceae_X_sp. 95.3 361 16 1 1 361 1 1451 -1 0 +seq39 EU091877.1.1445_U|18S_rRNA|nucleus|clone_C1-3m4|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1445 -1 0 +seq30 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 +seq32 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 90.1 374 36 1 1 374 1 1461 -1 0 +seq27 CK163334.1.1076_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Triticum|Triticum_aestivum 100.0 380 0 0 1 380 1 1075 -1 0 +seq10 MT991542.1.2393_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_sp. 99.5 379 2 0 1 379 1 2393 -1 0 +seq26 L24420.1.1764_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Spinacia|Spinacia_oleracea 96.6 380 13 0 1 380 1 1757 -1 0 +seq41 EU154978.1.1268_U|18S_rRNA|nucleus|clone_HC8_BASS|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 1268 -1 0 +seq44 GU972467.1.809_U|18S_rRNA|nucleus|clone_10_1F_18.b1.ab1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 809 -1 0 +seq52 AY244968.1.1492_U|16S_rRNA||clone_BS14|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Rikenellaceae|Rikenellaceae_RC9_gut_group|Rikenellaceae_RC9_gut_group_sp. 93.3 371 25 0 1 371 1 1446 -1 0 +seq31 MZ558750.1.2453_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_orientale 100.0 379 0 0 1 379 1 2453 -1 0 +seq36 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 +seq62 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.4 272 25 1 1 272 1 1483 -1 0 +seq51 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.6 274 22 1 1 274 1 1489 -1 0 +seq63 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.1 274 26 1 1 274 1 1489 -1 0 +seq28 DQ886365.1.1600_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Chamaebatia|Chamaebatia_foliolosa 99.7 379 1 0 1 379 1 1600 -1 0 +seq40 L49287.1.1743_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Lamium|Lamium_amplexicaule 100.0 380 0 0 1 380 1 1743 -1 0 +seq37 FJ820679.1.912_U|18S_rRNA|nucleus|clone_S191|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 912 -1 0 +seq60 NGBS01000218.13865.15598_U|18S_rRNA|nucleus|strain_LA|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Proteromonadidae|Proteromonas|Proteromonas_lacertae 88.5 348 34 4 1 344 1 1734 -1 0 +seq71 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.2 272 23 1 1 272 1 1483 -1 0 +seq54 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.5 274 25 1 1 274 1 1489 -1 0 +seq46 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 +seq50 EU091868.1.1343_U|18S_rRNA|nucleus|clone_P4-3m8|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1343 -1 0 +seq57 AY212760.1.1508_U|16S_rRNA||clone_wet57|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Lachnospiraceae|Lachnospiraceae_NK4A136_group|Lachnospiraceae_NK4A136_group_sp. 91.4 360 31 0 1 360 1 1439 -1 0 +seq66 DQ459618.1.841_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_carnescens 100.0 380 0 0 1 380 1 841 -1 0 +seq65 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.5 380 2 0 1 380 1 1143 -1 0 +seq68 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 +seq78 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 +seq47 EU091849.1.1430_U|18S_rRNA|nucleus|clone_P2-3m2|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Watanabea-Clade_XX|Watanabea-Clade_XX_sp. 99.5 379 2 0 1 379 1 1430 -1 0 +seq74 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.2 380 3 0 1 380 1 1745 -1 0 +seq72 FJ490216.1.935_U|18S_rRNA|nucleus|clone_C_47|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Cystobasidiomycetes_X|Cystobasidiomycetes_X_sp. 98.7 383 5 0 1 383 1 935 -1 0 +seq87 AF432140.1.1538_U|16S_rRNA||clone_FM046|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Mollicutes_RF39|Mollicutes_RF39_X|Mollicutes_RF39_XX|Mollicutes_RF39_XX_sp. 83.0 218 37 0 1 218 1 1498 -1 0 +seq80 AB032627.1.1781_U|18S_rRNA|nucleus|strain_JCM_8974|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_dimennae 99.2 380 3 0 1 380 1 1781 -1 0 +seq73 MF599080.1.1241_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Chromadorea|Chromadorea_X|Miculenchus|Miculenchus_muscus 81.8 385 59 2 1 377 1 1241 -1 0 +seq64 AANN01653944.1.1862_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Craniata|Craniata_X|Mammalia|Erinaceus|Erinaceus_europaeus 96.1 387 15 0 1 387 1 1862 -1 0 +seq82 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.2 380 3 0 1 380 1 1143 -1 0 +seq59 KF648870.1.1484_U|18S_rRNA|nucleus|strain_MTZ1|Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Isospora|Isospora_sp. 97.4 383 10 0 1 383 1 1484 -1 0 +seq85 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 92.3 274 15 3 1 269 1 1489 -1 0 +seq93 AY321149.1.1455_U|18S_rRNA|nucleus|strain_T|Eukaryota|Excavata|Metamonada|Parabasalia|Trichomonadea|Honigbergiellida|Hexamastigidae|Hexamastix|Hexamastix_kirbyi 97.9 243 5 0 1 243 1 1455 -1 0 +seq49 DQ415278.1.1656_U|18S_rRNA|nucleus|strain_MCA2391|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Pucciniomycetes|Puccinia|Puccinia_hordei 100.0 381 0 0 1 381 1 1656 -1 0 +seq69 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 +seq83 FJ592431.1.1083_U|18S_rRNA|nucleus|clone_E11_SE4A|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1083 -1 0 +seq75 AJ236055.1.1761_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Veronica|Veronica_anagallis-aquatica 99.5 380 2 0 1 380 1 1737 -1 0 +seq53 FJ355409.1.1048_U|18S_rRNA|nucleus|clone_051102_S1_W_T_SDP18_098|Eukaryota|TSAR|Stramenopiles|Gyrista|Chrysophyceae|Ochromonadales|Ochromonadales_clade-XIII|Ochromonadales_clade-XIII_X|Ochromonadales_clade-XIII_X_sp. 94.2 381 19 2 1 378 1 1048 -1 0 +seq56 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 +seq76 L24420.1.1764_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Spinacia|Spinacia_oleracea 96.1 380 15 0 1 380 1 1757 -1 0 +seq77 JF703099.1.1644_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 99.5 381 1 1 1 380 1 1644 -1 0 +seq90 DQ832220.1.1776_U|18S_rRNA|nucleus|strain_JAG_55|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Sphacelotheca|Sphacelotheca_koordersiana 99.2 382 3 0 1 382 1 1776 -1 0 +seq79 AB085801.1.1782_U|18S_rRNA|nucleus|strain_CBS_6578|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_sp. 100.0 380 0 0 1 380 1 1782 -1 0 +seq70 HQ427487.1.1161_UC|18S_rRNA|nucleus|clone_RU06182008DF1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 914 -1 0 +seq81 AF071069.1.1760_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Trifolium|Trifolium_repens 100.0 379 0 0 1 379 1 1760 -1 0 +seq91 JF703099.1.1644_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 99.7 381 0 1 1 380 1 1644 -1 0 +seq94 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 +seq88 Y15810.1.1761_U|18S_rRNA|nucleus|strain_CBS_1787|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Kazachstania|Kazachstania_telluris 100.0 381 0 0 1 381 1 1761 -1 0 +seq98 AB695526.1.1759_U|18S_rRNA|nucleus|clone_MPE2-32|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Allapsidae|Allapsidae_X|Allapsidae_X_sp. 100.0 388 0 0 1 388 1 1759 -1 0 +seq96 FJ592322.1.1085_U|18S_rRNA|nucleus|clone_B06_SE1A|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Prasiolales_XX|Prasiolales_XX_sp. 100.0 380 0 0 1 380 1 1085 -1 0 +seq84 D31648.1.1781_U|18S_rRNA|nucleus|strain_JCM_2961|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Dioszegia|Dioszegia_crocea 100.0 380 0 0 1 380 1 1781 -1 0 +seq92 MZ067863.1.738_U|18S_rRNA|nucleus|strain_ACKU_309-4|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Chlorophyceae|Chlamydomonadales|Chlamydomonadales_X|Chlorosarcinopsis|Chlorosarcinopsis_bastropiensis 100.0 379 0 0 1 379 1 738 -1 0 +seq117 AF432140.1.1538_U|16S_rRNA||clone_FM046|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Mollicutes_RF39|Mollicutes_RF39_X|Mollicutes_RF39_XX|Mollicutes_RF39_XX_sp. 78.5 214 46 0 1 214 1 1498 -1 0 +seq102 U40810.1.1747_U|18S_rRNA|nucleus|strain_CBS_7331|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Agaricostilbomycetes|Bensingtonia|Bensingtonia_yuccicola 99.2 388 3 0 1 388 1 1747 -1 0 +seq106 FJ592430.1.1088_U|18S_rRNA|nucleus|clone_E08_SE4A|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 98.9 380 4 0 1 380 1 1088 -1 0 +seq109 AY642700.1.1764_U|18S_rRNA|nucleus|clone_P34.42|Eukaryota|Obazoa|Opisthokonta|Fungi|Rozellomycota|Rozellomycota_X|Rozellomycota_XX|Rozellomycota_XXX|Rozellomycota_XXX_sp. 89.2 371 35 3 1 370 1 1764 -1 0 +seq105 EU091862.1.1441_U|18S_rRNA|nucleus|clone_P4-3m2|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 99.5 380 2 0 1 380 1 1441 -1 0 +seq95 DQ851582.1.1716_U|18S_rRNA|nucleus|strain_J130|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Agaricomycetes|Psathyrella|Psathyrella_gracilis 99.7 381 1 0 1 381 1 1716 -1 0 +seq101 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 +seq112 AY730662.1.1484_U|16S_rRNA||clone_|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Flavonifractor|Flavonifractor_sp. 97.1 374 11 0 1 374 1 1453 -1 0 +seq113 EU087208.1.865_U|18S_rRNA|nucleus|clone_1_39|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Eugregarinorida|Gregarinidae|Amoebogregarina|Amoebogregarina_sp. 84.5 367 52 3 1 367 1 865 -1 0 +seq108 DQ388549.1.1646_U|18S_rRNA|nucleus||Eukaryota|TSAR|Stramenopiles|Gyrista|Chrysophyceae|Ochromonadales|Ochromonadaceae|Spumella|Spumella_vulgaris 100.0 383 0 0 1 383 1 1646 -1 0 +seq115 KM020169.1.1775_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Chlorophyceae|Chlamydomonadales|Coccomyxaceae|Coccomyxa|Coccomyxa_sp. 100.0 380 0 0 1 380 1 1775 -1 0 +seq107 AY607851.1.1593_U|18S_rRNA|nucleus|specimen_Holz_&_Franzaring_CH_00-80_(NY)|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Dendrohypopterygium|Dendrohypopterygium_arbuscula 100.0 383 0 0 1 383 1 1593 -1 0 +seq104 EF200530.1.1708_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Prasiola|Prasiola_crispa 98.9 380 3 1 1 380 1 1708 -1 0 +seq103 EF024471.1.1761_U|18S_rRNA|nucleus|clone_Elev_18S_1147|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 92.3 376 25 1 1 376 1 1761 -1 0 +seq125 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.5 274 25 1 1 274 1 1489 -1 0 +seq100 EU091871.1.1443_U|18S_rRNA|nucleus|clone_P6-3m2|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1443 -1 0 +seq119 DQ886380.1.1647_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Rubus|Rubus_idaeus 100.0 380 0 0 1 380 1 1647 -1 0 +seq116 X74000.1.2286_R|18S_rRNA|nucleus|strain_Andreyeva_748-I|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Chlorellales|Chlorellales_X|Chlorella|Chlorella_mirabilis 100.0 380 0 0 1 380 1 1798 -1 0 +seq118 EU709161.6.1494_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 100.0 389 0 0 1 389 1 1489 -1 0 +seq124 AF206944.1.1735_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Juncus|Juncus_effusus 99.5 380 2 0 1 380 1 1735 -1 0 +seq99 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 84.9 344 50 2 1 344 1 1692 -1 0 +seq126 GU901855.1.1740_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Annelida|Annelida_X|Annelida_XX|Achaeta|Achaeta_bifollicula 99.2 380 3 0 1 380 1 1740 -1 0 +seq137 AY189313.1.1515_U|16S_rRNA||strain_LB-12|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Entomoplasmatales|Spiroplasmataceae|Spiroplasma|Spiroplasma_sp. 82.7 214 37 0 1 214 1 1441 -1 0 +seq135 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.3 275 20 2 1 273 1 1489 -1 0 +seq127 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 90.8 371 29 3 1 367 1 1760 -1 0 +seq120 EF023954.1.1792_U|18S_rRNA|nucleus|clone_Amb_18S_1419|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1792 -1 0 +seq132 JN573894.1.1681_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Diplosphaera|Diplosphaera_sp. 100.0 380 0 0 1 380 1 1681 -1 0 +seq129 JQ941712.1.1035_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Pezizomycotina|Laboulbeniomycetes|Hesperomyces|Hesperomyces_virescens 100.0 364 0 0 1 364 1 1035 -1 0 +seq114 EU090188.1.1662_U|18S_rRNA|nucleus|strain_AFC_1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Rhodotorula|Rhodotorula_yarrowii 98.7 382 5 0 1 382 1 1662 -1 0 +seq144 EF586127.1.796_U|18S_rRNA|nucleus|clone_121-O29|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Eugregarinorida|Gregarinidae|Leidyana1|Leidyana1_sp. 98.0 306 6 0 1 380 1 796 -1 0 +seq134 JQ993661.1.1396_U|18S_rRNA|nucleus||Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Eimeria6|Eimeria6_sp. 99.2 383 3 0 1 383 1 1396 -1 0 +seq133 FR693368.1.1633_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Apatococcus|Apatococcus_lobatus 100.0 380 0 0 1 380 1 1633 -1 0 +seq130 AY884326.1.1078_U|18S_rRNA|nucleus|strain_CeS-2|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Cercomonadida|Cercomonadidae|Cercomonas|Cercomonas_sp. 100.0 390 0 0 1 390 1 1078 -1 0 +seq131 AJ579340.1.1732_U|18S_rRNA|nucleus|strain_SAG_30.83|Eukaryota|TSAR|Stramenopiles|Gyrista|Xanthophyceae|Xanthophyceae_X|Xanthophyceae_XX|Botrydiopsis|Botrydiopsis_callosa 100.0 396 0 0 1 396 1 1732 -1 0 +seq110 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 87.8 344 39 2 1 343 1 1692 -1 0 +seq140 AB032621.1.1783_UC|18S_rRNA|nucleus|strain_JCM_1775|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Agaricomycetes|Mrakiella|Mrakiella_aquatica 99.5 380 2 0 1 380 1 1513 -1 0 +seq147 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 98.4 372 4 1 1 372 1 1760 -1 0 +seq154 AF311292.1.1439_U|16S_rRNA||strain_WH_8018|Bacteria|CMS|Cyanobacteria|Cyanobacteria_X|Cyanophyceae|Synechococcales|Cyanobiaceae|Synechococcus_CC9902|Synechococcus_CC9902_sp. 77.2 219 49 1 1 218 1 1406 -1 0 +seq145 MG775599.1.1357_U|18S_rRNA|nucleus|strain_SF33_R_Tri|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Cercomonadida|Cercomonadidae|Neocercomonas|Neocercomonas_sp. 97.2 389 11 0 1 389 1 1357 -1 0 +seq143 EF023318.1.1626_U|18S_rRNA|nucleus|clone_Amb_18S_635|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 1626 -1 0 +seq141 AY919792.1.1726_U|18S_rRNA|nucleus|clone_LG32-03|Eukaryota|TSAR|Alveolata|Chrompodellids|Colpodellidea|Colpodellida|Colpodellida_CHR1|Colpodellida_CHR1_X|Colpodellida_CHR1_X_sp. 76.9 381 65 5 1 359 1 1726 -1 0 +seq148 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 90.6 371 30 3 1 367 1 1760 -1 0 +seq121 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 88.0 343 39 2 1 342 1 1692 -1 0 +seq146 AM114805.1.2266_U|18S_rRNA|nucleus|clone_WIM47|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 99.2 389 3 0 1 389 1 2266 -1 0 +seq142 AJ431572.1.1716_U|18S_rRNA|nucleus|strain_SAG_35.83|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Desmococcus|Desmococcus_olivaceus 99.0 381 3 1 1 381 1 1716 -1 0 +seq159 AY124366.1.737_U|18S_rRNA|nucleus|clone_CSE28|Eukaryota|TSAR|Alveolata|Ciliophora|Litostomatea|Litostomatea_X|Litostomatea_XX|Litostomatea_XXX|Litostomatea_XXX_sp. 98.7 310 4 0 1 310 1 737 -1 0 +seq157 HG796065.1.800_U|18S_rRNA|nucleus|clone_A1_18_43|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomycetales_X|Saccharomycetales_X_sp. 100.0 379 0 0 1 379 1 800 -1 0 +seq164 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 91.4 374 31 1 1 374 1 1461 -1 0 +seq171 AY189313.1.1515_U|16S_rRNA||strain_LB-12|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Entomoplasmatales|Spiroplasmataceae|Spiroplasma|Spiroplasma_sp. 82.7 214 37 0 1 214 1 1441 -1 0 +seq158 HQ865510.1.886_U|18S_rRNA|nucleus|clone_SGSF770|Eukaryota|Obazoa|Opisthokonta|Fungi|Neocallimastigomycota|Neocallimastigomycota_X|Neocallimastigomycota_XX|Neocallimastigomycota_XXX|Neocallimastigomycota_XXX_sp. 87.7 390 39 3 1 384 1 886 -1 0 +seq168 AF206852.1.1739_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 98.2 380 7 0 1 380 1 1739 -1 0 +seq167 JN635482.1.1281_U|18S_rRNA|nucleus|clone_hay20|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Glissomonadida_X|Glissomonadida_XX|Glissomonadida_XX_sp. 93.9 393 21 1 1 393 1 1281 -1 0 +seq160 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 +seq166 GU824996.1.1063_U|18S_rRNA|nucleus|clone_BCB5F13RM2C08|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomycetales_X|Saccharomycetales_X_sp. 100.0 373 0 0 1 373 1 1063 -1 0 +seq173 EU709208.1.1193_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandonidae_X|Sandonidae_X_sp. 100.0 389 0 0 1 389 1 1193 -1 0 +seq174 JQ014042.1.996_U|18S_rRNA|nucleus|strain_FSU_9823|Eukaryota|Obazoa|Opisthokonta|Fungi|Mucoromycota|Mortierellaceae|Mortierellaceae_X|Mortierella|Mortierella_hyalina 99.7 380 1 0 1 380 1 996 -1 0 +seq176 EF468654.1.1782_U|18S_rRNA|nucleus|strain_HC06-28|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Blastocystis-Group|Blastocystis|Blastocystis_hominis 96.0 374 14 1 1 373 1 1782 -1 0 +seq175 EU090188.1.1662_U|18S_rRNA|nucleus|strain_AFC_1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Rhodotorula|Rhodotorula_yarrowii 99.2 382 3 0 1 382 1 1662 -1 0 +seq177 MG076806.1.532_U|18S_rRNA|nucleus|strain_AGC1-24|Eukaryota|Obazoa|Opisthokonta|Fungi|Mucoromycota|Glomeromycotina|Paraglomus_family|Paraglomus|Paraglomus_sp. 99.7 379 1 0 1 379 1 532 -1 0 +seq178 AB757862.1.1584_U|18S_rRNA|nucleus||Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Isospora|Isospora_sp. 96.9 383 12 0 1 383 1 1584 -1 0 +seq181 EU709173.71.1479_U|18S_rRNA|nucleus||Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 98.7 389 5 0 1 389 1 1409 -1 0 +seq161 MZ558750.1.2453_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_orientale 99.2 379 3 0 1 379 1 2453 -1 0 +seq184 AB055193.1.1761_U|18S_rRNA|nucleus|strain_CBS8732|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Occultifur|Occultifur_externus 99.7 383 1 0 1 383 1 1761 -1 0 +seq182 JN635479.1.1258_U|18S_rRNA|nucleus|clone_hay11|Eukaryota|TSAR|Alveolata|Ciliophora|Colpodea|Colpodea_X|Colpodida|Colpodida_X|Colpodida_X_sp. 98.9 378 4 0 1 378 1 1258 -1 0 +seq186 EF468654.1.1782_U|18S_rRNA|nucleus|strain_HC06-28|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Blastocystis-Group|Blastocystis|Blastocystis_hominis 100.0 374 0 0 1 374 1 1782 -1 0 +seq185 MG589001.1.1014_U|18S_rRNA|nucleus|strain_HU9268|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Sporobolomyces|Sporobolomyces_sp. 100.0 382 0 0 1 382 1 1014 -1 0 +seq183 DQ536473.1.3565_G|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Chytridiomycota|Chytridiales|Chytridiaceae|Phlyctochytrium|Phlyctochytrium_planicorne 84.3 381 55 4 1 377 1 3565 -1 0 +seq194 EU709205.65.1542_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandonidae_X|Sandonidae_X_sp. 99.5 389 2 0 1 389 1 1478 -1 0 +seq196 AY821994.1.1627_U|18S_rRNA|nucleus|clone_CV1_B1_42|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 95.8 380 16 0 1 380 1 1627 -1 0 +seq195 HQ888716.1.1022_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Blastocladiomycota|Blastocladiomycotina|Blastocladiomycetes|Physoderma|Physoderma_lycopi 99.0 388 3 1 1 388 1 1022 -1 0 From 64e7147faf247589cc91e87b7aad8cd9836ce174 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sun, 1 Feb 2026 23:37:59 +0100 Subject: [PATCH 005/175] Edit .gitignore --- .gitignore | 6 +++++- DADA2/.~lock.tax_counts_fasta.csv# | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 DADA2/.~lock.tax_counts_fasta.csv# diff --git a/.gitignore b/.gitignore index 5b4522f..36bce83 100644 --- a/.gitignore +++ b/.gitignore @@ -255,4 +255,8 @@ rsconnect/ # Things with ambiguous intellectual property internal inputs/fastq/ -cutadapt/output_* \ No newline at end of file +cutadapt/output_* + +### Lock files ### +.~lock. +.csv# \ No newline at end of file diff --git a/DADA2/.~lock.tax_counts_fasta.csv# b/DADA2/.~lock.tax_counts_fasta.csv# deleted file mode 100644 index 51e5ef1..0000000 --- a/DADA2/.~lock.tax_counts_fasta.csv# +++ /dev/null @@ -1 +0,0 @@ -,joshua,parrot,01.02.2026 22:46,file:///home/joshua/.config/libreoffice/4; \ No newline at end of file From 252299ee998290ffc87b4117249121efa7e7cdde Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Mon, 9 Feb 2026 14:55:27 +0100 Subject: [PATCH 006/175] Merge tables logic. --- .gitignore | 9 +- .vscode/settings.json | 2 + DADA2/tax_counts_fasta.csv | 199 ----------------------------- cutadapt_primer_trimming_stats.txt | 129 ------------------- pipelinesteps.txt | 4 +- protist_filtered.csv | 59 --------- src/dada2.r | 0 src/main.jl | 7 +- src/merge_and_filter_taxa.jl | 2 +- vsearch/taxonomy.tsv | 159 ----------------------- 10 files changed, 18 insertions(+), 552 deletions(-) create mode 100644 .vscode/settings.json delete mode 100644 DADA2/tax_counts_fasta.csv delete mode 100644 cutadapt_primer_trimming_stats.txt delete mode 100644 protist_filtered.csv delete mode 100644 src/dada2.r delete mode 100644 vsearch/taxonomy.tsv diff --git a/.gitignore b/.gitignore index 36bce83..2e3069e 100644 --- a/.gitignore +++ b/.gitignore @@ -258,5 +258,10 @@ inputs/fastq/ cutadapt/output_* ### Lock files ### -.~lock. -.csv# \ No newline at end of file +.~lock.* +.csv# + +### Bits I don't want to share yet ### +hide.* +DADA2 +vsearch \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/DADA2/tax_counts_fasta.csv b/DADA2/tax_counts_fasta.csv deleted file mode 100644 index 0a151d2..0000000 --- a/DADA2/tax_counts_fasta.csv +++ /dev/null @@ -1,199 +0,0 @@ -sequence,tax.Domain,tax.Supergroup,tax.Division,tax.Subdivision,tax.Class,tax.Order,tax.Family,tax.Genus,tax.Species,boot.Domain,boot.Supergroup,boot.Division,boot.Subdivision,boot.Class,boot.Order,boot.Family,boot.Genus,boot.Species,SeqName,SeqName.y,JIN-Nu-mul,JIN-Nu-mul2 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCCGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,100,100,100,seq1,seq1,45046,45046 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,76,seq2,seq2,11240,11240 -AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGAACGTGCACGGACGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,77,seq3,seq3,10288,10288 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,99,99,seq4,seq4,8606,8606 -TAGGTAGCGAGCGTTATCCGGATTTACTGGGTGTAAAGGGCGCGTAGGCGGGCTGACAAGTCAGGAGTGAAAACTATGGGCTTAACCCATAGCCTGCTTTTGAAACTGTGAGTCTTGAGTATCGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAAGAACACCAGTGGCGAAGGCGGATTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGGAGTTAACACAATAAGTATTCCACCTGGGGAGTACGGCCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_NK4A214_group,Ruminococcaceae_NK4A214_group_sp.,100,100,100,100,100,100,100,44,44,seq5,seq5,8346,8346 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,62,62,seq6,seq6,6751,6751 -TAGGTTGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGGCGGAGAAGCAAGTTGGGAGTGAAATCCATGGGCTCAACCCATGAACTGCTCTCAAAACTGTTTCCCTTGAGTATCGGAGAGGCAAGCGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGTTAACACAATAAGTATTCCACCTGGGGAGTACGACCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_UCG005,Ruminococcaceae_UCG005_sp.,100,100,100,100,100,100,100,95,95,seq7,seq7,1400,1400 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTTGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGACATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGTGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,65,65,seq8,seq8,1073,1073 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCTGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,37,37,37,seq9,seq9,1000,1000 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTTTGGTGTGCACTGGCATGGGCTCGCCTCGCTGGCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGCCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTACGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,100,100,100,100,67,seq10,seq10,920,920 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCCGGTTGGCCGGTCCGATTTTTTCGTGTACTGGATTTCCAACGGGGCCTTTCCTTCTGGCTAACCTTGAGTCCTTGTGGCTCTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Candida,Candida_wounanorum,100,100,100,100,100,100,100,100,100,seq11,seq11,909,909 -AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGATCGACGACGGTCGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,75,seq12,seq12,614,614 -AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGGTCGACGACGGTCGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTGTTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,70,seq13,seq13,499,499 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGTTGGGCCGATCGGTCCGCCTTCTGGTGTGTACCGGTCGTCTCGTCCCTTCTGCTGGCGATGCGCTCCTGGCCTTAACTGGCCGGGTCGTTCCTCCAGCACTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Plantago,Plantago_lanceolata,100,100,100,100,100,100,100,99,99,seq14,seq14,421,421 -AGCTTTCCAAGTGCATAAAATGATTGTTGTGGTTAAAAAGCTCGTAGTTGGATTATAAAGATTGTATAATGAGCATCTTGGATGTTTTTCATTATCATCTTACTTTTTTATTATATTAGTAATAATATAATAACTGTTACTTTGAATAAATCAGAGGGTTTAAACCAGGCATTATATGCTTGTATGGTCTAGCATGGAATAACACTATAGGAAAAGTTAGTGTGGTTTCACTAATCTTTTTCATGATTAATAGGAACAAACGGGGGCATTCGTATCGCTACGTTAGAGGTGAAATTCTTGGACCGTAGCGAGACGTCCTACTGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Chromadorea,Chromadorea_X,Strongyloides,Strongyloides_myopotami,100,100,100,100,100,100,100,100,73,seq15,seq15,369,369 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCCGGTTGGCCGGTCCGATTTTTTCGTGTACTGGATTTCCAACGGGGCCTTTCCTTCTGGCTAACCTTGAGTCCTTGTGGCTCTTGGCGAACCGGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Saccharomyces,Saccharomyces_cerevisiae,100,100,100,100,100,100,100,63,63,seq16,seq16,361,361 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCATTGTGCTTATGTGTCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,86,86,83,seq17,seq17,339,339 -GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGGCGTCTAAGTCAGCGGTGAAAGGTTCCGGCTCAACCGGGACAGTGCCGATGATACTGGCTGCCTTGAATGCGGTCAAGGCCGGCGGAATGTGGCGTGTAGCGGTGAAATGCATAGATATGCCACAGAACACCGATAGCGAAGGCAGCTGGCCGGGCCTGCATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGGACACTCGTCGTCGGCGACAGACAGCCGGCGGCCAAGCGAAAGTGATAAGTGTCCCACCTGGGGAGTACGGTCGCAAGGCTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Sphingobacteriales,Sphingobacteriaceae,Parapedobacter,Parapedobacter_sp.,99,99,99,99,99,46,31,25,25,seq18,seq18,336,336 -GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCGGGCCGTCAAGTCAGCGGTAAAATTGCGGGGCTCAACCCCGTCGAGCCGTTGAAACTGGCAGCCTTGAGTGGGCGAGAAGTATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACTCCGATTGCGAAGGCAGCATGCCGGCGCCCAACTGACGCTGAAGCACGAAAGCGTGGGTATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGAGCGCTAATTGTTTGCGGAGAATGATCCGTGAGTGATACAGCGAAAGCGTTAAGCGCTCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,71,71,71,seq19,seq19,316,316 -GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGAGAGTCAAGCCGGCGGTCAAATTGCGGGGCCCAACCCCGTACCGCCGTCGGAACTGGCTCCCTTGAGTGGACGAGAAGTAAGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCTTACCGGTGTCCAACTGACGCTCAGGCACGAAAGCGTGGGGATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGGATACTAGCTGTCCGGGTCGAGTGAGACCTGGGGGGCACAGCGAAAGCGTTAAGTATCCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,97,43,43,43,seq20,seq20,314,314 -TAGGTAGCGAGCGTTATCCGGATTTACTGGGTGTAAAGGGCGCGTAGGCGGGCTGACAAGTCAGGAGTGAAAACTATGGGCTTAACCCATAGCCTGCTTTTGAAACTGTGAGTCTTGAGTATCGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAAGAACACCAGTGGCGAAGGCGGATTGCTGGACGACAACTGACGCTGAGGCGCGAAAGCGTGGGGAACAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGAATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGGAGTTAACACAATAAGTATTCCACCTGGGGAGTACGGCCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Ruminococcaceae_NK4A214_group,Ruminococcaceae_NK4A214_group_sp.,100,100,100,100,100,100,100,46,46,seq21,seq21,287,287 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTCGCAAGAGGCGAGAGTGCCATTAGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTTTACCTACAGGTAAGATCAATGAGAGCCACCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,98,98,98,98,90,seq22,seq22,275,275 -AGCTCCAATAGCGTATATTAAAGTTGCTGCGGTTAAACCGCTCGTAGTTGGATTACGGTGAACGTGCACGGACGTCCTAAGCAGGAGTCGTTCCGTCGCTCGTCACCCGTTGTGATCAAGATTGTCCTTGATGCTCTTTAGTGAGTGTCCTGGGCGACTTGAAAGTTTACTTTGAGAAAATGGAAGCGCTCAAGGCAAGCCGTAGTGCTTGAACAGTGGTGCATGGAATAATGAAAGATGGCCTCGGTGCTATTTTATTGGTTTACGGTGACGAGGCAATGATTAAAAGAGACAGACGGGGACATTCGTATTGCTGCGTTAGAGGTGAAATTCTTGGATCGCAGCAAGACGCACAATTGCGAAAGCATTTGTCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Enoplea,Enoplea_X,Trichuris,Trichuris_vulpis,100,100,100,100,100,100,100,100,67,seq23,seq23,273,273 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCTTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGAGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCATTGCTAGAGGTGAAATTCTTAGATTTATGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,99,99,99,67,67,seq24,seq24,252,252 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,94,seq25,seq25,238,238 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGATGGGACGATCGGTCCGCCTTTAGGTGAGCACCGGTCGTCCGGTCTCTTACGCCGGCGATGCGCTCCTAGCCTTAGTTGGCCGGGTCGTGCCTCCGGCACAGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATTATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Spinacia,Spinacia_oleracea,99,99,99,99,99,99,99,7,7,seq26,seq26,236,236 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCGGCCGGCATCGCGCTCCTAGCCTTAATTGGCCGGGTCGTGTTTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Aegilops,Aegilops_tauschii,100,100,100,100,100,100,100,82,82,seq27,seq27,220,220 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGACCGGTCCGCCTATGGTGTGCACCGGTCGGCTCGTCCCTTCTACCGGCGATACGCTCCTGGTCTTAATTGGCCGGGTCGTGCCTCCGGTGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTCTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Prunus,Prunus_persica,100,100,100,100,100,100,100,73,73,seq28,seq28,216,216 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTATTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,100,100,seq29,seq29,209,209 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTATTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,68,68,seq30,seq30,188,188 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,61,seq31,seq31,188,188 -GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGCAGGCGGAGAGTCAAGCCGGCGGTCAAATCGCGGGGCCCAACCCCGTGCCGCCGTCGGAACTGGCTCCCTTGAGTGGGCGAGAAGTATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCTTACCGGCGCCCAACTGACGCTCAGGCACGAAAGCGTGGGGATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGGATGCTAGCTGTCCGGGGGGAATGGCCCCTGGGCGGCACAGCGAAAGCGTTAAGCATCCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,83,83,83,seq32,seq32,154,154 -TAGGTGGCAAGCGTTATCCGGATTTATTGGGTGTAAAGGGCGTGTAGGCGGGACTGCAAGTCAGATGTGAAAACTATGGGCTCAACCCATAGCCTGCATTTGAAACTGTAGTTCTTGAGTGCTGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAGGAACACCAGTGGCGAAGGCGGATTGCTGGACAGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGGATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGCAAACGCAATAAGTATCCCACCTGGGGAGTACGATCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Flavonifractor,Flavonifractor_sp.,100,100,100,100,100,100,100,98,98,seq33,seq33,124,124 -TATGGAGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGCATGGCAAGTCTGATGTGAAAGGCCCGGGCCCAACCCGGGAACTGCATTGGAAACTGTCAGGCTGGAGTGCAGGAGAGGTAAGTGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTACTGGACTGTAACTGACGTTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTCGGGGGTTTAAGGACCTCCGGTGCCGCAGCAAACGCAATAAGTATTCCACCTGGGGAGTACGTTC,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Lachnospiraceae,Lachnospiraceae_X,Lachnospiraceae_X_sp.,100,100,100,100,100,100,100,96,96,seq34,seq34,121,121 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTGGCGTATGTGCCTTAGGGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,80,80,80,80,79,seq35,seq35,120,120 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAACTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,94,94,seq36,seq36,117,117 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCATTGCTAGAGGTGAAATTCTTAGATTTATGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,92,91,seq37,seq37,117,117 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTCGGCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTCTACTTCCAAGTAAAATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAAGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,81,81,78,78,75,seq38,seq38,107,107 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTTGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCAACGGGGACTTACCTCCTGGTGAACTGCAATGTCCTTTACTGGGTGTTGTAGCGAACTAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Leucosporidium,Leucosporidium_fragarium,100,100,100,100,100,100,100,98,49,seq39,seq39,103,103 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGCCGGCCGGTCCGCCTTTCGGTGTGCACCGGTCGTCTCGTCCCTTCTGCCGGCGATGCGCTCCTGGTCTTAACTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Pogostemon,Pogostemon_cablin,100,100,100,100,100,100,100,82,82,seq40,seq40,91,91 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTACGGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq41,seq41,91,91 -GAGGATGCAAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCTGTTCAGCAAGTCAGAGGTGAAATACTTGAGCTTAACTCGGGAACTGCCTTTGATACTGTTGAGCTGGAATACGGATGCCGTGGGAGGAATGAGTAGTGTAGCGGTGAAATGCATAGATATTACTCAGAACACCGATTGCGAAGGCATCTCACGAATCCGTCATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATAACTAACCGTCGGCGATAGACAGTCGGTGGCCAAGCGAAAGCGATAAGTTATCCACCTGGGGAGTACGTTC,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Rikenellaceae,Rikenellaceae_RC9_gut_group,Rikenellaceae_RC9_gut_group_sp.,100,100,100,100,100,100,100,99,99,seq42,seq42,91,91 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCGGAATGACTCAGCGCAGTATGATATCTTTACCTCGAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,63,63,62,62,61,seq43,seq43,78,78 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTTCAGGTGTGCACCGGTTTACTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Cirsium,Cirsium_pendulum,100,100,100,100,100,100,100,92,92,seq44,seq44,64,64 -GAACGGATTAGGCTATGAATTTTCCATGCGTGAAGTCGCAGACCGCATGGTCGGTTTCTGGGCGGACGCGCTTGAAGAGGAAGGCATCATCGAAACGCCTGAACAGAAGCAGATTTTCTACGATGAAGTCGTCTATGCGCTGCTGGCCCAGATGTGGGCGCCGAACTCGCCGCAGTGGTTCAACACCGGACTGAAGCGCAGCTACGGGATCGCCGGAGACAAGGACGATCTGTATTACTATGATGAAAAGACGGGGGAAGTCGTCGAATCGGAAGACCGCTACACCCGCACTCAGGCTTCCGCCTGCTTTATTCTC,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,88,24,12,12,12,12,12,12,12,seq45,seq45,62,62 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,66,66,seq46,seq46,58,58 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCCGCCGGTCCGCCTCTGGTGTGCACTGGCGTGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,94,65,seq47,seq47,58,58 -AACTCCAAGAGTGTCTATGGTGGATGCTGCAGTTAAAGGGTCCGTAGTCGTGAATGCAATTAAATGTCGTTGTTCAATAGCGATGAGTTTGCTAATGTTTGCGGAACGGATAGGGAGTGTAGTATAGACTGGCGAAGAATGAAATCTCAAGACCCAGTTTGGACTAACGGAGGCGAAGGCGACACTCTTAGACGTATCTGAGGA,Eukaryota,Obazoa,Opisthokonta,Fungi,Opisthosporidia,Microsporida,Microsporida_X,Enterocytozoon,Enterocytozoon_bieneusi,100,100,100,100,100,100,100,100,100,seq48,seq48,53,53 -AGCTCCAATAGCGTATATTAAAGTTGTTGACGTTAAAAAGCTCGTAGTCGAACTTCGGCCTCTGGCAGTTGGTCCGCCTTTTGGTGTGTACTGATTTGTTGGAGGCTTACCTCTTGGTGAACTTCAATGCACTTTACTGGGTGTTGGAGGGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCTTATGCCTGAATACATTAGCATGGAATAATAAAATAGGACGTGTGATTCTATTTTGTTGGTTTCTAGGATTACCGTAATGATGAATAGGGTCAGTTGGGGGCATTTGTATTACATCGTCAGAGGTGAAATTCTTGGATTGATGTAAGACAAACTACTGCGAAAGCATCTGCCAAGGATGACTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Pucciniomycetes,Aecidium,Aecidium_kalanchoe,100,100,100,100,100,100,100,100,100,seq49,seq49,53,53 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTTGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCAACGGGGACTTACCTCCTGGTGAACTGCGATGTCCTTTACTGGGTGTCGTAGCGAACTAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Leucosporidium,Leucosporidium_scottii,100,100,100,100,100,100,100,100,100,seq50,seq50,53,53 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCTTGTCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,83,83,82,82,79,seq51,seq51,49,49 -GAGGATGCAAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCTGTTTATCAAGTCAGGGGTGAAATACCGGGGCTCAACTCCGGAATTGCCTCTGATACTGATAGGCTTGAATACTGTTGCCGTGGGAGGAATGAGTAGTGTAGCGGTGAAATGCATAGATATTACTCAGAACACCGATTGCGAAGGCATCTCACGAAACAGGGATTGACGCTGAGGCACGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATGACTAACCGTCGGCGATATACAGTCGGTGGCCAAGCGAAAGCGATAAGTCATCCACCTGGGGAGTACGACCGCAAGGTTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Rikenellaceae,Rikenellaceae_RC9_gut_group,Rikenellaceae_RC9_gut_group_sp.,100,100,100,100,100,100,100,100,100,seq52,seq52,49,49 -AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTCGAGACGCAGCCAGGCTCAAGGGCCGATACTGCGGATTGGGACCATCCTCGAGAAGAACATATCTGTCATTGAGTTGATGGGTATGGGACTCTCGTCTTTTACTGTGAGCAAAATAGAGTGTTCAAAGCAGGCTTACGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACTTCGGTCTATTTTGTTGGTTATACTCCGAAGTAATGATTAATAGGGACAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,100,100,100,100,100,100,70,70,70,seq53,seq53,48,48 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTATGTACCTCCAAGTACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,88,88,88,seq54,seq54,47,47 -CCACATGCTCATGCGCGCCCGAAGCAAAGTTAAACAGCGAGGTCTTGCCGCAGTTCGGGTTGCCAACCAACGCCACGTTGATGGTCTTACCCTTGTTGAGCGCCATCGCGCGCCACTCCTCATCAGAGGGGAGGGTATACTCATTCGCCGTCTGTGCGCTCCCCGCCGTGTGCAACAGTTTCTCCTCCTCAAACTCAGCCGCGCTGACCACT,Eukaryota,Excavata,Discoba,Euglenozoa,Euglenida,Aphagea,Rhabdomonadales,Rhabdomonas,Rhabdomonas_intermedia,89,14,13,13,12,10,8,7,5,seq55,seq55,47,47 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,99,99,seq56,seq56,44,44 -TAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGCGAAGCAAGTCTGAAGTGAAAACCCAAGGCTCAACCATGGGAGTGCTTTGGAAACTGTATTGCTGGAGTGCAGGAGAGGTAAGTGGAATTCCTAGTGTAGCGGTGAAATGCGTAGATATTAGGAGGAACACCAGTGGCGAAGGCGGCTTACTGGACTGTAACTGACGTTGAGGCTCGAAGGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCGGTAAACGATGATCACTAGGTGTCTGTGGTTTAGAACCATAGGTGCCGCAGCAAACGCAGTAAGTGATCCACCTGGGGAGTACGTTC,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Lachnospiraceae,Lachnospiraceae_X,Lachnospiraceae_X_sp.,100,100,100,100,100,100,100,58,58,seq57,seq57,42,42 -GAACGGATTAGGCTATGAATTTTCCATGCGTGAAGTCGCAGACCGCATGGTCGGTTTCTGGGCGGATGCGCTTGAAGAAGAAGGCATCATCGAAACGCCTGAACAGAAGCAGATTTTCTACGATGAAGTCGTCTATGCGCTGCTGGCCCAGATGTGGGCGCCGAACTCGCCGCAGTGGTTCAACACCGGACTGAAGCGCAGCTATGGAATCGCCGGAGACAAGGACGATCTGTATTACTATGATGAAAAGACGGGGGAAGTCGTCGAATCGGAAGACCGCTACACCCGCACTCAGGCTTCCGCCTGCTTTATTCTC,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,76,13,9,9,9,9,9,9,9,seq58,seq58,41,41 -AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCAGCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,100,100,100,100,100,100,100,58,54,seq59,seq59,40,40 -ATCATTACAAGCGTATATTAAAATTGTTGCATTTAAAAAGCTCGTAGTTGAATAATAGATTTGAAGTTAGATTGACCTAGTCAAGATATTCTTCATTTCTTTTGTTATATTTTCGGATATAACCATTTACTGTGAAAAAATTAGAGTGTTTAAAGCAAATTGTAAATTTGAATATTATTAGCATGGAATAATAATATATGATTAATATTATATATTATGGTAATATAGTATTAATAATGATTAATAGGGATAGTTGTGGGTATTCATATTTCATAGTCAGAGGTGAAATTCAAGGATTTATGAAAGATGAACGAATGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,99,99,97,97,97,97,77,77,77,seq60,seq60,39,39 -AGCTCTCCTGGTGTATGCAAACGCTGCTGCAGTTAAAGCGCTCCTAGTTGGCGGTCGGGCGCGGCCGCGGCGCCGGGACACTGCGCGCCGTCGCCGCGCGCTCGGGGTTACCATGAGAAAACCGTGACGCTCAAGGTAGGCGACTGAGCGCCCTAGCATGGGATAGCGGCGGGCTCCACGTGCGCTGCTGGTCCGTTCGGCGGAGCGAGAGGAAGAGGGGCAGTCGGGGGCCTCAGTACGATGGCGCCAGAGGTGAAATTCCGAGACCGCCGTCAGACTGCCGGCAGCGAAAGCGCTGGCCAAGGATGCTTTCG,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,75,73,73,73,71,71,71,71,seq61,seq61,38,38 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCAAGGCTTATGTGCCTCACAGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGGATTTAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,72,72,70,70,65,seq62,seq62,38,38 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCGGAAGTGCCTCCAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,86,86,85,85,82,seq63,seq63,37,37 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGAGCGGGCGGGCGGTCCGCCGCGAGGCGAGCCACCGCCCGTCCCCGCCCCTTGCCTCTCGGCGCCCCCTCGATGCTCTTAGCTGAGTGTCCCGCGGGGCCCGAAGCGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCCGAGCCGCCTGGATACCGCAGCTAGGAATAATGGAATAGGACCGCGGTTCTATTTTGTTGGTTTTCGGAACTGAGGCCATGATTAAGAGGGACGGCCGGGGGCATTCGTATTGCGCCGGTCCAAGAATTTCACCTCTAGCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Craniata,Craniata_X,Craniata_XX,Capra,Capra_hircus,100,100,100,100,100,100,55,34,34,seq64,seq64,33,33 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGTGTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,98,98,seq65,seq65,33,33 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCTGGGCCTTACCTCTTGGTGAGACCTCATGCTCTTTACTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq66,seq66,33,33 -CCAGAAGCTAAAGAGAAAACAGACTTGGCGTTTTCTTTAATACAACGTGTCCATGTAATGAAAACTGTGCATCATTATCCATTCCTTTAGGAAGCTAAGTCATCCCCATGGTAGAAGTGATGATTGGTGCCTTTTGCCTTTCTGACGGTAAGCACATTCTCTGGATATGAGATCCAGCTCCACTCACTCTGAACATCCAGGAAGCGGGGCCAGCTGTTCCCATGCCAGGCCCAAGCACTATCAGGTGTCCTCTGTATCCATCTTTGGTGTTCTTCCCAGCATGCACCACTCCAAGCGCCTCCCCCTTCTCAATCAGGCTCGGGTCTGAAGTCAAAGCAACACACGTCACAGCCGTCAAAAGCATTCTTTGTGGTACAGGCCACACCCAAGGAGCCACAGTATGA,Eukaryota,Excavata,Discoba,Euglenozoa,Euglenida,Euglenophyceae,Euglenaceae,Trachelomonas,Trachelomonas_grandis,78,31,31,31,30,12,12,9,8,seq67,seq67,33,33 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCAGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,65,65,seq68,seq68,32,32 -AGCTCCAATAGCGTATATTTAAGTTGTGGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,62,62,seq69,seq69,31,31 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTCGAACTTCGGGCCTGGCGGGACGGTCCGCCTTACGGTGTGTACTGTCCGGCCGGGTCTTACCTCCTGGTGAGGCCGTATGCCCTTTACTGGGTGTGCGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCATATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCTTTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Papiliotrema,Papiliotrema_fonsecae,100,100,100,100,100,100,100,95,95,seq70,seq70,31,31 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCCTATGTGCCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCATGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,73,73,67,67,64,seq71,seq71,31,31 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCCTCTGCCGCCCGGTCCGCCTATTTGGGTGTGTACTGGAGCGGTGGAGGCTTACCTCGTGGTGAACGATCATGCACTTTATTGGGTGTGGTCGGGAACCATGACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTACGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTCCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Cystobasidiomycetes_X,Cystobasidiomycetes_X_sp.,100,100,100,100,100,100,100,40,40,seq72,seq72,30,30 -AGCTCTGCCAGTGCATAGAACTATTGCTGCGGTTAAAAAGCTCGTAGTTGGATCTCTGTTCGCGGCCGGGTCGCTCCTTCGGGGGTGTACTCGCGACGTGGACATCCAGTCGATTCGTCCTCTGCTCGGGTTCGCCCTTGTGATTGGCGGTCGGCGTGTTTACCTTGAGCAAATCAGGGTGCTCAGGACAGGCATTGCGCCTGAATGTTCTTGCATGGAATAATAGAAGAGGATTTCGGTTCTGTTTTGTTGGTTTTGAAGCCGAGATAATGGTCAATAGAGACAAACGGGGGCATCGGTATTTCTGCGTGAGAGGTGAAATTCTTGGACCGCAGGAGGACCAACAACAGCGAAGGCAGTTGCCAAGAATGTCTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Nematoda,Chromadorea,Chromadorea_X,Miculenchus,Miculenchus_muscus,100,100,100,100,97,97,97,86,39,seq73,seq73,30,30 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCACCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCATTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTAGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,82,82,seq74,seq74,28,28 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCAATCGGTGTGCACCGGTCGTCTCGTCCCTTCTGCCGGCGATACGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATTATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Veronica,Veronica_anagallis-aquatica,100,100,100,100,100,100,100,64,64,seq75,seq75,28,28 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGATGGGACGATCGGTCCGCCTTTAGGTGAGCACCGGTCGTCCGGTCTCTTACGCCGGCGATGCGCTCCTAGCCTTAGTTGGCCGGGTCGTGCCTCCGGCACAGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAGGCCTACGCTCTGTATACATTAGCATGGGATAACATTATAGGATTCCGGTCCTATTGTGTTGGCCTTTGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Spinacia,Spinacia_oleracea,99,99,99,99,99,99,99,8,8,seq76,seq76,27,27 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCGTTTGGTGTGCACCGGTCGCCTCGTCCCTTCTGCCGGCGATGCGCTCCTGTCCTTAATTGGCCGGGTCGTGCCTCCGGCGTTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Apium,Apium_graveolens,100,100,100,100,100,100,100,72,72,seq77,seq77,27,27 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCCAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCTCCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,98,98,seq78,seq78,26,26 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCCGGCCGGGCCTTACCTCTTGGTGAGACCTCATGCACTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,100,100,100,100,100,seq79,seq79,24,24 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCCTGGCTGGACGGTCCGCCTTACGGTGTGCACTGTCCGGCCGGGCCTTACCTCCTGGTGAGGCCTCATGCCCTTTACTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGAGTCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTGACGGAAGACTAACAACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_dimennae,100,100,100,100,100,100,100,97,81,seq80,seq80,23,23 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGATCGGTCCGCCTCTGGTGTGCACCGGTCGGCTCGTCCCTTCTGCCGGCGATGCGCTCCTGGTCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACACCACAGGATTCTGATCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Embryophyceae_XXX,Embryophyceae_XXX_sp.,100,100,100,100,100,100,100,40,40,seq81,seq81,22,22 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGTGATGCGCTCCTAGCCTTAATTGGCCAGGTCGTGCCTCCGACATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,84,84,seq82,seq82,22,22 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTTCGCCTCACGGCGAGCACCGACCTACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Brachypodium,Brachypodium_distachyon,100,100,100,100,100,100,100,69,69,seq83,seq83,22,22 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTCGAACCTCGGGTCCGGCGGGATGGTCCGCCTTACGGTGTGTACTGTCTTGCTGGATCTTACCTCTTGGTGAAGCCTTATGCCCTTTACTGGGTGTAGGGTCGAACCAGGAATTTTACTTTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTAGTATTCCGTTGCTAGAGGTGAAATTCTTAGATTTACGGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Dioszegia,Dioszegia_crocea,100,100,100,100,100,100,100,95,60,seq84,seq84,22,22 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTTTTGGCTTATGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,85,85,73,seq85,seq85,22,22 -GCTCCTCCTCATACTGATGGCTCAGCGCCTCGTTGCGCTCGGTAAGCTCATCGTTGATCTCCTCCAGCTCCTCTTGCAGGCGCAGAAGCTCGGAGAGGTCCGCCGTCCACAGCACATGGCCGCCGGGGACGGGCATTCCCTGAAGCTGCACGCCGCCCGGAAGCATAACGGGGCCTGATTCCGTTTGGCGCATGACCGAT,Eukaryota,Obazoa,Opisthokonta,Metazoa,Mollusca,Cephalopoda,Cephalopoda_X,Spirula,Spirula_spirula,98,66,66,55,12,10,10,2,2,seq86,seq86,22,22 -TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGACGGCTTATTAAGTCTAGAATCAAAGCCCGGAGCTTAACTCCGGTTCGTTCTAGAAACTGGTAGGCTTGAGTATAGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCTATTACTGACGT,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,75,44,25,25,25,13,13,13,13,seq87,seq87,22,22 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTTGGGCCTGGTTGGCCAGTCTGGGTTTTTCCACGTACTGGGATGCAACCGGGCCTTTCCTTCTGGCTAACTGTGTGCTCCTTGTGGGTGCGCAGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCGTATTGCTCGGATATATTAGCATGGAATAATGGAATAGGACGTTTGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Kazachstania,Kazachstania_telluris,100,100,100,100,100,100,100,100,100,seq88,seq88,21,21 -TCCGACGACTACCACCCCGGCAAGAACATCTCCAGTCTGGTCCATACCATGGCACAGGCCTATCAGGGCACCGAGTCCATCTTGTATGCCCCCATGTTCGTGGCCAGCGATATCGTGCGCGGCGAGCTGATGAAGGAGCATATCATCCAAAACGCCATGAGCGAGATCGAGCAGGTCAATTGGATCCTCACCGGGATCGCAGACGTTTCTCAGGGGATGCCCAACTCCTGGGCTGGCTACATGACAGATGAGATACGCAACGAGCTGACAGCAAAGGGAGCCGTCGGTTATATCTGCGGGTATTTCTTCGACAGGAACGGACGGCTGCTTCAAAATCCCATCAATCGGAGCCTGATAGGCGTGTCTTTTCAGCAGA,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,87,21,21,21,15,14,14,14,14,seq89,seq89,21,21 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCTTGCGAGTCGGTCCGCCTTCTTGGTGTGTACTTACTTCGCGGGGACTTACCTCCTGGTGAACTGCAATGTCCTTTACTGGGTGTTGTAGCGAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Microbotryomycetes,Sphacelotheca,Sphacelotheca_koordersiana,100,100,100,100,100,100,97,60,60,seq90,seq90,20,20 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCGTTTGGTGTGCACCGGTCGCCTCGTCCCTTCTGCCGGCGATGCGCTCCTGTCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Apium,Apium_graveolens,100,100,100,100,100,100,100,84,84,seq91,seq91,20,20 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGTGCGACGACGCGGTCTGCCTCTGGTATGTACTGCGCTCGGCGCACCTTTCTGCCGGGGACGGGCTCCTGGGCTTTATTGTCTGGGACTCGGAGTCGGCGAGGTGACCTTGAGCAAACGAGAGTGTTCAAAGCAAGCCTACGCTCTGAATCATTTAGCATGGAATCACGTGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGAACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,100,100,100,100,100,100,100,100,100,seq92,seq92,20,20 -AGCTCTGCGAGTTTGCTCCCGTATTGTTGCAGTTAAAACGCCTGTAGCCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,66,seq93,seq93,20,20 -TGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGTGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,72,72,seq94,seq94,20,20 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGACCTGGCTGGGTGGTCCGCTTAACGGCGTGTACTGCCTGGCTGGGCCTTACCTCTTGGTGAGCCGGCGTGCCCTTTATTGGGGTGCGTCGGGGAACCAGGACTTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCCTATGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGAGTCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTGGTATTGAGTCGCTAGAGGTGAAATTCTTGGATTGACTCAAGACCGACTATTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Agaricomycetes,Coprinellus,Coprinellus_congregatus,100,100,100,100,100,100,100,31,30,seq95,seq95,19,19 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,60,60,seq96,seq96,19,19 -TCCCAAGGCACTTACAACAACGACGACATCCTCGTCCTGCTCTTCCACAATACCCTTCACAGTCAGCATGCTTGCCGCCGTTCCCACCGAAGTGCCGCCAAATTTTAATACTTTCATTCGCTTATTCAGGTCATTTATGCGCAAATTTACACATATTTGGTCATATTTGTTACATTTAATTGAATTTTTTGTACTTTTGTGTTGGCAAACACATATCAAACACCAAAGAATTATGAAAAGACTACTTATTATACTCTCAGTACTTGTGACAACGCTTTCCTCAGCGACGGGACAAACCTATTACAAA,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,85,60,48,47,36,28,28,28,28,seq97,seq97,19,19 -AGCTCCAATAGCGTATATTAATGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGACTTGGCGCACCTGGCCCGCCACGGTTACGTGTGTGAGTGCCGGGATGCGCCTGTCACTTTTCTAGTAAACTATTGTGCTCTTCATTGAGTGTGATAGGTAGCTAGATAATTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCGTTTGCTATGAATACATTAGCATGGAATAATAACTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,100,100,100,100,100,100,100,93,93,seq98,seq98,18,18 -AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGGCGTTCGGGTGGTACTGGGAGGCCGGGCCCTAGGCTCTGTGCTTCCTGGAGCCACCTTCTGAGTGAGGCAACTCACTTCGGTTACCATGAGAAAAGTGTAGCGCTCAAAGCAAGCTAAGCTGAGCATTTAAGCATGGGATAACAGGCTATGACTCCATAGACGCTGTTGGTCCGTTTAGCGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCCAAGACCGCCGTCAGACTAACTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,98,98,seq99,seq99,18,18 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCTTGTGAGTCGGTCCGCCTTCTTGGTGTGTACTTACTTCACGGGGACTTACCTCTTGGTGAATTACCATGTCCTTTACTGGGTGTGGTAACGAACCAAGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_hordea,100,100,100,100,100,100,80,80,78,seq100,seq100,17,17 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCATAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCACCGACCAACTCGACCCTTCAGCCGGCGATGCTCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,84,84,seq101,seq101,17,17 -AGCTCCAATAGCGTATATTAAAATTGTTGGCGTTAAAAAGCTCGTAGTCGAACTTCGGTGGCCGTCAGCCGGTCCGCTTTTAACGAGTGTGTACTGGATCTGATGGTTACTTTACCTCCTGGTGAACTAGCATGTCGTTTATTCGGCGTGTTAGGGAACCCGGACATTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCATGTTTGCCCGAATATCTTAGCATGGAATAATAGAATAGGACGTGCACGCCTATTGTGTTGGTCTCTAGGTGTGCCGTAATGATGAATAGGGGCGGTTGGGGGCATTTGTATTCAATTGCTAGAGGTGAAATTCTTGGATTTATTGAAGACAAACTACTGCGAAAGCATTTGCCAAGGACGCTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Agaricostilbomycetes,Bensingtonia,Bensingtonia_yuccicola,100,100,100,100,100,100,100,100,100,seq102,seq102,16,16 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTCTGCTAGCGAGAATAGGTCATCTCTTTGAGTATGTACTTGTTGTCGTTGGCATTAATCCGATTCATCTAACAGTTAAACCAAACTGTTGGATATTCGGAGCTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCCTTGAATACTCCAGCATGGAATAACAAGTAAGGACTCAAGTTCTTCTTGTTGGTTTAAGAGCCTGAGTAATGATTAAGAGGAACAGTTGGGGGCATTCGTACTTAGTAGTCAGAGGTGAAATTCTTAGATTTACTAAAGACGAACTACTGCGAAGGCATCTGCCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,75,75,seq103,seq103,16,16 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCTCACGGTGAGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCTGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,100,100,100,100,100,100,100,48,48,seq104,seq104,16,16 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGACTCGGTTGGGTGGTCTGCCTTACGGTATGTACTGCTCGACTGAGTCTTACCTCCTGGTGAGCCTGCATGTCCTTTACGGGGTGTGTAGGGGAACCAGGAATTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCATATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTTGGGGGCATTAGTATTCCGGTGCTAGAGGTGAAATTCTTAGATTGCCGGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Holtermannia,Holtermannia_corniformis,100,100,100,100,100,100,100,94,94,seq105,seq105,15,15 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCCGCCGGTCCGCCTCACGGCAAGCACCGACCAACTCGACCCTTTAGCCGGCGATGCGCTCCTAGCCTTGATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,88,88,seq106,seq106,15,15 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATCTTGGGTCGGGGGGAGCGGTCCGCCCCTCGTGGGTGTGCACTGGTCCACCCGACCTTTCTGCCGGGGACGCGCTCCTGGCCTTCGCTGGTCGGGACGCGGAGTCGGCGATGTTACTTTGAAAAAATTAGAGTGCTCAAAGCAAGCCTATGCTCTGAATACATTAGCATGGAATAACGTGATAGGACTCTGGTCCTGTTGTGTTGGTCTTCGGGACCGGAGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Sanionia,Sanionia_uncinata,100,100,100,100,100,100,100,100,100,seq107,seq107,15,15 -AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTTGGATCTCGGTCTGCCTCAAACGAGGTATGTACCAGGGATCTGAGACCATCCTCGAAGAAAACATGTCTGTCATTAAGTTAATGGGCATGGGATCTTCGTCATTTACTGTGAGCAAAATAGGGTGTTCAAAGCAGGCTTATGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACCTTGGTCTATTTTGTTGGTTTGTACTCCAAGGTAATGATTAATAGGGATAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,100,100,100,100,100,100,100,100,100,seq108,seq108,14,14 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTAGGCAAGGCCTGCTGGTCTGCCGACAGGCATGACTGGCTGTGCTGCGCCTCCTTTCGGGGAACGCCCCGGCTTAGCGGTCGGGGTTGGAACCGAGTTTTACTTTGAAGAAATTAGAGTGTTTAAGGCAGGCGTTTGCTTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTCTTATTTTGTTGGTTTGAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTAGTATTCAGTAGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTAGTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Rozellomycota,Rozellomycota_X,Rozellomycota_XX,Rozella,Rozella_sp.,100,79,79,69,42,42,42,28,28,seq109,seq109,14,14 -AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGATGATCGAGCATAGGTGAGGCACCGGGCTTTTTAGCTCTGAGTACCTTAGCGATGCCCTAAGGATGTTTAATCATCCCGGTTACCATGAGAAAAATGTAGCGCTCCAAGCAAGCTTTGCTGAGCATTTTAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCATAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,seq110,seq110,14,14 -GAGTGTTTTTCATTCCTCCACATCCTTTCCAAGGCGACCGTCCACCAGGAACAGGATGCCGATGATGACCACCATAAAGAGGGCCAGCACCAGCGCTGCGGAGGAGAGCTTTTGATAGTCCATGCTCTCAAAGGTATTGTTCATGAAATGCTGCAAAAGATACAGCCGGTCGAAGGGATATTTTCCCGTCAGCAGGTAGACCTCCCGAAATATCTTTAGAGAA,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,76,16,15,14,13,11,11,11,11,seq111,seq111,14,14 -TAGGTGGCAAGCGTTATCCGGATTTATTGGGTGTAAAGGGCGTGTAGGCGGGACTGCAAGTCAGATGTGAAAACTCAGGGCTCAACCCTGAGCCTGCATTTGAAACTGTAGTTCTTGAGTGCTGGAGAGGCAATCGGAATTCCGTGTGTAGCGGTGAAATGCGTAGATATACGGAGGAACACCAGTGGCGAAGGCGGATTGCTGGACAGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGGATACTAGGTGTGGGGGGACTGACCCCCTCCGTGCCGCAGCAAACGCAATAAGTATCCCACCTGGGGAGTACGATCGCAAGGTTGAAACT,Bacteria,Terrabacteria,Firmicutes,Firmicutes_X,Clostridia,Clostridiales,Ruminococcaceae,Flavonifractor,Flavonifractor_sp.,100,100,100,100,100,100,100,99,99,seq112,seq112,14,14 -AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTCGGATCTCGTCCCCTTAATAAGGAAGCGGAGATTTTTTCTAATCTCTATGCGCCTTATTGTCTGGACTACGGTGACTCAAGGCAACTTGAATGTCACCCTGTTACTTTGAGCAAATTGGAGTGCTCCAACCAAGCCTAAGCTTGTACAGCTCAGCATGGAATAACGAGATAGGACTTTGATTCTTCTTGTTGGTGTCACGAATCGATAGTAATGATTGATAAGGAAATTCGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCTGCAAAGACAAACGAATGCGAAAGCATTTGCCCAGTATCTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,100,98,98,98,98,98,98,41,26,seq113,seq113,13,13 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGCCGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCGGCGGGGACTTACCTCCTGGTGAGCTGCGCTGCCCTTTACTGGGTGGCGTAGGGAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_yarrowii,100,100,100,100,100,100,71,63,63,seq114,seq114,13,13 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Stichococcus,Stichococcus_bacillaris,100,100,100,100,100,99,99,54,54,seq115,seq115,13,13 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTGCCGTCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCATATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACCGCTGCGAAAGCTTTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,100,100,100,100,100,99,99,99,99,seq116,seq116,13,13 -TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTATAGAATTAAACTTCGGGGCTTAACCCCGTCTCGTTCTATAAACTGATAGACTAGAGTGTGGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCCATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,73,47,24,24,24,14,14,14,14,seq117,seq117,13,13 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGCTCAAGAATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTTCTGTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGATTGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCTGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,98,seq118,seq118,12,12 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGTTGGGTCGACCGGTCCGCCTTTTGGTGTGCACCGGTCGGCTCGTCCCTTCTACCGGCGATACGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGTGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGGATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTCTGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Prunus,Prunus_persica,100,100,100,100,100,100,100,85,85,seq119,seq119,12,12 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTCACGGTGTGCACCGGTTTACTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCTGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Artemisia,Artemisia_annua,100,100,100,100,100,100,100,91,91,seq120,seq120,12,12 -AGCTCTCCTAGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGACGATCAAGTACAAATGGATTGCCGGGCATTGAGCTCTGAGCAGTCTAGATGTACTTTACAAACCTGAAAAGGTTTCTGTTACCATGAGAAAATTGTAGCGCTCAAAGCAAGCTTAGCTGAGCATTATAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGACAGAGGTGAAATTCCGAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,seq121,seq121,12,12 -TGGATTTTGTCAACCGGCCGTCACCTCAAAACCACCTAAACAACTATATATCAATTGATTAAGAAAATATTTATGTTTTTATATCGGAAAGTAGTATTCATAAATGCAGAATGTCTGTTTATGTAAAGTTTACACAATAACTTGAAATGTTTGAAACGCTTAGGCACGAATTGCCGTGAATTCAGATTCGAATTTTATGCGATTTTTATTCCATCGGGATGCGGAAAGCGCTTTACCTTTGCACCGTCGAAAG,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,81,42,32,29,21,17,13,13,13,seq122,seq122,12,12 -AGCAGCTGCGGTGTGTACATGTACTACGACGCCGAGGGGACGGTGATATATGTCGGTAAGGCCAAAAATCTCAAGCGGCGCGTCTCGTCGTATTTCAACCGCACGCACGTCTCTACCCGTACAAATCTGCTCGTGCGCGCGATTGCCGATATGACCTACATCGTGGTCCCCACAGAGCAGGACGCGCTCAATCTTGAGAACTCCATGATTAAGGAATACCAGCCGCGCTACAACGTG,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Hexapoda,Insecta,Mengenilla,Mengenilla_chobauti,95,62,62,57,41,16,14,4,4,seq123,seq123,11,11 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACCTTGGGCCGGGTCGGCCGGTCCGCCTCACGGTGTGCACCGACCTACCCGACCCTTTTGTCGGCGATGCGTGCCGGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCCACGCTCTGCATACATTAGCATGGGATAACATCACAGGATTTCGGTCCTATTTTGTTGGCCTTCGGGATCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Juncus,Juncus_effusus,100,100,100,100,100,100,100,89,89,seq124,seq124,11,11 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTATCGGCGTAAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCAGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,71,71,68,68,64,seq125,seq125,11,11 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAATCTCGTAGTTGGATCTCAGGTCCAGGCTCGCGGTTCGTTTCGCGACGATACTGCCCGTCCTGACCTACCTCCCGGTTCTCCCTCGGTGCCCTTCGTTGAGTGCCCTGGGTAGCCGGAACGTTTACTTTGAAAAAATTAGAGTGCTCAAAGCAGGCAGTCTGCCTGAATAACCGCGCATGGAATAATGGAATAGGACCTCGGTTCTATTTTGTTGGTTTTCGGAACTCGAGGTAATGATTAAGAGAGACAGACGGGGGCATTCGTATTACGGTGTTAGAGGTGAAATTCTTGGATCGCCGTAAGACGAACTACTGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Annelida,Annelida_X,Annelida_XX,Achaeta,Achaeta_bifollicula,100,100,100,100,100,100,100,99,49,seq126,seq126,10,10 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAGTTGTGGCAGTAATAGTGGGTCATCTTTAACGAGCATGCACTTATTGTTATTGCCATTATTCTGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,100,99,99,99,99,98,66,66,seq127,seq127,10,10 -TCTATAAGAAGTTTTTCGCGGAAGACTTTGAGCAGATGATGAAATATGCTGACTGCTTGATTGTTCATAACGATTCCATGAAGCAGTTCTTTATCAACCGGGGCGTTGCACCAGAGAAGCTGGTTACCTTGGGGATTTTTGACTACTTGATCCCGGATGGCGAGATCAACCAGGCCAAGTTTGAACGGGCTGTTTCTGTCGCCGGTAACCTGGATGTCCGTAAGACCCAGTATTTGAACGATATCGGCAAAATCGATGCCAAGTTTAACCTTTACGGGCTGAATTTCACTTTGGACGCCTACAAGAATGTCGAATACCATGGGGCCTTTCCAGCCGATGAAATTCCCAAACAGCTGAATTCCGGCTTTGGC,Eukaryota,Amoebozoa,Evosea,Eumycetozoa,Myxogastria_Fuscisporidia,Myxogastria_Fuscisporidia_X,Myxogastria_Fuscisporidia_XX,Kelleromyxa,Kelleromyxa_fimicola,90,23,21,19,19,17,17,17,17,seq128,seq128,10,10 -AGCTCCAAGAGCGTATCTTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGTCTGGCTGGCCGGTCCGCTTTTTTGCGAGTACTGGACCCAGCCGGGCCTTTCCTTCTGGCTAGCCTTTTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGAACGGTTGGGGACATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTACTGCGAAAGCATTTGTCAAAGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Hyphopichia,Hyphopichia_pseudoburtonii,100,100,100,100,100,100,100,95,93,seq129,seq129,9,9 -AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTTCGGCGCACTTGGCCCGTCTCGGTTTACGGGATTGTGTGCCGGTGTGCGCCATCCATCCTTCGAGAGAACACTTCTACCCTTCACTGGGTCGGGAGTGCTATCTCGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,100,100,100,100,100,100,100,100,91,seq130,seq130,9,9 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGTCGCTGGATGGCCCCCTGCCTCACGGCAGCTGGTTTGGCTGACTCCTAGCGTCCCATCCTCGGGTGGGTCCTGCTTGGCATTAGGTTGTTGGGCAGGGGAAGCCCGTCTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTAGGCCGTTGAATACATTAGCATGGAATAATGAGATAGGGCCTTGATGGATTCTTCTATTTTGTTGGTTTGCACGCCAAGGCAATGATTAACAGGGACAGTTGGGGGTATTCGTATTCAAATGTCAGAGGTGAAATTCTTGGATTTTTTGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,100,100,100,100,100,100,100,89,89,seq131,seq131,9,9 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGTCGGTCCGCCGTTTCGGTGTGCACTGGCGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACCCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Pseudostichococcus,Pseudostichococcus_monallantoides,100,100,100,100,100,100,100,99,99,seq132,seq132,9,9 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGTGCTGCCGGTCCGCCCTTTGGGTGTGCACCGGTTGCGCCCGTCCTGCTGCCGGGGACGGGTGCCTGGGCTTCACTGTCCGGGTCCTGGAGTCGGTGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCCCGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,100,100,100,100,100,100,100,100,100,seq133,seq133,9,9 -AGCTCCAATAGTGTATGTTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGTGGTCATCCGGCTCCGCCCGTATGGGTGGGCGCCTGGTTTGCCCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTCACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,53,53,seq134,seq134,9,9 -AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGTGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCACCAAGTAAGGTCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,89,89,89,89,84,seq135,seq135,9,9 -GACAGAGATTGTTTATCAGGGGCAGACGATTTCGATTGCCCGCTCCTATATTTCTATCGAAGAAGGCGCTTTCAAAGGCAACACGCAACTTGAAACGATCGTGATTCCCTCTCACGTGAATATTATCGGCAAGGAAGCGTTCGCTCAGTGCACGGGGCTTAAGACGGTTGTCATTGAAGGCTCGACGGGCTTGCTTTCCAACTACAATGGTTTGGAGA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,7,7,7,7,6,6,6,6,6,seq136,seq136,9,9 -TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGACGGTTTGCTAAGTTTAGAATCAAACCCTGGAGCTTAACTCCAGTTCGTTCTAAAAACTGGCAGACTTGAGTGTAGTAGAGGCAAGTGGAATTTCTAGTGTAGCGGTAGAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGACTTGCTGGGCTATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,81,45,23,23,23,13,13,13,13,seq137,seq137,9,9 -AGCACCTGCGGTCTGCAGGGCACAGAGATCATCACCCAGGCCGATGGCTCCAACAACGACGCTCTCGACAAGCTCCAGGGCAAGTCGACCCGCACGGGCTACTACATGCGCAAACTCCTGCGTCAGGACGTGAGCCTCGACCCCGTATCGGCCACCGACCAGTATCACTACACCCCGCGTATCCGCTACACCGAGATCTTCCTGGCCTACGCAGAGGCAGCCAACGAGGCTTACGGCCCCATGGGTAAAGGCGGCAACAGCTACTCGGCCTACGACGTGATCAAGGCTATCCGCGAGCGCGCCGGCATCAGCGGCGACGCCTATCTGGAG,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,89,22,8,8,8,8,8,8,8,seq138,seq138,8,8 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTCTTCCTTCCTGTTTTAACCTTGCGGGGCTAAAATGAGGTTGGACTCCGTGATTGAGGTTATCTTAATTCATGGTGTTACTTTGACAAAACTATAGTGTTCCTGGCCGATCTTTGTCAGAATAATGAAGCATGGAATAACAGTTTAAAGCCCGTTTTTAGCTGTGGTCTGCTAAATGTGGAGCTATGATGAAAAGGAACTGCTGGGGGTAATCGAATTCATGGGTCAGAGGTGAAATTCTTGGATTCTGTGAAGACGAACGACTGCGAAAGCATCTATCAAATATGCTTCCA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,100,12,5,5,5,5,5,5,5,seq139,seq139,8,8 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACTTCAGGCTTGGTTGATTGGTCCGCCTCACGGTGTGTACTGTTCGACCGAGCCTTACCTCTTGGTGAGCCAGCATGCCGTTTATTCGGTGTGTTGGGGAACCAGGATTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTATGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTAGTATTCAGTCGCTAGAGGTGAAATTCTTGGATTGACTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Agaricomycetes,Mrakiella,Mrakiella_aquatica,100,100,100,100,100,100,100,93,93,seq140,seq140,8,8 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGGACTTCTGTCCGGGTGCTGTTCTCCGCCGTAAGGCGTGTAGATCAATACCTCGACATCCGTTCGTTTGAGCTTCTTGCTCTTTCGAACTCTTTACTTTGAGAAAATTAGAATGTTTCAAGTAGGCTTTCGCCTGAATACTACAGCATGGAATAATAAGATAGGACTCTGGTTCCTTCTTGTTGGTTCTTAGAACTAGAGTAATGGTTAATAGGGACAGTCGGGGGCATTCGTATTCTACCGTTAGAGGTGAAATTCTTGGATCGGTTGAAGACGAACAACTGCGAAAGCATCTGCCAAGGATGTACTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Branchiopoda,Caenestheria,Caenestheria_lutraria,98,55,55,55,11,5,2,1,1,seq141,seq141,8,8 -AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCGTTTCGGTGTGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,63,63,seq142,seq142,8,8 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGTTGGGTCGGCCGGTCCGCCTTTTGGTGTGCACCGGTTTACTCGTCCCTTTTGTCGGCGATACGCTCCTGGCCTTAGTTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Taraxacum,Taraxacum_kok-saghyz,100,100,100,100,100,100,100,64,64,seq143,seq143,8,8 -AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTTGAATTTCGTTCAAATTGAATAGTGCCGAGTTTTCCTCGTGTTAATTCGAGTACTTTTTGATTAGAACTTGGGTGAGATGTACTCGTTTCGACGAGTTCGTTTCGCTCCGTTACTTTGAGCAAATTGGAGTGCTCCAACCAGGCTTAAGCTTGAACAGCTCAGCATGGAATAACAAGATAAGACTTTAGTTCTTCTTGTTGGTGACATGAACTAATAGTAATGGTTGATAAGGACATACGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACAAACAAGTGCGAAAGCATTTGCCCAGTATGTACCTGTTAA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,100,100,100,100,100,100,100,87,76,seq144,seq144,7,7 -AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTATGGCGCACTTGGCCCGCCGAGGTTTCTCGGTCGTGTGCCGGTGTGCGCCTGCCATCCTTCTGGGGAACGGCCTTGCCCTTCACTGGGTGGAGGTCGGTATCCAGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_jutlandica,100,100,100,100,100,100,100,49,21,seq145,seq145,7,7 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGACTGAAGCATTTTGCCGGCCGCGGTTTCGCGTGTTAGTGCTTGATGCTCTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGAATGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,100,seq146,seq146,7,7 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAACTTTGGTAGTAATAATGGGTTATCTCTTTGAGTATGTACCTATTGTTACTACCATTATTCCGATCTATACAAGGGGTAATTCCCTTGTGTTCTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTGATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAGCGCCTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAGCAGTCGGGGGTATTCGTATTCAGTCGTTAGAGGTGAAATTCTTAGATTGACTAAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,100,100,seq147,seq147,7,7 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTGTGGTAATAACAATGAATCATCTTTAACGAGCATGCACTTATTGTTGTTGCCATTATTCCGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,99,96,96,96,96,96,51,51,seq148,seq148,7,7 -AGGATGTCCGTCACCTCGCTGCGCTCCACCCGCAGCACCTCATAGCCGTACCGGGGGATGCTGCGCAGGATCCCGGTATGGCACAGCGCCACAAGGGCCTCCCGCACAGAGGTCTTGCTGCATCCGTACTCATCGATCAGCGCCCGCTCCGTCAGGATATCCCCGGCCTTGTACCGGGAATTGAAGATATCATTTATCACGGCGTTGTAGACCGTGGAGGACACGGAGTTCTTCATTCGGCATCGCCTCGCTTTCCATTGATCATACCGGATATATGCGGTCTTGTCAAACAACTCTTGCATATTCTTGCTTATGGTGGTATGATTACTGGTATGACCAGCAGCTTTTTACTCAGACAGCCATGCCGCCATGCGAAGAAAGGAGAGACCGACCATGTCCAAGA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,88,33,15,15,15,15,15,15,15,seq149,seq149,7,7 -CTCCGTAAATACAATAGGCGCAAAGACACAGATGGTGGTAAGAGTGGATGCCGCAATAGCTCCCGCCACCTGCTTCGCCCCCTCAATCGCGGCCTCACTCGGAGAAAGCCCCTCTTCACTTCTCATGCGGTAAATATTTTCAATTACAACGATAGAATTATCCACCAGCATACCCACGCCCAACGCCAGGCCGGACAGAGAGATGATATTCAGAGTAATCCCTGTAAAGTACATTGCCACCAGCGCCGTCAGGATACTGATAGGGATTGAACAGGCGATCACCAGTGTAGACCTGACACTGCGCAGGAATACTAAAAGAATCAGAATC,Eukaryota,TSAR,Rhizaria,Cercozoa,Sainouroidea,Sainouroidea_X,Guttulinopsidae,Rosculus,Rosculus_liberus,79,22,10,6,6,6,6,5,5,seq150,seq150,7,7 -CTGCAGAAAAAGAAAAAGGCCAGAAGCGATAGCTTTCTGACCTGGGTTTTTGGTGGTCGATGAGGGATTTGAACCCCCGACCTTGTCCTTGTAAGGGACCTGCGCTCCCGCTGCGCCAATCGACCGGATGAAGCGTGCTGCTTGCAACAGCGTTTCATTCTCCCATAATCATCGAGTTGACGCAAGGGTATATGCTGCCAAAGGTGTGAAGCG,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,91,26,16,16,7,7,7,7,7,seq151,seq151,7,7 -GAAATCTCAGTTAAGTCTAAGGTTGGCGACGAGCGCAAGTACTTAGTGCCACTCTCCAAGCAGATTCTGGTACAAGAGAACGACTACGTTCGTGCTGGCACAGCACTCTCCGACGGTGCTATCACTCCTGCCGACATCCTCAACATCATGGGTCCGACAGCTGTGCAAGAGTACATCGTGAATGAGGTGCAAGACGTGTACCGCATGCAGGGTGTGAAGATCAATGACAAGCACTTCGAGGTTATCGTACGTCAGATGATGCGCAAGGTTAACATCCTTGAGCCTGGCGATACTATCTTCCTCGAGAGCCAAATCGTTGACAAACGCGACTTCATGGAGGAGAACGACCGCATCTGGGGCAAGAAGTTTGTAA,Eukaryota,Excavata,Discoba,Discoba_X,Heterolobosea,Tetramitia_VI,Tetramitia_VI_X,Euplaesiobystra,Euplaesiobystra_salpumilio,83,37,37,9,9,9,9,9,9,seq152,seq152,7,7 -GAACTGCCGCGAAAGGAGAACAGATATGAAGCTTAAAAAATTCGGAGCTACGGAACAGCGCGTCTACGATCTGGTGAAGCCTGTCACGGATGAGCTGGGATATTATCTTTGGGACGTCTGCTTTGTCAAAGAGGGCGCTGTACGCTATCTTCGTATTTTCATCGACTGCGACGAGGGCATTTCAATAGAGGACTGCGAAAGAGTCACAGCGCCTGTTGATCGCCTGCTTGACGAGGCTGAC,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,87,18,14,14,11,11,11,11,11,seq153,seq153,7,7 -TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTCCAAAATTAAAGCCCGAAGCTTAACTTCGGTTCGTTTTGGAAACTGGTAGACTCGAGTGTGGTAGAGGCAAGTGGAACTTCTAGTGTAGCGGTAAAATGCGTAGATATTAGAAAGAACACCAGTGGCGAAGGCGACTTGCTGGGCCACCACTGACGG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,64,40,12,12,12,8,8,8,8,seq154,seq154,7,7 -TGCGCTGAGTGCCGGGAAATACAGTATGGAGATGGTCAAAGAGCCCTCTGTTGGCAATCCCGCCGCGGGTTCATTTTGAAATATGTCCGGTCATCATAATTGAAATTATTCTCAACAAAAACTTGTACTGCAGTATCAGTGGTGCTATAATATAGCCAATGACAGAGAGCGGATTCAAATTACCAATAAAGGGGCGCATACATATGGACATCACAAGAGATATTTACTACGTCGGCGTCAACGATCATCAGCTTGATCTGTTCGAAAGTCAATACATTGTCCCCAACGGTATGGCTTATAATTCCTAT,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,2,2,2,2,2,2,2,2,2,seq155,seq155,7,7 -AACCAAACCAGCAAGAGAAGCATTCAAACACATGGAAACATCTGGTTTGCCATATTTGATCCATGTGAAGATCATGCAGGCAAATGTGGCAACTGCTGGTGCAATAGTAGTGGTAACGAAAATAGAGCCAAGCTGGGAAACAGAAGTAGCAGCGGCACCATTAAAGCCGTACCAACCAAACCAGAGAATAAAGCAGCCCAATGCACCAATTGTCAAAGAGTGACCGGGAATGGCATTGACTTTGATTTTCCCATCTTCTCCCTTTGTGAATTTACCAATTCGAGGGCCAAGAATGGCTGCACCAA,Eukaryota,TSAR,Stramenopiles,Gyrista,Bacillariophyceae,Bacillariales,Bacillariaceae,Fragilariopsis,Fragilariopsis_kerguelensis,90,21,11,11,10,9,9,9,9,seq156,seq156,6,6 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGCTTGGTTGGCCGGTCCGCCTTTTTGGCGAGTACTGGACCCAACCGAGCCTTTCCTTCTGGCTAACCTTTCGCCCTTGTGGTGTTTGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGTCGGGGGCATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTACCTGAAGACTAACTACTGCGAAAGCATTTGCCAAGGACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Debaryomyces,Debaryomyces_hansenii,100,100,100,100,100,100,100,100,100,seq157,seq157,6,6 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCGGAGCACCTTTTGCGAGTAACCGAAAGGTTATTACAATATAGGGGTGCGTCCTTTATCTGACCCATTGGTATGCCATTCATTTGGTGTGCCACTTTGGTTAGAAGTTTACCTTGAAAAAATTAGAGTGTTTAAAGCAAGTGAATAACAGCCTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTTCTATTTTGTTGGTTTCTAGGACCGAAGTAATGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Chytridiomycota,Olpidiales,Olpidiaceae,Olpidium,Olpidium_brassicae,100,94,94,86,37,16,16,16,16,seq158,seq158,6,6 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCAAGGGTCGCATCGGGGACAACCCGCATGCTCCCTACCAGTCTTAGACTGTTACTGTGAGAAAATTAGAGTGTTTCAAGCAGGCTGTTGCAGGAATACATTAGCATGGAATAACGAATGTGTCTAGAATCTTGGTTAATTCTAGATTACGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTGTTAAAGACTAACGTATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,100,100,100,100,100,98,98,78,78,seq159,seq159,6,6 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGCCGGGTCGGCCGGTCCGCCTCACGGCGAGCGTCGACCAACTCGACCCTTCAGCCGGCGATGCGCTCCTAGCCTTAATTGGCCGGGTCGTGCCACCGGCATCGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCATCGCTCTGGATACATTAGCATGGGATAACATCATAGGATTCCGGTCCTATTGTGTTGGCCTTCGGGATCGGAGTAATGATTAATAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Festuca,Festuca_rubra,100,100,100,100,100,100,100,67,67,seq160,seq160,6,6 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,99,99,99,99,34,seq161,seq161,6,6 -CATAACTATCGGTACGATTAGCTGTGACAATCTGCTCACCAGGAAGATTATAAATTTGGTCAGGCACCCAGTCAGCAATCACTTGGATACCAGCAGCGTGAAGAGAACGAACGGCATCTAGCAAGTCATTGAAAGAACCATACTTGTTGTTTTTACTCATAGCAATATCATAGCGATCCTCAAAGGCGTAGCCGTTTCCGATGATAGCG,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,14,13,12,12,12,12,12,12,12,seq162,seq162,6,6 -CTACACCGTGCAGGGCACCGGCACGAAGGTTGTAACCCTGTATGTGGACGGCAGTCAGTACGAGACGGCGACAGTGACGAGGAGCGGCGTGACCAACGGCAGTTTCACGATAGCGATGAGCGGTCTGGGCGTAGGCCGCCACACGGTGCAGCTGGTGGCCGAGATGGAGGCCAGCGCAGACCTAACGCTGAGGAGCGAGAGCATCTACATGGACATCT,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Diplomonadida,Hexamitidae,Enteromonas,Enteromonas_hominis,85,13,5,4,4,4,4,1,1,seq163,seq163,6,6 -GAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGCGGGCTGCTAAGTCAGCGGTAAAATGTCGGGGCTCAACCCCGGCCGGCCGTTGAAACTGGTGGTCTTGAGTGGGCGAGAAGCATGCGGAATGCGTGGTGTAGCGGTGAAATGCATAGATATCACGCAGAACGCCGATTGCGAAGGCAGCATGCCGGCGCCCGACTGACGCTGAAGCACGAAAGCGTGGGTATCGAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGAATGCTAGGTGTCCGGGGCGAGCGAGTCCTGGGTGCCACAGCGAAAGCGTTAAGCATTCCACCTGGGGAGTACGCCGGCAACGGTGAAACT,Bacteria,FCB,Bacteroidetes,Bacteroidetes_X,Bacteroidia,Bacteroidales,Muribaculaceae,Muribaculaceae_X,Muribaculaceae_X_sp.,100,100,100,100,100,100,76,76,76,seq164,seq164,6,6 -ACATGGAAGTATCCCAAGAAGCCCTATCGTCTGAAATTCGCAAAAAAGACAGACATGCCCGGGTCTCTGCGCTCTAAGAATTTTGCATTGATAGCTAATTATATAGATTGCACACTGATGCGCAACGCCATTGCGTTTGAAGTCGGGCGTCTGTTGGGAATGCCATTTACCAACCATGCCGTCCCCGTGCGGGTTTATCTCAACGGACGTCTGAAGGGTGCGTATTTCCTGACCGAAAAGATTGGCATCAGTAGCAGCAGCGTGGATATCGATGAAAGCACTGGAGTTCTCCTTGAGCTCGACTCCAATTATGATGAGAAATATTGTTTCCGCTCTCCGATATACAATCTGCCGGTGATGA,Eukaryota,Archaeplastida,Prasinodermophyta,Prasinodermophyta_X,Prasinodermophyceae,Prasinodermales,Prasinodermaceae,Prasinoderma,Prasinoderma_singulare,79,8,4,4,4,4,4,4,4,seq165,seq165,5,5 -AGCTCCAAAAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAACCTTGGGCTTGGTTAGCCGGTCCGCCTTTTGGTGAGTACTGGATCTAACCGAGCCTTTCCTTCTGGGTAACCTTTCTTTCGGGGAAGGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTTTGCTCGAATATATTAGCATGGAATAATAGAATAGGACGTTATGGTTCTATTTTGTTGGTTTCTAGGACCATCGTAATGATTAATAGGGACGGACGGGGGTATCAGTATTCAGTTGTCAGAGGTGAAATTCTTGGATTTACTGAAGACTAACTACTGCGAAAGCATTTACCAAGCACGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Ascomycota,Saccharomycotina,Saccharomycetales,Candida,Candida_austromarina,100,100,100,100,100,100,100,75,37,seq166,seq166,5,5 -AGCTCCAATAGCGTATATTAAAGTTGCTGCTGTTAAAAAGCTCGTAGTTGGATTTCTGATATAATATTACTGTCCCGCTGTGGTTACACATGTGAGTGACGGATAATATTTATCTGTTTTGTGTTTTTTGTTTTGATAGTATGTAAATATTATCAAAAACATACACATCTTTTACTTTGAACAAATTAGAGTGTTTCAAGCAGGCATTTATGCCTTGAATACATGAGCATGGAATAATCATCGAGGACTTTTAGTTCTATGTTGATTGGTTCTAGAACTATAGTAATGATGGATAGGGATAGTTGGGGGTGCTAGTATTCCGAGGCCAGAGGTGAAATTCTTGGATTCTCGGAAGACTCACTTAGGCGAAAGCATTCACCAAGGATGTCTTCA,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,100,100,100,100,100,100,99,99,99,seq167,seq167,5,5 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGAGATGGGTCGGCCGGTCCGCCTTTTGGTGTGCACCGATCGTCTCGTCTCTTCTGCCGGCGATACGCTCCTGTACTTAATTGGACGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Panax,Panax_notoginseng,100,100,100,100,100,100,100,85,83,seq168,seq168,5,5 -CCAAAGGATAAATATGAACAGGAGGAAAAACAAAATGGTAAAACAGAAAATCGGAAAGAGAATTCTGAGCCTTGTAATGGTGCTTTGCCTTGCTGCTTCGGTATTTTCTATCCCGGTTGTTGCGAACGCTGTAACAACCGAAAACGTAACGGCACAGGCAACGGATTACGGTCTCGTTGACGATGTTCAGCAGGGACAGATACTGCAGTGCTGGAACTGGTCGTACAACGGCATAAAGAACAATATGCAGAAGATTGCCGAGCAGGGCTTCTCTGCTATCCAGACCTCGCCTATCCAG,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,85,19,14,14,14,14,14,14,14,seq169,seq169,5,5 -CGCTTTAGCTCCGATTACCGCCGTCACTGATTTGCGCGGCGTTAAAGTGCAGCTTTCCGTCGCGGTCAGCCCGATGCGCTTGTCTGCCTGTAAAAAGCGGATAAGCTCGCTCTGCACGGCTATGGGATAATCGCCGTAGCCGGGACTGTACCGCCATGTGAACCCGCAGCCTGCAAGTTCGGCGGAGGCTTTTATCTCCTTTTCGGCGTCGTCGCAGAAGCTTTCCGTAAGCGCGGAAGCCGCCGCG,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Malacostraca,Porcellionides,Porcellionides_sexfasciatus,93,69,69,65,44,24,24,4,4,seq170,seq170,5,5 -TAGGTGGCGAGCGTTATCCGGATTTATTGGGCGTAAAGCGTCCGCAGCCGGTTTATTAAGTCTAGAATAAAAGCCTGGAGCTTAACTCCAGTTCGTTCTAGAAACTGATATACTTGAGTGTAGTAGAGGCAAATGGAATTTCTAGTGTAGCGGTAAAATGCGTAGATATTAGAAGGAACACCAGTGGCGAAGGCGATTTGCTAGGCTATTACTG,Bacteria,Terrabacteria,Tenericutes,Tenericutes_X,Mollicutes,Mollicutes_RF39,Mollicutes_RF39_X,Mollicutes_RF39_XX,Mollicutes_RF39_XX_sp.,64,43,21,21,21,6,6,6,6,seq171,seq171,5,5 -TGGAGCCGATTTATAAGATGCTCGCCCACTACACGGGCATTCTCCACCAGGTTTTCAGCCTCAACCACTTCAGCCACGGCGATAGCAGCAGCACATGCCAGGTGATTTCCGCCGAATGTGGTGCCGAGCATACCCTTTTTAGCCTCAAACTCAGGAGAGATCAGCACTGCTCCTACAGGGAAGCCGTTGGCTATACCTTTTGCCATAGTGATAAGGTCAGGACGGATATCGGCATACTGATGAGCGAAGAACTTGCCTGTGCGCCCATATCCGCTCTGTATCTCATCAAGGATAAGCATCACGCCATGCTTTTTCGTCACCTCACGCAGAGCGCGGAGGAAGTCGTCGCCGGGCATACGGATCCCTGCCACACCCTGTATGCCC,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,90,26,17,17,11,11,11,11,11,seq172,seq172,5,5 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTGTTCTGGCCATCCTTCCAATCGCTGCGTGCTTTTCTTCATTGATTAGTGCGTGGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,63,63,seq173,seq173,4,4 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTAGGCCTGGTTGGACGGTCTGCTCTAGGGTTTGTACTGTCCTGACCGGGTCTTACCTTCTGGTGAGCTGTCGTATTGTTTACTCAGTGCGGCAGGGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCATTCGCTTGAATACATTAGCATGGAATAATAGAATAGGACTTTGGTTCTATTTTGTTGGTTTCTAGGACCGAAGTAATGATTAATAGGGATAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Mucoromycota,Mortierellaceae,Mortierellaceae_X,Mortierella,Mortierella_hyalina,100,100,100,100,100,100,100,100,64,seq174,seq174,4,4 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGTCCCTGTCGGTCGGTCCGCCTTCTTGGTGTGTACTTACTCGACGGGGACTTACCTCCTGGTGAACTGCGATGTCCTTTACTGGGTGTCGTAGCGAACCAGGACTTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Rhodotorula,Rhodotorula_yarrowii,100,100,100,100,100,100,61,61,61,seq175,seq175,4,4 -AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGAAGTATATAATGAATTTCTAATTCAATTGTATATTGATTCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_sp.,100,99,99,96,96,96,96,89,70,seq176,seq176,4,4 -AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGACTTTGGGATGGGCCGGCCGGTCCGCCCTAGGTGTGCACCGGTCGTCTCGTCCCTTCTGTCGGCGATGCGCTCCTGGCCTTAATTGGCCGGGTCGTGCCTCCGGCGCTGTTACTTTGAAGAAATTAGAGTGCTCAAAGCAAGCCTACGCTCTGTATACATTAGCATGGGATAACATCATAGGATTTCGGTCCTATTACGTTGGCCTTCGGGATCGGAGTAATGATTAACAGGGACAGTCGGGGGCATTCGTATTTCATAGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Streptophyta,Streptophyta_X,Embryophyceae,Embryophyceae_X,Embryophyceae_XX,Nicotiana,Nicotiana_attenuata,100,100,100,100,100,100,100,84,84,seq177,seq177,4,4 -AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCGTCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAATTGCGTGTGTTGGTGTTTCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,31,30,seq178,seq178,4,4 -ATACTGGGCTGTACTTGGAGCTGGAAAGAAAAGGAAAAGATTAAATCAAAATCTAATTTGTGTCTCTCAGCATCATAACGACAGTCAAGAGGCAGCCAGGTGTACTGGGGAGCTCACAGGGTTAGGGTGGGACAGAATTAGGCTCACACCTGATTCTGCTCTGTAAGAATGTACAGCCCATTCAACTAACTTCTCCAAGCTTCAGAGCTTTAATTTTC,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,2,2,2,2,2,2,2,2,2,seq179,seq179,4,4 -ACAGGACTTCAAAGCCGTTTTTATGGTTATACGCGCGCGAGCTGTTTTTCGCGGCAGTTTTGGCGAGGTTTTCCGAGGCGGCAGGGAGTGATTTTTCTGATATTTTAGGTGTGGGATCTTTCTTTGATTTAGGCTGTTGGGGTTTCTTTTTTTTGCTCATTTTTTCGACTCCTATCTGCTTTATACTGCGAACATTATAGCACAAAAAATTATTTGTG,Eukaryota,TSAR,Rhizaria,Foraminifera,Globothalamea,Robertinida,Robertinidae,Robertina,Robertina_arctica,92,17,9,7,3,3,3,3,3,seq180,seq180,3,3 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTTGCCGGCCACGGTTTCGTGTGTTAGTGCTTGATGTTCTGGCCATCCTTCCAATCGCTATCGGCTGCTCTTCATTGAGCGGTCGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,99,88,85,seq181,seq181,3,3 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCAGCGCTGTTCTTGGCTCTCTGAGTCGGCTCAGTGTTTGGTCATCCGTATGGGAAGCTAGCTCGGCCTTCACTGGTCGGCTAGTGGATCATACACTTTACTTTGAAAAAATTAGAGTGTTTCAGGCAGGCAATTGCTTGGATACTTCAGCATGGAATAATGGAATAGGACTTTGACCTATTTGTTGGTTTCTTCGAGGTCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCATATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGATGAACTTATGCGAAAGCATTTGCCAAGGATGTTTTCATTAA,Eukaryota,TSAR,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Pseudoplatyophrya,Pseudoplatyophrya_nana,100,100,100,100,100,100,100,49,49,seq182,seq182,3,3 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTGGGGTGGGTTGACCGGTTTATCGAAAGATACATACTGGTCGACTGACTCTTTTCTTCTGGGATAGCTTCTGCTCTTTACTGAGTGGTTGTGATTTCCAGGATCTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCTTGTATACATTAGCATGGAATAATAGAATAGGACTTTTGATTCTATTTTGTTGGTTATAGAATCGAGTAATGATTAACAGGAACAGTCGTGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACTAACTAATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Chytridiomycota,Rhizophydiales,Gorgonomycetaceae,Gorgonomyces,Gorgonomyces_haynaldii,100,62,62,45,26,8,5,5,5,seq183,seq183,3,3 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCCTTTGCCGCCCGGTCCGCCTATTTGGGTGTGTACTGGAGCGGTGGAGGCTTACCTCGTGGTGAACGGCCATGTCCTTAACTGGGTGTGGTCGGGAACCATGACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTACGCCCGAATACATTAGCATGGAATAATAAAATAGGACGTGCGGTCCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATTGAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Occultifur,Occultifur_externus,100,100,100,100,100,100,100,35,35,seq184,seq184,3,3 -AGCTCCAATAGCGTATATTAAAGTTGTTGCCGTTAAAAAGCTCGTAGTCGAACTTCGGCGCCTGATGGCCGGTCCGCCTTCTTGGTGTGTACTTGCTCATCGGGCGCTTACCTCCTGGTGAACTCTGATGTCCTTTATTGGGTGTCAGAGCAAACCAGGACGTTTACTTTGAAAAAATTAGAGTGTTCAAAGCAGGCCTACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGCGCGTTCCCATTTTGTTGGTTTCTGAGATCGCCGTAATGATTAATAGGGATAGTTGGGGGCATTTGTATTCCGTCGTCAGAGGTGAAATTCTTGGATTGCCGGAAGACAAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Pucciniomycotina,Cystobasidiomycetes,Sporobolomyces,Sporobolomyces_roseus,100,100,100,100,100,100,60,60,59,seq185,seq185,3,3 -AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGATTGTTATGATGAATCTATGTATTCGATTATAATATGAATCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,100,98,97,97,97,97,97,95,46,seq186,seq186,3,3 -AGCTCCAATAGCGTATATTAAGATTGTTGCAGTTGAAACGTCCGTAGTTGAATTTCACTCAGAAACAGAGTACTCTTGCATTTCGCTTGAGAACTTTGTTTCGAGCGGGAATTTGCGCAAGCATTTTCTCAACGTTACTTTGAGCAAACTGGAGGGCTCCAGCCGGGTATTATGCCTTAACAGCTCAGCATGGAATAACAAGCTAGGATTTTGGTCCATCTTGTTGGTAACATGGATTAAAATAATGGTTGATAAGGATGTACGGGGACATTCGGACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACGAACAAGTGCGAAAGCATTTGTCCAGTACTTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,98,84,81,81,80,80,78,47,39,seq187,seq187,3,3 -CGAAATGAACCCCGCAAGATAGAATTTGGCACGATCTCCCGCATCAATATATCTTTTTTTGACGGGAGAAAAATACCCCGAAATAAAGCTCTCATTGACGATTCTGTTAGCTTCTTTTATAAGTTGAGTTTCCGTTATTTTGTTTTTTTGCGCCGTTTCGCTCATGACAAAATAAGCCTCTTCAAAAAACTGCGAACCTGTATTTTTCAGCATAAGCATTCGTTTTTCCTGACTGTTCACAAATATCACCGCTCAAATTATCGACCGTGTTTTTTTGCAATATTCAAAAACGGCATAATTATTTCGGCGCAA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,36,36,36,36,36,36,36,36,36,seq188,seq188,3,3 -CGGCAAAATCAAAAGTTTTACCCCTGCCCTTAGCTCCGACACTATGGATTCCGTGACAATCTGCAAAATGTAACGCTAAGGCATCTGACAGATTCGTCTTGCCTGTTCCGCTTGAACCCTCAATATATATCAGAGTACGCGAACAAGGATTTTTTTCATAGAAACGCTTTACATATTGAAGCCAGAAGTCATAGGCACGGTCGTACATCGGCAGA,Eukaryota,TSAR,Rhizaria,Foraminifera,Monothalamids,Monothalamids_Clade-C,Monothalamids_Clade-C_X,Shinkaiya,Shinkaiya_lindsayi,84,38,21,21,17,10,10,10,10,seq189,seq189,3,3 -TTCCCGGAGGCGGTGAACTACCTGCTGGACTTCCATGGCCGCGCCAGGGACTCCCCCAACGAGCGCCCCAAGCCCAGGGCCAAGCCGCCCCAGGACGAAAAGGTGCCCTTTGCCCTGCCCCCGCCCAACGCCGACCAGCGCCGGGTGTTTGCCTATCTGCGCAAGCGGGGGATCGCCGCCCAGGTGATCAACGGCTTCATCAACGCCGGCCTGCTCTACGAGGACGCCGAGCACCACAACTGCGTCTTTGTGGGCCGGGATACCGAGGGTGCGCCGGTGTTCGCCAACAAGCGCGGCACCTACGACCGGGACGGCAGCAGCT,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Diplomonadida,Hexamitidae,Giardia,Giardia_ardeae,97,21,8,8,8,8,8,8,6,seq190,seq190,3,3 -ACAACTGAACTCAATGCATTTACAAGGAATTCATTGCTCTTTTCCAATTTTTCCCTGCTTTCCATCAACTGTTTTGTCCGTTTTTCCAGCTTCTTCTCCATATTGAATCTGTTTTCAAACAGTTCAAGGATATTTAAAGTACGGCGTCTGATTACTTTAGCATCAAAAGGCTTATAAATAATATCGGATGCACCGTATTCATACGCTTTCTCATCTGTCTCTGCCGTTGCCTCTCCGGTAATCATAATGACGGGAATCTTATTCATATACTCCTTTTCCCGCATATGTTCCATTACTTCAAGACCTGATTTTTTCGGCATTACAAGA,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,19,18,18,18,15,15,15,15,15,seq191,seq191,2,2 -AGCACCCGCGGTCAATGATGTATTTGCCAAAGCGGCTGCAAGTGATGCAAACGGCGAATATATGCGTGATGCAAATGGCTGTTCCTGGGCAGTATTCAATATCAAAGGCGGGAACGGACTTTCCGTAAAGGACGGGAGCGCCGAAGACGGGCTTTATACCGAACAGCATTTGAAGGACTTAGGGCTGTATGATTACTATACCGTTGAAAACCTGATCATCAATGCGGATTTAAAGCTGGCGCCCTCACAGCTCAGCTTTAAGATGCCGGATGATAAGA,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,87,5,1,1,0,0,0,0,0,seq192,seq192,2,2 -AGCAGCTGCCGATCGAATCGTGCGCGTGATTTTTCATCAGCTCTTTCCATATTGCCTCGATAAGCCCATGATGATATTTGAAGCCTAAGCTGTAAGCAATGGATGCGAGCGGTTCAAGAATATTTGTGATCTTATTCTCAATTCTTGTGTTCGCGGATTTTAGGTCAGCTCTTGAGGAATAGATGCTGCGGTGAACACGCATATATTTTCCGTCAAGGAACTCGCCCTCAATAGTATCGAGCGGAACCTTTTCCAGCTTGTCAAAAATTTCCTCATACCTTCCCATGCGCGTTTTTCTATCGGGGTACAGCTTT,Eukaryota:plas,Archaeplastida:plas,Chlorophyta:plas,Chlorophyta_X:plas,Trebouxiophyceae:plas,Watanabea-Clade:plas,Watanabea-Clade_X:plas,Chloroidium:plas,Chloroidium_ellipsoideum:plas,9,8,8,8,8,8,8,8,8,seq193,seq193,2,2 -AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTAAGCCGGCCACGGTTTCGTGTGTTAGTGCTTGGTGTTCTGGCCATCCTTCCAATCGCTGCTGGCCTTTCTTCATTGATAGGTTGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,73,73,seq194,seq194,2,2 -AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTTGGATCTGCTGCAGTGGTCAGCCTTCACGGTTTGTACTGCTTGCTGGTGGATCTTTTTACCTTCTAGGGAGCCATCATCCTCTTAGTTGAGTGTGATGGGGATCTAGAACTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCATCTTACGCTTGAATACATTAGCATGGAATAATAAAATAGGACTTTGGTTTTATTTTGTTGGTTTCTAGGACCGAAGTAATGGTTAATAGGGATAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACTAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Blastocladiomycota,Blastocladiomycotina,Blastocladiomycetes,Physoderma,Physoderma_lycopi,100,100,100,100,100,100,100,100,99,seq195,seq195,2,2 -AGCTCCAGTAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTCGAACTTCGGGCTTGGCGGGATGGTCCGCCTTACGGTGTGTACTATTCTGCTGAGCCTTACCTCTTGGTGAGACCTCATGCTCTTTATTGGGTGTGGGGTGGAACCAGGAATTTTACCTTGAGAAAATTAGAGTGTTCAAAGCAGGCAAACGCCCGAATACATTAGCATGGAATAATAGAATAGGACGTGCGGTTCTATTTTGTTGGTTTCTAGGATCGCCGTAATGATTAATAGGGACGGTCGGGGGCATTGGTATTCCGTTGCTAGAGGTGAAATTCTTGGACCGGCGCAAGACGGACCAGAGCGAAAGCATTTGCCAAGAATGTTTTCA,Eukaryota,Obazoa,Opisthokonta,Fungi,Basidiomycota,Agaricomycotina,Tremellomycetes,Cryptococcus,Cryptococcus_carnescens,100,100,100,100,90,90,90,88,87,seq196,seq196,2,2 -CGCTTCTGGGCGAAATTCCTGTACGACATCGGCGTGATCCAATGCAAGGAGCCTTATCGCAAGAGAACCGCTCACGGCATGATCCTCGGCGAGGACGGCGGCAAAATGTCCAAATCCCGCGGAAACGTGATCAATCCCGACGATATCGTAAGAGACTACGGTGCCGATTCTCTGCGTATGTACGAAATGTTTATCGGCGACTTCGAAAAGAGTGCGCCTTGGTCGCAGGCT,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Malacostraca,Bathymedon,Bathymedon_obtusifrons,90,50,50,45,36,29,28,6,6,seq197,seq197,2,2 -GTGTCACGTCACAACAATCTCCATTCTGCCGCAAAGCGGCAACATAACTCGTGATGAAATTCGGCGGTCACGCCGACAAAATGGGGTTGGGTCCCATTTTGAGCTGAATTGGGGGCGTGAAATAACGTGTTTCCCGTTTCCGCATACCTCTATTTTCACGCTATGCCTTCCCCGATGCCGGTGCCGTTTTTCCGAATGGCGGCCGACTGTTTTTCGTGCTCGGAGGCGGTCTTTGAGAAGTAAAAAAGACCGTCATCGTCGCTGACGAAGAAGAAATAACTGTGCTCCTCGGGA,Eukaryota,Archaeplastida,Rhodophyta,Rhodophyta_X,Stylonematophyceae,Stylonematales,Stylonemataceae,Bangiopsis,Bangiopsis_sp.,96,19,17,17,16,16,16,16,16,seq198,seq198,2,2 diff --git a/cutadapt_primer_trimming_stats.txt b/cutadapt_primer_trimming_stats.txt deleted file mode 100644 index 1593477..0000000 --- a/cutadapt_primer_trimming_stats.txt +++ /dev/null @@ -1,129 +0,0 @@ -This is cutadapt 4.2 with Python 3.11.2 -Command line parameters: -g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G CYCCTACYYTMGYYCTKGA -m 200 --discard-untrimmed -o ./outputs/fastq/JIN-Nu-mul_R1_trimmed.fastq.gz -p ./outputs/fastq/JIN-Nu-mul_R2_trimmed.fastq.gz ./outputs/fastq/JIN-Nu-mul_*_L001_R1_001.fastq.gz ./outputs/fastq/JIN-Nu-mul_*_L001_R2_001.fastq.gz -Processing paired-end reads on 1 core ... -ERROR: [Errno 2] No such file or directory: './outputs/fastq/JIN-Nu-mul_*_L001_R1_001.fastq.gz' -This is cutadapt 4.2 with Python 3.11.2 -Command line parameters: -g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G CYCCTACYYTMGYYCTKGA -m 200 --discard-untrimmed -o ./outputs/fastq/JIN-Nu-mul_R1_trimmed.fastq.gz -p ./outputs/fastq/JIN-Nu-mul_R2_trimmed.fastq.gz ./inputs/fastq/JIN-Nu-mul_S54_L001_R1_001.fastq.gz ./inputs/fastq/JIN-Nu-mul_S54_L001_R2_001.fastq.gz -Processing paired-end reads on 1 core ... -Finished in 7.733 s (32.926 µs/read; 1.82 M reads/minute). - -=== Summary === - -Total read pairs processed: 234,851 - Read 1 with adapter: 229,487 (97.7%) - Read 2 with adapter: 177,592 (75.6%) - -== Read fate breakdown == -Pairs that were too short: 404 (0.2%) -Pairs discarded as untrimmed: 60,591 (25.8%) -Pairs written (passing filters): 173,856 (74.0%) - -Total basepairs processed: 117,895,202 bp - Read 1: 58,947,601 bp - Read 2: 58,947,601 bp -Total written (filtered): 80,551,787 bp (68.3%) - Read 1: 39,926,366 bp - Read 2: 40,625,421 bp - -=== First read: Adapter 1 === - -Sequence: CCAGCASCYGCGGTAATTCC; Type: regular 5'; Length: 20; Trimmed: 229487 times - -Minimum overlap: 3 -No. of allowed errors: -1-9 bp: 0; 10-19 bp: 1; 20 bp: 2 - -Overview of removed sequences -length count expect max.err error counts -3 604 3669.5 0 604 -4 100 917.4 0 100 -5 10 229.3 0 10 -9 1 0.9 0 1 -10 8 0.2 1 3 5 -11 4 0.1 1 3 1 -12 16 0.0 1 8 8 -13 10 0.0 1 2 8 -14 16 0.0 1 7 9 -15 10 0.0 1 4 6 -16 16 0.0 1 2 14 -17 28 0.0 1 13 15 -18 120 0.0 1 15 44 61 -19 1565 0.0 1 373 697 495 -20 51044 0.0 2 42814 4174 4056 -21 77925 0.0 2 66902 5338 5685 -22 49668 0.0 2 42264 3789 3615 -23 48268 0.0 2 42172 2777 3319 -24 71 0.0 2 14 50 7 -29 1 0.0 2 1 -68 1 0.0 2 1 -78 1 0.0 2 0 0 1 - - -=== Second read: Adapter 2 === - -Sequence: ACTTTCGTTCTTGATYRA; Type: regular 5'; Length: 18; Trimmed: 120098 times - -Minimum overlap: 3 -No. of allowed errors: -1-9 bp: 0; 10-18 bp: 1 - -Overview of removed sequences -length count expect max.err error counts -3 281 3669.5 0 281 -4 254 917.4 0 254 -5 2 229.3 0 2 -6 1 57.3 0 1 -7 3 14.3 0 3 -10 2 0.2 1 0 2 -11 4 0.1 1 3 1 -12 2 0.0 1 1 1 -13 6 0.0 1 1 5 -14 8 0.0 1 4 4 -15 19 0.0 1 10 9 -16 11 0.0 1 3 8 -17 443 0.0 1 50 393 -18 63922 0.0 1 32105 31817 -19 19015 0.0 1 13804 5211 -20 13766 0.0 1 10057 3709 -21 21942 0.0 1 15923 6019 -22 33 0.0 1 9 24 -60 2 0.0 1 0 2 -61 2 0.0 1 0 2 -184 2 0.0 1 0 2 -185 80 0.0 1 1 79 -186 197 0.0 1 169 28 -187 42 0.0 1 32 10 -188 37 0.0 1 22 15 -189 19 0.0 1 17 2 -190 2 0.0 1 0 2 -191 1 0.0 1 1 - - -=== Second read: Adapter 3 === - -Sequence: CYCCTACYYTMGYYCTKGA; Type: regular 5'; Length: 19; Trimmed: 57494 times - -Minimum overlap: 3 -No. of allowed errors: -1-9 bp: 0; 10-19 bp: 1 - -Overview of removed sequences -length count expect max.err error counts -3 351 3669.5 0 351 -4 103 917.4 0 103 -9 1 0.9 0 1 -10 5 0.2 1 2 3 -11 13 0.1 1 2 11 -12 19 0.0 1 4 15 -13 487 0.0 1 21 466 -14 51676 0.0 1 7573 44103 -15 2734 0.0 1 387 2347 -16 1438 0.0 1 352 1086 -17 647 0.0 1 129 518 -19 1 0.0 1 0 1 -47 1 0.0 1 0 1 -165 3 0.0 1 3 -182 7 0.0 1 0 7 -183 1 0.0 1 0 1 -184 5 0.0 1 2 3 -185 2 0.0 1 0 2 diff --git a/pipelinesteps.txt b/pipelinesteps.txt index c067831..e170048 100644 --- a/pipelinesteps.txt +++ b/pipelinesteps.txt @@ -5,8 +5,8 @@ L bash cutadapt to trim .fastq.gz files `python3 cutadapt_test.sh` L R filter and taxonomic assignment L output .fasta and taxonomy.csv sftp .fasta to server -S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i input.fasta -o output.fasta` -S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global input.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` +S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i fastaseqs.fasta -o output.fasta` +S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global fastaseqs.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` S output taxonomy.tsv sftp taxonomy.tsv to local L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` diff --git a/protist_filtered.csv b/protist_filtered.csv deleted file mode 100644 index 8b6d7c7..0000000 --- a/protist_filtered.csv +++ /dev/null @@ -1,59 +0,0 @@ -SeqName,Pident,Accession,rRNA,Organellum,specimen,Domain,Supergroup,Division,Subdivision,Class,Order,Family,Genus,Species,sequence,tax.Domain,tax.Supergroup,tax.Division,tax.Subdivision,tax.Class,tax.Order,tax.Family,tax.Genus,tax.Species,boot.Domain,boot.Supergroup,boot.Division,boot.Subdivision,boot.Class,boot.Order,boot.Family,boot.Genus,boot.Species,JIN-Nu-mul,JIN-Nu-mul2 -seq2,99.2,AY321149.1.1455_U,18S_rRNA,nucleus,strain_T,Eukaryota,Metamonada,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,76,11240,11240 -seq10,99.5,MT991542.1.2393_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTTTGGTGTGCACTGGCATGGGCTCGCCTCGCTGGCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGCCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTACGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,100,100,100,100,67,920,920 -seq17,92.0,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCATTGTGCTTATGTGTCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,86,86,83,339,339 -seq22,95.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTCGCAAGAGGCGAGAGTGCCATTAGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTTTACCTACAGGTAAGATCAATGAGAGCCACCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,98,98,98,98,90,275,275 -seq25,100.0,EU091849.1.1430_U,18S_rRNA,nucleus,clone_P2-3m2,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Watanabea-Clade_XX,Watanabea-Clade_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,94,238,238 -seq31,100.0,MZ558750.1.2453_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_orientale,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,100,61,188,188 -seq35,91.2,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTGGCGTATGTGCCTTAGGGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,80,80,80,80,79,120,120 -seq38,90.9,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTCGGCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGTTTCAGGAATGACTCAGCGCAGTATGATATCTCTACTTCCAAGTAAAATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAAGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,81,81,78,78,75,107,107 -seq43,90.8,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCGGAATGACTCAGCGCAGTATGATATCTTTACCTCGAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,63,63,62,62,61,78,78 -seq47,99.5,EU091849.1.1430_U,18S_rRNA,nucleus,clone_P2-3m2,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Watanabea-Clade_XX,Watanabea-Clade_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCCGCCGGTCCGCCTCTGGTGTGCACTGGCGTGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_saccharophila,100,100,100,100,100,100,100,94,65,58,58 -seq51,91.6,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCTTGTCGGCGTATGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,83,83,82,82,79,49,49 -seq53,94.2,FJ355409.1.1048_U,18S_rRNA,nucleus,clone_051102_S1_W_T_SDP18_098,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTCGAGACGCAGCCAGGCTCAAGGGCCGATACTGCGGATTGGGACCATCCTCGAGAAGAACATATCTGTCATTGAGTTGATGGGTATGGGACTCTCGTCTTTTACTGTGAGCAAAATAGAGTGTTCAAAGCAGGCTTACGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACTTCGGTCTATTTTGTTGGTTATACTCCGAAGTAATGATTAATAGGGACAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadales_clade-XIII,Ochromonadales_clade-XIII_X,Ochromonadales_clade-XIII_X_sp.,100,100,100,100,100,100,70,70,70,48,48 -seq54,90.5,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTATGTACCTCCAAGTACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,88,88,88,47,47 -seq59,97.4,KF648870.1.1484_U,18S_rRNA,nucleus,strain_MTZ1,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCAGCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,100,100,100,100,100,100,100,58,54,40,40 -seq60,88.5,NGBS01000218.13865.15598_U,18S_rRNA,nucleus,strain_LA,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,ATCATTACAAGCGTATATTAAAATTGTTGCATTTAAAAAGCTCGTAGTTGAATAATAGATTTGAAGTTAGATTGACCTAGTCAAGATATTCTTCATTTCTTTTGTTATATTTTCGGATATAACCATTTACTGTGAAAAAATTAGAGTGTTTAAAGCAAATTGTAAATTTGAATATTATTAGCATGGAATAATAATATATGATTAATATTATATATTATGGTAATATAGTATTAATAATGATTAATAGGGATAGTTGTGGGTATTCATATTTCATAGTCAGAGGTGAAATTCAAGGATTTATGAAAGATGAACGAATGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Proteromonadidae,Proteromonas,Proteromonas_lacertae,99,99,97,97,97,97,77,77,77,39,39 -seq62,90.4,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGCAAGGCTTATGTGCCTCACAGCACGTTCACTGTGAACAAATCAGGACGCTTAGGGTATGGGATTTAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,72,72,70,70,65,38,38 -seq63,90.1,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCGGAAGTGCCTCCAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCTATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,86,86,85,85,82,37,37 -seq71,91.2,KC953859.1.1483_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTACGGCCTATGTGCCTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCATGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTCTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,73,73,67,67,64,31,31 -seq85,92.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCTTTTGGCTTATGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCGTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCTAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,88,88,85,85,73,22,22 -seq92,100.0,MZ067863.1.738_U,18S_rRNA,nucleus,strain_ACKU_309-4,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGTGCGACGACGCGGTCTGCCTCTGGTATGTACTGCGCTCGGCGCACCTTTCTGCCGGGGACGGGCTCCTGGGCTTTATTGTCTGGGACTCGGAGTCGGCGAGGTGACCTTGAGCAAACGAGAGTGTTCAAAGCAAGCCTACGCTCTGAATCATTTAGCATGGAATCACGTGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGAACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Chlamydomonadales_X,Chlorosarcinopsis,Chlorosarcinopsis_bastropiensis,100,100,100,100,100,100,100,100,100,20,20 -seq93,97.9,AY321149.1.1455_U,18S_rRNA,nucleus,strain_T,Eukaryota,Metamonada,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,AGCTCTGCGAGTTTGCTCCCGTATTGTTGCAGTTAAAACGCCTGTAGCCAGAATACTGCAAACTGCGAATAAATCAGGACGCTCAGGGTACATTTTATGGAATGACCATGCGCAGGATTTGAAATTTTCTACGGAAAGCTCAATGAGAGCCATCGGGGTCAGTGCTATTCAGTGGCGAACGGTGGAATGCAGTGACCCACTGGAGAGGAGCGGAAGCGAAGGCTACTGACAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Trichomonadea,Honigbergiellida,Hexamastigidae,Hexamastix,Hexamastix_kirbyi,100,100,100,100,100,100,100,100,66,20,20 -seq96,100.0,FJ592322.1.1085_U,18S_rRNA,nucleus,clone_B06_SE1A,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,60,60,19,19 -seq98,100.0,AB695526.1.1759_U,18S_rRNA,nucleus,clone_MPE2-32,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,AGCTCCAATAGCGTATATTAATGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGACTTGGCGCACCTGGCCCGCCACGGTTACGTGTGTGAGTGCCGGGATGCGCCTGTCACTTTTCTAGTAAACTATTGTGCTCTTCATTGAGTGTGATAGGTAGCTAGATAATTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCGTTTGCTATGAATACATTAGCATGGAATAATAACTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Allapsidae,Allapsidae_X,Allapsidae_X_sp.,100,100,100,100,100,100,100,93,93,18,18 -seq99,84.9,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGGCGTTCGGGTGGTACTGGGAGGCCGGGCCCTAGGCTCTGTGCTTCCTGGAGCCACCTTCTGAGTGAGGCAACTCACTTCGGTTACCATGAGAAAAGTGTAGCGCTCAAAGCAAGCTAAGCTGAGCATTTAAGCATGGGATAACAGGCTATGACTCCATAGACGCTGTTGGTCCGTTTAGCGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCCAAGACCGCCGTCAGACTAACTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,98,98,18,18 -seq103,92.3,EF024471.1.1761_U,18S_rRNA,nucleus,clone_Elev_18S_1147,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTCTGCTAGCGAGAATAGGTCATCTCTTTGAGTATGTACTTGTTGTCGTTGGCATTAATCCGATTCATCTAACAGTTAAACCAAACTGTTGGATATTCGGAGCTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCCTTGAATACTCCAGCATGGAATAACAAGTAAGGACTCAAGTTCTTCTTGTTGGTTTAAGAGCCTGAGTAATGATTAAGAGGAACAGTTGGGGGCATTCGTACTTAGTAGTCAGAGGTGAAATTCTTAGATTTACTAAAGACGAACTACTGCGAAGGCATCTGCCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,75,75,16,16 -seq104,98.9,EF200530.1.1708_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCTCACGGTGAGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCTGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiola,Prasiola_crispa,100,100,100,100,100,100,100,48,48,16,16 -seq108,100.0,DQ388549.1.1646_U,18S_rRNA,nucleus,,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,AGCTCCAATAGCGTATACTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCTTGGATCTCGGTCTGCCTCAAACGAGGTATGTACCAGGGATCTGAGACCATCCTCGAAGAAAACATGTCTGTCATTAAGTTAATGGGCATGGGATCTTCGTCATTTACTGTGAGCAAAATAGGGTGTTCAAAGCAGGCTTATGCCGTTGAATACATTAGCATGGAATAATAAGATAGGACCTTGGTCTATTTTGTTGGTTTGTACTCCAAGGTAATGATTAATAGGGATAGTTGGGGGTATTCGTATTCAATTGTCAGAGGTGAAATTCTTGGATTTATGGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Chrysophyceae,Ochromonadales,Ochromonadaceae,Spumella,Spumella_vulgaris,100,100,100,100,100,100,100,100,100,14,14 -seq110,87.8,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTGGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGATGATCGAGCATAGGTGAGGCACCGGGCTTTTTAGCTCTGAGTACCTTAGCGATGCCCTAAGGATGTTTAATCATCCCGGTTACCATGAGAAAAATGTAGCGCTCCAAGCAAGCTTTGCTGAGCATTTTAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGCCAGAGGTGAAATTCATAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,14,14 -seq113,84.5,EU087208.1.865_U,18S_rRNA,nucleus,clone_1_39,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Amoebogregarina,Amoebogregarina_sp.,AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTCGGATCTCGTCCCCTTAATAAGGAAGCGGAGATTTTTTCTAATCTCTATGCGCCTTATTGTCTGGACTACGGTGACTCAAGGCAACTTGAATGTCACCCTGTTACTTTGAGCAAATTGGAGTGCTCCAACCAAGCCTAAGCTTGTACAGCTCAGCATGGAATAACGAGATAGGACTTTGATTCTTCTTGTTGGTGTCACGAATCGATAGTAATGATTGATAAGGAAATTCGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCTGCAAAGACAAACGAATGCGAAAGCATTTGCCCAGTATCTGCCTG,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Gregarina_caledia,100,98,98,98,98,98,98,41,26,13,13 -seq115,100.0,KM020169.1.1775_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Chlorophyceae,Chlamydomonadales,Coccomyxaceae,Coccomyxa,Coccomyxa_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGCCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Stichococcus,Stichococcus_bacillaris,100,100,100,100,100,99,99,54,54,13,13 -seq116,100.0,X74000.1.2286_R,18S_rRNA,nucleus,strain_Andreyeva_748-I,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTGCCGTCGGTCCGCCGTTTCGGTGTGCACTGACGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCATATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGATGAACCGCTGCGAAAGCTTTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Chlorellales,Chlorellales_X,Chlorella,Chlorella_mirabilis,100,100,100,100,100,99,99,99,99,13,13 -seq118,100.0,EU709161.6.1494_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGCTCAAGAATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTTCTGTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGATTGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCTGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,98,12,12 -seq121,88.0,OM987996.1.1692_U,18S_rRNA,nucleus,strain_PHEM1,Eukaryota,Metamonada,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,AGCTCTCCTAGTGTATGCAAAAGTTGCTGCAGTTAAAGCGCTCCTATTTGACGATCAAGTACAAATGGATTGCCGGGCATTGAGCTCTGAGCAGTCTAGATGTACTTTACAAACCTGAAAAGGTTTCTGTTACCATGAGAAAATTGTAGCGCTCAAAGCAAGCTTAGCTGAGCATTATAGCATGGGATAACAAGCCACGACTCCACGTGCGCTGTTGGTCCGTTCGGAGGAGTAATGAGTAAGAGGGGCAGTTGGGGGCCTTAGTACGATGGCGACAGAGGTGAAATTCCGAGACCGCCGTCAGACTACCTGAAGCGAAAGCGTTGGCCAAGGATGCTTTCA,Eukaryota,Excavata,Metamonada,Fornicata,Fornicata_X,Fornicata_XX,Caviomonadidae,Iotanema,Iotanema_spirale,100,100,100,100,100,100,100,100,100,12,12 -seq125,90.5,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTATCGGCGTAAGTGCCTTAAAGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTAGGGATTCCAGAATGACTCAGCGCAGTATGATATCTTTACTTCCAAGTAAGATCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,71,71,68,68,64,11,11 -seq127,90.8,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAGTTGTGGCAGTAATAGTGGGTCATCTTTAACGAGCATGCACTTATTGTTATTGCCATTATTCTGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,100,99,99,99,99,98,66,66,10,10 -seq130,100.0,AY884326.1.1078_U,18S_rRNA,nucleus,strain_CeS-2,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTTCGGCGCACTTGGCCCGTCTCGGTTTACGGGATTGTGTGCCGGTGTGCGCCATCCATCCTTCGAGAGAACACTTCTACCCTTCACTGGGTCGGGAGTGCTATCTCGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Cercomonas,Cercomonas_sp.,100,100,100,100,100,100,100,100,91,9,9 -seq131,100.0,AJ579340.1.1732_U,18S_rRNA,nucleus,strain_SAG_30.83,Eukaryota,Stramenopiles,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGGTCGCTGGATGGCCCCCTGCCTCACGGCAGCTGGTTTGGCTGACTCCTAGCGTCCCATCCTCGGGTGGGTCCTGCTTGGCATTAGGTTGTTGGGCAGGGGAAGCCCGTCTTTTACTGTGAAAAAATTAGAGTGTTCAAAGCAGGCTTAGGCCGTTGAATACATTAGCATGGAATAATGAGATAGGGCCTTGATGGATTCTTCTATTTTGTTGGTTTGCACGCCAAGGCAATGATTAACAGGGACAGTTGGGGGTATTCGTATTCAAATGTCAGAGGTGAAATTCTTGGATTTTTTGAAGACGAACTACTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Gyrista,Xanthophyceae,Xanthophyceae_X,Xanthophyceae_XX,Botrydiopsis,Botrydiopsis_callosa,100,100,100,100,100,100,100,89,89,9,9 -seq132,100.0,JN573894.1.1681_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Diplosphaera,Diplosphaera_sp.,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTTCCGTCGGTCCGCCGTTTCGGTGTGCACTGGCGGCGCCCATCTTGCTGCCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACCCGGAGTCGGCGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Pseudostichococcus,Pseudostichococcus_monallantoides,100,100,100,100,100,100,100,99,99,9,9 -seq133,100.0,FR693368.1.1633_U,18S_rRNA,nucleus,,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGTGCTGCCGGTCCGCCCTTTGGGTGTGCACCGGTTGCGCCCGTCCTGCTGCCGGGGACGGGTGCCTGGGCTTCACTGTCCGGGTCCTGGAGTCGGTGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCCCGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTTCTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Apatococcus,Apatococcus_lobatus,100,100,100,100,100,100,100,100,100,9,9 -seq134,99.2,JQ993661.1.1396_U,18S_rRNA,nucleus,,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,AGCTCCAATAGTGTATGTTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGTGGTCATCCGGCTCCGCCCGTATGGGTGGGCGCCTGGTTTGCCCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTCACTGCGTGTGTTGGTGTTCCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,53,53,9,9 -seq135,91.3,KC953858.1.1489_U,18S_rRNA,nucleus,,Eukaryota,Metamonada,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,AGCTCTGCGAGTTTGCTCCCATATTGTTGCAGTTAAAACGCCCGTAGTCGGAGTTGCCGTTTTTCGGCGTGTGTGCTTCGGCACGTTCACTGTGAACAAATCAGGACGCTTAGTGTATGGATTCAGGAATGACTCAGCGCAGTATGATATCTTTACTCACCAAGTAAGGTCAATGAGAGCCATCGGGGGTAGATCTATTCCATGGCGAGCGGTGGAATGCTTTGACCCATGGGAGAGAAACGAAGGCGAAGGCATCTACCAAGAGGGTTTCTG,Eukaryota,Excavata,Metamonada,Parabasalia,Tritrichomonadea,Tritrichomonadida,Simplicimonadidae,Simplicimonas,Simplicimonas_similis,100,100,100,100,89,89,89,89,84,9,9 -seq141,76.9,AY919792.1.1726_U,18S_rRNA,nucleus,clone_LG32-03,Eukaryota,Alveolata,Alveolata,Chrompodellids,Colpodellidea,Colpodellida,Colpodellida_CHR1,Colpodellida_CHR1_X,Colpodellida_CHR1_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGGACTTCTGTCCGGGTGCTGTTCTCCGCCGTAAGGCGTGTAGATCAATACCTCGACATCCGTTCGTTTGAGCTTCTTGCTCTTTCGAACTCTTTACTTTGAGAAAATTAGAATGTTTCAAGTAGGCTTTCGCCTGAATACTACAGCATGGAATAATAAGATAGGACTCTGGTTCCTTCTTGTTGGTTCTTAGAACTAGAGTAATGGTTAATAGGGACAGTCGGGGGCATTCGTATTCTACCGTTAGAGGTGAAATTCTTGGATCGGTTGAAGACGAACAACTGCGAAAGCATCTGCCAAGGATGTACTCA,Eukaryota,Obazoa,Opisthokonta,Metazoa,Arthropoda,Crustacea,Branchiopoda,Caenestheria,Caenestheria_lutraria,98,55,55,55,11,5,2,1,1,8,8 -seq142,99.0,AJ431572.1.1716_U,18S_rRNA,nucleus,strain_SAG_35.83,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Desmococcus,Desmococcus_olivaceus,AGCTCCAATAGCGTATATTTAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGATGGGTACCGTCGGTCCGCCGTTTCGGTGTGTTACTGGCGGCGCCCATCTTGCTGTCGGGGACGGGCTCCTGGGCTTAACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACATTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTAGGACCGGAGTAATGATTAAGAGGGACAGTCGGGGGCATTCGTATTTCATTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Prasiolales,Prasiolales_X,Prasiolales_XX,Prasiolales_XX_sp.,100,100,100,100,100,100,100,63,63,8,8 -seq144,98.0,EF586127.1.796_U,18S_rRNA,nucleus,clone_121-O29,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,AGCTCCAATAGCGTATATTAAAATTGCTGCAGTTAAAGCGTCCGTAGTTGAATTTCGTTCAAATTGAATAGTGCCGAGTTTTCCTCGTGTTAATTCGAGTACTTTTTGATTAGAACTTGGGTGAGATGTACTCGTTTCGACGAGTTCGTTTCGCTCCGTTACTTTGAGCAAATTGGAGTGCTCCAACCAGGCTTAAGCTTGAACAGCTCAGCATGGAATAACAAGATAAGACTTTAGTTCTTCTTGTTGGTGACATGAACTAATAGTAATGGTTGATAAGGACATACGGGGGCATTTGTACTTGCTGGAGAGAGGTGAAATTCTAAGACCCAGCAAAGACAAACAAGTGCGAAAGCATTTGCCCAGTATGTACCTGTTAA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Eugregarinorida,Gregarinidae,Leidyana1,Leidyana1_sp.,100,100,100,100,100,100,100,87,76,7,7 -seq145,97.2,MG775599.1.1357_U,18S_rRNA,nucleus,strain_SF33_R_Tri,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_sp.,AGCTCCAATAGCGTATATTAAAATTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGTATGGCGCACTTGGCCCGCCGAGGTTTCTCGGTCGTGTGCCGGTGTGCGCCTGCCATCCTTCTGGGGAACGGCCTTGCCCTTCACTGGGTGGAGGTCGGTATCCAGATCTTTTACTTTGAAAAAATTAGAGTGTTTAAAGCAGGCTTGCGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTTGGTCTTATTTTGTTGGTTTCTAGGACTAAAGTAATGATTGATAGGGACAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTTGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Cercomonadida,Cercomonadidae,Neocercomonas,Neocercomonas_jutlandica,100,100,100,100,100,100,100,49,21,7,7 -seq146,99.2,AM114805.1.2266_U,18S_rRNA,nucleus,clone_WIM47,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGACTGAAGCATTTTGCCGGCCGCGGTTTCGCGTGTTAGTGCTTGATGCTCTGTCCATCCTTCCAATCGCTATTGGCTATTCTTCATTGAATGGTCAGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,100,100,100,7,7 -seq147,98.4,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAACTTTGGTAGTAATAATGGGTTATCTCTTTGAGTATGTACCTATTGTTACTACCATTATTCCGATCTATACAAGGGGTAATTCCCTTGTGTTCTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTGATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAGCGCCTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAGCAGTCGGGGGTATTCGTATTCAGTCGTTAGAGGTGAAATTCTTAGATTGACTAAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,100,100,100,100,100,100,100,100,100,7,7 -seq148,90.6,EF024854.1.1760_U,18S_rRNA,nucleus,clone_Elev_18S_1374,Eukaryota,Alveolata,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Monocystis,Monocystis_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAACGCTCGTAGTTGAAATTGTGGTAATAACAATGAATCATCTTTAACGAGCATGCACTTATTGTTGTTGCCATTATTCCGGTATATGCAGGGTAACCTGTATATTCGGAACTGTTACTTTGAGAAAATTAGAGTGTTTCAAGCAGGCGTAATGCTTTGAATACTCCAGCATGGAATGACAACAAAGGACTCAAGTCTTTCTTGTTGGTTTAAGGAGTTGAGTAATGATTAAGAGGAACAGTCGGGGGTATTTGTATTCAGTCGCTAGAGGTGAAATTCTTAGATTGACTGAAGACGAACTACTGCGAAGGCATCTACCATGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Gregarinomorphea,Neogregarinorida,Actinocephalidae,Syncystis,Syncystis_mirabilis,100,99,96,96,96,96,96,51,51,7,7 -seq159,98.7,AY124366.1.737_U,18S_rRNA,nucleus,clone_CSE28,Eukaryota,Alveolata,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCAAGGGTCGCATCGGGGACAACCCGCATGCTCCCTACCAGTCTTAGACTGTTACTGTGAGAAAATTAGAGTGTTTCAAGCAGGCTGTTGCAGGAATACATTAGCATGGAATAACGAATGTGTCTAGAATCTTGGTTAATTCTAGATTACGATTAATAGGGACAGTTGGGGGCATTAGTATTTAATTGTCAGAGGTGAAATTCTTGGATTTGTTAAAGACTAACGTATGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Ciliophora,Litostomatea,Litostomatea_X,Litostomatea_XX,Litostomatea_XXX,Litostomatea_XXX_sp.,100,100,100,100,100,98,98,78,78,6,6 -seq161,99.2,MZ558750.1.2453_U,18S_rRNA,nucleus,clone_,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_orientale,AGCTCCAATAGCGTATATTTAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCGGGCGGGGCCTGCCGGTCCGCCTCTGGTGTGCACTGGCATGGGCTCGCCTCGCTGTCGGGGACGGGCTCCTGGGCTTCACTGTCCGGGACTCGGAGTCGACGAGGTTACTTTGAGTAAATTAGAGTGTTCAAAGCAGGCCTACGCTCTGAATACGTTAGCATGGAATAACACGATAGGACTCTGGCCTATCTTGTTGGTCTGTGGGACCGGAGTAATGATTAAGAGGGACGGTCGGGGGCATTCGTATTTCGTTGTCAGAGGTGAAATTCTTGGATTTATGAAAGACGAACAACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,Archaeplastida,Chlorophyta,Chlorophyta_X,Trebouxiophyceae,Watanabea-Clade,Watanabea-Clade_X,Chloroidium,Chloroidium_sp.,100,100,100,100,99,99,99,99,34,6,6 -seq167,93.9,JN635482.1.1281_U,18S_rRNA,nucleus,clone_hay20,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCTGTTAAAAAGCTCGTAGTTGGATTTCTGATATAATATTACTGTCCCGCTGTGGTTACACATGTGAGTGACGGATAATATTTATCTGTTTTGTGTTTTTTGTTTTGATAGTATGTAAATATTATCAAAAACATACACATCTTTTACTTTGAACAAATTAGAGTGTTTCAAGCAGGCATTTATGCCTTGAATACATGAGCATGGAATAATCATCGAGGACTTTTAGTTCTATGTTGATTGGTTCTAGAACTATAGTAATGATGGATAGGGATAGTTGGGGGTGCTAGTATTCCGAGGCCAGAGGTGAAATTCTTGGATTCTCGGAAGACTCACTTAGGCGAAAGCATTCACCAAGGATGTCTTCA,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Glissomonadida_X,Glissomonadida_XX,Glissomonadida_XX_sp.,100,100,100,100,100,100,99,99,99,5,5 -seq173,100.0,EU709208.1.1193_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTGGCCGGCCACGGTTTCGTGTGTTAGTGCTTTGTGTTCTGGCCATCCTTCCAATCGCTGCGTGCTTTTCTTCATTGATTAGTGCGTGGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,63,63,4,4 -seq176,96.0,EF468654.1.1782_U,18S_rRNA,nucleus,strain_HC06-28,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGAAGTATATAATGAATTTCTAATTCAATTGTATATTGATTCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_sp.,100,99,99,96,96,96,96,89,70,4,4 -seq178,96.9,AB757862.1.1584_U,18S_rRNA,nucleus,,Eukaryota,Alveolata,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Isospora,Isospora_sp.,AGCTCCAATAGTGTATATTAGAGTTGTTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGTCGGGGTCTTCGGGTGCCGCCCGTATGGGTGCGCGTCTAGTTAGACCTCGGCTTTCTTCCGGTAGCCTTCCGCGCTTAATTGCGTGTGTTGGTGTTTCGGAACTTTTACTTTGAGAAAAATAGAGTGTTTCAAGCAGGCTTGTCGCCCTGAATACTTCAGCATGGAATAATAAGATAGGACCTTGGTTCTATTTTGTTGGTTTCTAGGACCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCGTATTTAACTGTCAGAGGTGAAATTCTTAGATTTGTTAAAGACGAACTACTGCGAAAGCATTTGCCAAGGATGTTTTCA,Eukaryota,TSAR,Alveolata,Apicomplexa,Coccidiomorphea,Eimeriida,Eimeriidae,Eimeria6,Eimeria6_sp.,100,100,100,100,100,100,100,31,30,4,4 -seq181,98.7,EU709173.71.1479_U,18S_rRNA,nucleus,,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTTTGCCGGCCACGGTTTCGTGTGTTAGTGCTTGATGTTCTGGCCATCCTTCCAATCGCTATCGGCTGCTCTTCATTGAGCGGTCGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandona,Sandona_sp.,100,100,100,100,100,100,99,88,85,3,3 -seq182,98.9,JN635479.1.1258_U,18S_rRNA,nucleus,clone_hay11,Eukaryota,Alveolata,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Colpodida_X,Colpodida_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGTTGCAGTTAAAAAGCTCGTAGTTGAATTTCTGGTCAGCGCTGTTCTTGGCTCTCTGAGTCGGCTCAGTGTTTGGTCATCCGTATGGGAAGCTAGCTCGGCCTTCACTGGTCGGCTAGTGGATCATACACTTTACTTTGAAAAAATTAGAGTGTTTCAGGCAGGCAATTGCTTGGATACTTCAGCATGGAATAATGGAATAGGACTTTGACCTATTTGTTGGTTTCTTCGAGGTCAAGGTAATGATTAATAGGGACAGTTGGGGGCATTCATATTTAATTGTCAGAGGTGAAATTCTTGGATTTATTAAAGATGAACTTATGCGAAAGCATTTGCCAAGGATGTTTTCATTAA,Eukaryota,TSAR,Alveolata,Ciliophora,Colpodea,Colpodea_X,Colpodida,Pseudoplatyophrya,Pseudoplatyophrya_nana,100,100,100,100,100,100,100,49,49,3,3 -seq186,100.0,EF468654.1.1782_U,18S_rRNA,nucleus,strain_HC06-28,Eukaryota,Stramenopiles,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,AGCTCCAATAGCGTATATTAACGTTGTTGCAGTTAAAAAGCTCGTAGTTGAAATGTGGGATGATTGTTATGATGAATCTATGTATTCGATTATAATATGAATCCCCTCTCCAATTTATTATTATGGGTATTAATTTATTTGTAATAAGAATTGGTCATTTACTGTGAGAAAATTAGAGTGTTCAAAGCAGGCATTTGCTTGAATAGATTAGCATGGAATAATAGTAAAAGGCTTTTATGGTGCTTAATTGGTTTAATCATAGAAGCAAGATTAAAAGGAACAGTTGGGGGTATTCATATTCAATAGTTAGAGGTGAAATTCTCGGATTTATGGAAGATGAACTAGTGCGAAAGCATTTACCAAGGATGTTTTCA,Eukaryota,TSAR,Stramenopiles,Bigyra,Opalozoa,Opalinata,Blastocystis-Group,Blastocystis,Blastocystis_hominis,100,98,97,97,97,97,97,95,46,3,3 -seq194,99.5,EU709205.65.1542_U,18S_rRNA,nucleus,clone_,Eukaryota,Rhizaria,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,AGCTCCAATAGCGTATATTAAAGTTGCTGCAGTTAAAAAGCTCGTAGTTGGATTTCTGGCTTGAAACATTAAGCCGGCCACGGTTTCGTGTGTTAGTGCTTGGTGTTCTGGCCATCCTTCCAATCGCTGCTGGCCTTTCTTCATTGATAGGTTGGTAGTATTGGATCTTTTACTTTGAAAAAATTAGAGTGTTTCAAGCAGGCTTTATGCTATGAATACATTAGCATGGAATAATAATTTAGGACTTCGGTTCTATTTTGTTGGTTTCTAGGACTGAAGTAATGATTGATAGGGATAGTTGGGGGTGCTAGTATTCAGCGGCCAGAGGTGAAATTCTCGGATTCGCTGAAGACTAACTTATGCGAAAGCATTCACCAAGGATGTCTTCT,Eukaryota,TSAR,Rhizaria,Cercozoa,Filosa-Sarcomonadea,Glissomonadida,Sandonidae,Sandonidae_X,Sandonidae_X_sp.,100,100,100,100,100,100,100,73,73,2,2 diff --git a/src/dada2.r b/src/dada2.r deleted file mode 100644 index e69de29..0000000 diff --git a/src/main.jl b/src/main.jl index 42ebf9e..a8ee96e 100644 --- a/src/main.jl +++ b/src/main.jl @@ -11,8 +11,13 @@ fastq_input_dir = "./inputs/fastq/" cutadapt_dir = "./cutadapt/" optional_args = "-m 200 --discard-untrimmed" +multiv = "./vsearch/taxonomy_multi_pool.tsv" +vespav = "./vsearch/taxonomy_vespa_pool_fwdonly.tsv" +multid = "./DADA2/tax_counts_fasta_multi_pool.csv" +vespad = "./DADA2/tax_counts_fasta_vespa_pool_fwdonly.csv" + ## Main #cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) -CSV.write("protist_filtered.csv", filter_table(merge_taxonomy_counts(),"./inputs/protist_filter.yml")) \ No newline at end of file +CSV.write("protist_filtered_multi_pool.csv", filter_table(merge_taxonomy_counts(multiv, multid),"./inputs/protist_filter_multiplex.yml")) diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index b968978..a3f6233 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -73,7 +73,7 @@ export merge_taxonomy_counts, filter_table Reads taxonomy TSV from vsearch and counts CSV from DADA2 and returns a merged DataFrame. """ function merge_taxonomy_counts( - taxonomy_vsearch_path::AbstractString = "./vsearch/taxonomy.tsv", # Fixed default paths + taxonomy_vsearch_path::AbstractString = "./vsearch/taxonomy.tsv", counts_csv_path::AbstractString = "./DADA2/tax_counts_fasta.csv") @info("Merging taxonomy counts.") diff --git a/vsearch/taxonomy.tsv b/vsearch/taxonomy.tsv deleted file mode 100644 index 99ab532..0000000 --- a/vsearch/taxonomy.tsv +++ /dev/null @@ -1,159 +0,0 @@ -seq2 AY321149.1.1455_U|18S_rRNA|nucleus|strain_T|Eukaryota|Excavata|Metamonada|Parabasalia|Trichomonadea|Honigbergiellida|Hexamastigidae|Hexamastix|Hexamastix_kirbyi 99.2 243 2 0 1 243 1 1455 -1 0 -seq17 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 92.0 274 16 3 1 269 1 1489 -1 0 -seq22 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 95.3 275 11 2 1 275 1 1489 -1 0 -seq21 AF129862.1.1457_U|16S_rRNA||clone_CA26|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_NK4A214_group|Ruminococcaceae_NK4A214_group_sp. 90.1 374 37 0 1 374 1 1454 -1 0 -seq11 KC790327.1.507_U|18S_rRNA|nucleus|strain_318|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomyces|Saccharomyces_cerevisiae 100.0 379 0 0 1 379 1 507 -1 0 -seq19 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 89.8 374 37 1 1 374 1 1461 -1 0 -seq4 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 1143 -1 0 -seq15 AB453313.1.1235_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Chromadorea|Chromadorea_X|Strongyloides|Strongyloides_myopotami 100.0 350 0 0 1 350 1 1235 -1 0 -seq8 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.2 380 3 0 1 380 1 1745 -1 0 -seq12 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 96.6 386 12 1 1 386 1 1829 -1 0 -seq13 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 96.6 386 12 1 1 386 1 1829 -1 0 -seq6 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 100.0 380 0 0 1 380 1 1745 -1 0 -seq7 AY212772.1.1503_U|16S_rRNA||clone_wet70|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_UCG005|Ruminococcaceae_UCG005_sp. 94.7 374 20 0 1 374 1 1434 -1 0 -seq14 AJ236046.1.1766_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Plantago|Plantago_lanceolata 100.0 380 0 0 1 380 1 1745 -1 0 -seq25 EU091849.1.1430_U|18S_rRNA|nucleus|clone_P2-3m2|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Watanabea-Clade_XX|Watanabea-Clade_XX_sp. 100.0 379 0 0 1 379 1 1430 -1 0 -seq3 GQ352558.1.1816_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_vulpis 96.1 386 14 1 1 386 1 1816 -1 0 -seq1 RDQT01000029.10.1006_U|16S_rRNA||strain_LYJ002|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Bacilli|Bacillales|Staphylococcaceae|Staphylococcus|Staphylococcus_aureus 100.0 387 0 0 1 387 1 997 -1 0 -seq5 AF129862.1.1457_U|16S_rRNA||clone_CA26|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Ruminococcaceae_NK4A214_group|Ruminococcaceae_NK4A214_group_sp. 90.4 374 36 0 1 374 1 1454 -1 0 -seq9 KC876030.20981.22849_U|18S_rRNA|nucleus|clone_LA13_165F6|Eukaryota|Obazoa|Opisthokonta|Metazoa|Craniata|Craniata_X|Mammalia|Homo|Homo_sapiens 100.0 387 0 0 1 387 1 1869 -1 0 -seq20 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 88.5 374 42 1 1 374 1 1461 -1 0 -seq38 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.9 275 21 1 1 275 1 1483 -1 0 -seq18 AY548930.1.1487_U|16S_rRNA||clone_1-1|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Sphingobacteriales|Lentimicrobiaceae|Lentimicrobiaceae_X|Lentimicrobiaceae_X_sp. 84.4 371 58 0 1 371 1 1444 -1 0 -seq43 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.8 273 25 0 1 273 1 1489 -1 0 -seq16 U02969.1.1039_U|18S_rRNA|nucleus|clone_YEpT7R1.0Xho|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomyces|Saccharomyces_cerevisiae 100.0 379 0 0 1 379 1 1039 -1 0 -seq23 HF586907.1.1829_U|18S_rRNA|nucleus|strain_Tm1|Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Enoplea|Enoplea_X|Trichuris|Trichuris_muris 95.6 386 16 1 1 386 1 1829 -1 0 -seq35 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.2 274 20 2 1 271 1 1489 -1 0 -seq24 AY821994.1.1627_U|18S_rRNA|nucleus|clone_CV1_B1_42|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 1627 -1 0 -seq29 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 -seq48 ABGB01001215.11.602_U|18S_rRNA|nucleus|strain_H348|Eukaryota|Obazoa|Opisthokonta|Fungi|Opisthosporidia|Microsporida|Microsporida_X|Enterocytozoon|Enterocytozoon_bieneusi 99.5 204 1 0 1 204 1 592 -1 0 -seq33 AY730662.1.1484_U|16S_rRNA||clone_|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Flavonifractor|Flavonifractor_sp. 97.1 374 11 0 1 374 1 1453 -1 0 -seq42 AY244968.1.1492_U|16S_rRNA||clone_BS14|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Rikenellaceae|Rikenellaceae_RC9_gut_group|Rikenellaceae_RC9_gut_group_sp. 92.4 357 27 0 1 357 1 1446 -1 0 -seq34 AF132250.1.1452_U|16S_rRNA||clone_adhufec236|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Lachnospiraceae|Lachnospiraceae_X|Lachnospiraceae_X_sp. 95.3 361 16 1 1 361 1 1451 -1 0 -seq39 EU091877.1.1445_U|18S_rRNA|nucleus|clone_C1-3m4|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1445 -1 0 -seq30 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 -seq32 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 90.1 374 36 1 1 374 1 1461 -1 0 -seq27 CK163334.1.1076_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Triticum|Triticum_aestivum 100.0 380 0 0 1 380 1 1075 -1 0 -seq10 MT991542.1.2393_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_sp. 99.5 379 2 0 1 379 1 2393 -1 0 -seq26 L24420.1.1764_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Spinacia|Spinacia_oleracea 96.6 380 13 0 1 380 1 1757 -1 0 -seq41 EU154978.1.1268_U|18S_rRNA|nucleus|clone_HC8_BASS|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 1268 -1 0 -seq44 GU972467.1.809_U|18S_rRNA|nucleus|clone_10_1F_18.b1.ab1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 809 -1 0 -seq52 AY244968.1.1492_U|16S_rRNA||clone_BS14|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Rikenellaceae|Rikenellaceae_RC9_gut_group|Rikenellaceae_RC9_gut_group_sp. 93.3 371 25 0 1 371 1 1446 -1 0 -seq31 MZ558750.1.2453_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_orientale 100.0 379 0 0 1 379 1 2453 -1 0 -seq36 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 -seq62 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.4 272 25 1 1 272 1 1483 -1 0 -seq51 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.6 274 22 1 1 274 1 1489 -1 0 -seq63 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.1 274 26 1 1 274 1 1489 -1 0 -seq28 DQ886365.1.1600_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Chamaebatia|Chamaebatia_foliolosa 99.7 379 1 0 1 379 1 1600 -1 0 -seq40 L49287.1.1743_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Lamium|Lamium_amplexicaule 100.0 380 0 0 1 380 1 1743 -1 0 -seq37 FJ820679.1.912_U|18S_rRNA|nucleus|clone_S191|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 912 -1 0 -seq60 NGBS01000218.13865.15598_U|18S_rRNA|nucleus|strain_LA|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Proteromonadidae|Proteromonas|Proteromonas_lacertae 88.5 348 34 4 1 344 1 1734 -1 0 -seq71 KC953859.1.1483_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.2 272 23 1 1 272 1 1483 -1 0 -seq54 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.5 274 25 1 1 274 1 1489 -1 0 -seq46 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 -seq50 EU091868.1.1343_U|18S_rRNA|nucleus|clone_P4-3m8|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1343 -1 0 -seq57 AY212760.1.1508_U|16S_rRNA||clone_wet57|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Lachnospiraceae|Lachnospiraceae_NK4A136_group|Lachnospiraceae_NK4A136_group_sp. 91.4 360 31 0 1 360 1 1439 -1 0 -seq66 DQ459618.1.841_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_carnescens 100.0 380 0 0 1 380 1 841 -1 0 -seq65 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.5 380 2 0 1 380 1 1143 -1 0 -seq68 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 -seq78 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 -seq47 EU091849.1.1430_U|18S_rRNA|nucleus|clone_P2-3m2|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Watanabea-Clade_XX|Watanabea-Clade_XX_sp. 99.5 379 2 0 1 379 1 1430 -1 0 -seq74 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.2 380 3 0 1 380 1 1745 -1 0 -seq72 FJ490216.1.935_U|18S_rRNA|nucleus|clone_C_47|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Cystobasidiomycetes_X|Cystobasidiomycetes_X_sp. 98.7 383 5 0 1 383 1 935 -1 0 -seq87 AF432140.1.1538_U|16S_rRNA||clone_FM046|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Mollicutes_RF39|Mollicutes_RF39_X|Mollicutes_RF39_XX|Mollicutes_RF39_XX_sp. 83.0 218 37 0 1 218 1 1498 -1 0 -seq80 AB032627.1.1781_U|18S_rRNA|nucleus|strain_JCM_8974|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_dimennae 99.2 380 3 0 1 380 1 1781 -1 0 -seq73 MF599080.1.1241_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Nematoda|Chromadorea|Chromadorea_X|Miculenchus|Miculenchus_muscus 81.8 385 59 2 1 377 1 1241 -1 0 -seq64 AANN01653944.1.1862_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Craniata|Craniata_X|Mammalia|Erinaceus|Erinaceus_europaeus 96.1 387 15 0 1 387 1 1862 -1 0 -seq82 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.2 380 3 0 1 380 1 1143 -1 0 -seq59 KF648870.1.1484_U|18S_rRNA|nucleus|strain_MTZ1|Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Isospora|Isospora_sp. 97.4 383 10 0 1 383 1 1484 -1 0 -seq85 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 92.3 274 15 3 1 269 1 1489 -1 0 -seq93 AY321149.1.1455_U|18S_rRNA|nucleus|strain_T|Eukaryota|Excavata|Metamonada|Parabasalia|Trichomonadea|Honigbergiellida|Hexamastigidae|Hexamastix|Hexamastix_kirbyi 97.9 243 5 0 1 243 1 1455 -1 0 -seq49 DQ415278.1.1656_U|18S_rRNA|nucleus|strain_MCA2391|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Pucciniomycetes|Puccinia|Puccinia_hordei 100.0 381 0 0 1 381 1 1656 -1 0 -seq69 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.7 380 1 0 1 380 1 1745 -1 0 -seq83 FJ592431.1.1083_U|18S_rRNA|nucleus|clone_E11_SE4A|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1083 -1 0 -seq75 AJ236055.1.1761_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Veronica|Veronica_anagallis-aquatica 99.5 380 2 0 1 380 1 1737 -1 0 -seq53 FJ355409.1.1048_U|18S_rRNA|nucleus|clone_051102_S1_W_T_SDP18_098|Eukaryota|TSAR|Stramenopiles|Gyrista|Chrysophyceae|Ochromonadales|Ochromonadales_clade-XIII|Ochromonadales_clade-XIII_X|Ochromonadales_clade-XIII_X_sp. 94.2 381 19 2 1 378 1 1048 -1 0 -seq56 FN598332.1.1143_U|18S_rRNA|nucleus|clone_BS12_C1|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1143 -1 0 -seq76 L24420.1.1764_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Spinacia|Spinacia_oleracea 96.1 380 15 0 1 380 1 1757 -1 0 -seq77 JF703099.1.1644_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 99.5 381 1 1 1 380 1 1644 -1 0 -seq90 DQ832220.1.1776_U|18S_rRNA|nucleus|strain_JAG_55|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Sphacelotheca|Sphacelotheca_koordersiana 99.2 382 3 0 1 382 1 1776 -1 0 -seq79 AB085801.1.1782_U|18S_rRNA|nucleus|strain_CBS_6578|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Cryptococcus|Cryptococcus_sp. 100.0 380 0 0 1 380 1 1782 -1 0 -seq70 HQ427487.1.1161_UC|18S_rRNA|nucleus|clone_RU06182008DF1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 100.0 380 0 0 1 380 1 914 -1 0 -seq81 AF071069.1.1760_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Trifolium|Trifolium_repens 100.0 379 0 0 1 379 1 1760 -1 0 -seq91 JF703099.1.1644_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 99.7 381 0 1 1 380 1 1644 -1 0 -seq94 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 -seq88 Y15810.1.1761_U|18S_rRNA|nucleus|strain_CBS_1787|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Kazachstania|Kazachstania_telluris 100.0 381 0 0 1 381 1 1761 -1 0 -seq98 AB695526.1.1759_U|18S_rRNA|nucleus|clone_MPE2-32|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Allapsidae|Allapsidae_X|Allapsidae_X_sp. 100.0 388 0 0 1 388 1 1759 -1 0 -seq96 FJ592322.1.1085_U|18S_rRNA|nucleus|clone_B06_SE1A|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Prasiolales_XX|Prasiolales_XX_sp. 100.0 380 0 0 1 380 1 1085 -1 0 -seq84 D31648.1.1781_U|18S_rRNA|nucleus|strain_JCM_2961|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Dioszegia|Dioszegia_crocea 100.0 380 0 0 1 380 1 1781 -1 0 -seq92 MZ067863.1.738_U|18S_rRNA|nucleus|strain_ACKU_309-4|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Chlorophyceae|Chlamydomonadales|Chlamydomonadales_X|Chlorosarcinopsis|Chlorosarcinopsis_bastropiensis 100.0 379 0 0 1 379 1 738 -1 0 -seq117 AF432140.1.1538_U|16S_rRNA||clone_FM046|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Mollicutes_RF39|Mollicutes_RF39_X|Mollicutes_RF39_XX|Mollicutes_RF39_XX_sp. 78.5 214 46 0 1 214 1 1498 -1 0 -seq102 U40810.1.1747_U|18S_rRNA|nucleus|strain_CBS_7331|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Agaricostilbomycetes|Bensingtonia|Bensingtonia_yuccicola 99.2 388 3 0 1 388 1 1747 -1 0 -seq106 FJ592430.1.1088_U|18S_rRNA|nucleus|clone_E08_SE4A|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 98.9 380 4 0 1 380 1 1088 -1 0 -seq109 AY642700.1.1764_U|18S_rRNA|nucleus|clone_P34.42|Eukaryota|Obazoa|Opisthokonta|Fungi|Rozellomycota|Rozellomycota_X|Rozellomycota_XX|Rozellomycota_XXX|Rozellomycota_XXX_sp. 89.2 371 35 3 1 370 1 1764 -1 0 -seq105 EU091862.1.1441_U|18S_rRNA|nucleus|clone_P4-3m2|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 99.5 380 2 0 1 380 1 1441 -1 0 -seq95 DQ851582.1.1716_U|18S_rRNA|nucleus|strain_J130|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Agaricomycetes|Psathyrella|Psathyrella_gracilis 99.7 381 1 0 1 381 1 1716 -1 0 -seq101 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 -seq112 AY730662.1.1484_U|16S_rRNA||clone_|Bacteria|Terrabacteria|Firmicutes|Firmicutes_X|Clostridia|Clostridiales|Ruminococcaceae|Flavonifractor|Flavonifractor_sp. 97.1 374 11 0 1 374 1 1453 -1 0 -seq113 EU087208.1.865_U|18S_rRNA|nucleus|clone_1_39|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Eugregarinorida|Gregarinidae|Amoebogregarina|Amoebogregarina_sp. 84.5 367 52 3 1 367 1 865 -1 0 -seq108 DQ388549.1.1646_U|18S_rRNA|nucleus||Eukaryota|TSAR|Stramenopiles|Gyrista|Chrysophyceae|Ochromonadales|Ochromonadaceae|Spumella|Spumella_vulgaris 100.0 383 0 0 1 383 1 1646 -1 0 -seq115 KM020169.1.1775_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Chlorophyceae|Chlamydomonadales|Coccomyxaceae|Coccomyxa|Coccomyxa_sp. 100.0 380 0 0 1 380 1 1775 -1 0 -seq107 AY607851.1.1593_U|18S_rRNA|nucleus|specimen_Holz_&_Franzaring_CH_00-80_(NY)|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Dendrohypopterygium|Dendrohypopterygium_arbuscula 100.0 383 0 0 1 383 1 1593 -1 0 -seq104 EF200530.1.1708_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Prasiola|Prasiola_crispa 98.9 380 3 1 1 380 1 1708 -1 0 -seq103 EF024471.1.1761_U|18S_rRNA|nucleus|clone_Elev_18S_1147|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 92.3 376 25 1 1 376 1 1761 -1 0 -seq125 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 90.5 274 25 1 1 274 1 1489 -1 0 -seq100 EU091871.1.1443_U|18S_rRNA|nucleus|clone_P6-3m2|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Microbotryomycetes|Microbotryomycetes_X|Microbotryomycetes_X_sp. 100.0 382 0 0 1 382 1 1443 -1 0 -seq119 DQ886380.1.1647_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Rubus|Rubus_idaeus 100.0 380 0 0 1 380 1 1647 -1 0 -seq116 X74000.1.2286_R|18S_rRNA|nucleus|strain_Andreyeva_748-I|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Chlorellales|Chlorellales_X|Chlorella|Chlorella_mirabilis 100.0 380 0 0 1 380 1 1798 -1 0 -seq118 EU709161.6.1494_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 100.0 389 0 0 1 389 1 1489 -1 0 -seq124 AF206944.1.1735_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Juncus|Juncus_effusus 99.5 380 2 0 1 380 1 1735 -1 0 -seq99 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 84.9 344 50 2 1 344 1 1692 -1 0 -seq126 GU901855.1.1740_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Metazoa|Annelida|Annelida_X|Annelida_XX|Achaeta|Achaeta_bifollicula 99.2 380 3 0 1 380 1 1740 -1 0 -seq137 AY189313.1.1515_U|16S_rRNA||strain_LB-12|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Entomoplasmatales|Spiroplasmataceae|Spiroplasma|Spiroplasma_sp. 82.7 214 37 0 1 214 1 1441 -1 0 -seq135 KC953858.1.1489_U|18S_rRNA|nucleus||Eukaryota|Excavata|Metamonada|Parabasalia|Tritrichomonadea|Tritrichomonadida|Simplicimonadidae|Simplicimonas|Simplicimonas_similis 91.3 275 20 2 1 273 1 1489 -1 0 -seq127 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 90.8 371 29 3 1 367 1 1760 -1 0 -seq120 EF023954.1.1792_U|18S_rRNA|nucleus|clone_Amb_18S_1419|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 99.7 380 1 0 1 380 1 1792 -1 0 -seq132 JN573894.1.1681_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Diplosphaera|Diplosphaera_sp. 100.0 380 0 0 1 380 1 1681 -1 0 -seq129 JQ941712.1.1035_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Pezizomycotina|Laboulbeniomycetes|Hesperomyces|Hesperomyces_virescens 100.0 364 0 0 1 364 1 1035 -1 0 -seq114 EU090188.1.1662_U|18S_rRNA|nucleus|strain_AFC_1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Rhodotorula|Rhodotorula_yarrowii 98.7 382 5 0 1 382 1 1662 -1 0 -seq144 EF586127.1.796_U|18S_rRNA|nucleus|clone_121-O29|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Eugregarinorida|Gregarinidae|Leidyana1|Leidyana1_sp. 98.0 306 6 0 1 380 1 796 -1 0 -seq134 JQ993661.1.1396_U|18S_rRNA|nucleus||Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Eimeria6|Eimeria6_sp. 99.2 383 3 0 1 383 1 1396 -1 0 -seq133 FR693368.1.1633_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Apatococcus|Apatococcus_lobatus 100.0 380 0 0 1 380 1 1633 -1 0 -seq130 AY884326.1.1078_U|18S_rRNA|nucleus|strain_CeS-2|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Cercomonadida|Cercomonadidae|Cercomonas|Cercomonas_sp. 100.0 390 0 0 1 390 1 1078 -1 0 -seq131 AJ579340.1.1732_U|18S_rRNA|nucleus|strain_SAG_30.83|Eukaryota|TSAR|Stramenopiles|Gyrista|Xanthophyceae|Xanthophyceae_X|Xanthophyceae_XX|Botrydiopsis|Botrydiopsis_callosa 100.0 396 0 0 1 396 1 1732 -1 0 -seq110 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 87.8 344 39 2 1 343 1 1692 -1 0 -seq140 AB032621.1.1783_UC|18S_rRNA|nucleus|strain_JCM_1775|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Agaricomycetes|Mrakiella|Mrakiella_aquatica 99.5 380 2 0 1 380 1 1513 -1 0 -seq147 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 98.4 372 4 1 1 372 1 1760 -1 0 -seq154 AF311292.1.1439_U|16S_rRNA||strain_WH_8018|Bacteria|CMS|Cyanobacteria|Cyanobacteria_X|Cyanophyceae|Synechococcales|Cyanobiaceae|Synechococcus_CC9902|Synechococcus_CC9902_sp. 77.2 219 49 1 1 218 1 1406 -1 0 -seq145 MG775599.1.1357_U|18S_rRNA|nucleus|strain_SF33_R_Tri|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Cercomonadida|Cercomonadidae|Neocercomonas|Neocercomonas_sp. 97.2 389 11 0 1 389 1 1357 -1 0 -seq143 EF023318.1.1626_U|18S_rRNA|nucleus|clone_Amb_18S_635|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Embryophyceae_XXX|Embryophyceae_XXX_sp. 100.0 380 0 0 1 380 1 1626 -1 0 -seq141 AY919792.1.1726_U|18S_rRNA|nucleus|clone_LG32-03|Eukaryota|TSAR|Alveolata|Chrompodellids|Colpodellidea|Colpodellida|Colpodellida_CHR1|Colpodellida_CHR1_X|Colpodellida_CHR1_X_sp. 76.9 381 65 5 1 359 1 1726 -1 0 -seq148 EF024854.1.1760_U|18S_rRNA|nucleus|clone_Elev_18S_1374|Eukaryota|TSAR|Alveolata|Apicomplexa|Gregarinomorphea|Neogregarinorida|Actinocephalidae|Monocystis|Monocystis_sp. 90.6 371 30 3 1 367 1 1760 -1 0 -seq121 OM987996.1.1692_U|18S_rRNA|nucleus|strain_PHEM1|Eukaryota|Excavata|Metamonada|Fornicata|Fornicata_X|Fornicata_XX|Caviomonadidae|Iotanema|Iotanema_spirale 88.0 343 39 2 1 342 1 1692 -1 0 -seq146 AM114805.1.2266_U|18S_rRNA|nucleus|clone_WIM47|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 99.2 389 3 0 1 389 1 2266 -1 0 -seq142 AJ431572.1.1716_U|18S_rRNA|nucleus|strain_SAG_35.83|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Prasiolales|Prasiolales_X|Desmococcus|Desmococcus_olivaceus 99.0 381 3 1 1 381 1 1716 -1 0 -seq159 AY124366.1.737_U|18S_rRNA|nucleus|clone_CSE28|Eukaryota|TSAR|Alveolata|Ciliophora|Litostomatea|Litostomatea_X|Litostomatea_XX|Litostomatea_XXX|Litostomatea_XXX_sp. 98.7 310 4 0 1 310 1 737 -1 0 -seq157 HG796065.1.800_U|18S_rRNA|nucleus|clone_A1_18_43|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomycetales_X|Saccharomycetales_X_sp. 100.0 379 0 0 1 379 1 800 -1 0 -seq164 AJ400267.1.1500_U|16S_rRNA||clone_F8|Bacteria|FCB|Bacteroidetes|Bacteroidetes_X|Bacteroidia|Bacteroidales|Muribaculaceae|Muribaculaceae_X|Muribaculaceae_X_sp. 91.4 374 31 1 1 374 1 1461 -1 0 -seq171 AY189313.1.1515_U|16S_rRNA||strain_LB-12|Bacteria|Terrabacteria|Tenericutes|Tenericutes_X|Mollicutes|Entomoplasmatales|Spiroplasmataceae|Spiroplasma|Spiroplasma_sp. 82.7 214 37 0 1 214 1 1441 -1 0 -seq158 HQ865510.1.886_U|18S_rRNA|nucleus|clone_SGSF770|Eukaryota|Obazoa|Opisthokonta|Fungi|Neocallimastigomycota|Neocallimastigomycota_X|Neocallimastigomycota_XX|Neocallimastigomycota_XXX|Neocallimastigomycota_XXX_sp. 87.7 390 39 3 1 384 1 886 -1 0 -seq168 AF206852.1.1739_U|18S_rRNA|nucleus||Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Apium|Apium_graveolens 98.2 380 7 0 1 380 1 1739 -1 0 -seq167 JN635482.1.1281_U|18S_rRNA|nucleus|clone_hay20|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Glissomonadida_X|Glissomonadida_XX|Glissomonadida_XX_sp. 93.9 393 21 1 1 393 1 1281 -1 0 -seq160 AF168844.1.1745_U|18S_rRNA|nucleus|specimen_Hahn_7227,_WIS|Eukaryota|Archaeplastida|Streptophyta|Streptophyta_X|Embryophyceae|Embryophyceae_X|Embryophyceae_XX|Festuca|Festuca_rubra 99.5 380 2 0 1 380 1 1745 -1 0 -seq166 GU824996.1.1063_U|18S_rRNA|nucleus|clone_BCB5F13RM2C08|Eukaryota|Obazoa|Opisthokonta|Fungi|Ascomycota|Saccharomycotina|Saccharomycetales|Saccharomycetales_X|Saccharomycetales_X_sp. 100.0 373 0 0 1 373 1 1063 -1 0 -seq173 EU709208.1.1193_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandonidae_X|Sandonidae_X_sp. 100.0 389 0 0 1 389 1 1193 -1 0 -seq174 JQ014042.1.996_U|18S_rRNA|nucleus|strain_FSU_9823|Eukaryota|Obazoa|Opisthokonta|Fungi|Mucoromycota|Mortierellaceae|Mortierellaceae_X|Mortierella|Mortierella_hyalina 99.7 380 1 0 1 380 1 996 -1 0 -seq176 EF468654.1.1782_U|18S_rRNA|nucleus|strain_HC06-28|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Blastocystis-Group|Blastocystis|Blastocystis_hominis 96.0 374 14 1 1 373 1 1782 -1 0 -seq175 EU090188.1.1662_U|18S_rRNA|nucleus|strain_AFC_1|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Rhodotorula|Rhodotorula_yarrowii 99.2 382 3 0 1 382 1 1662 -1 0 -seq177 MG076806.1.532_U|18S_rRNA|nucleus|strain_AGC1-24|Eukaryota|Obazoa|Opisthokonta|Fungi|Mucoromycota|Glomeromycotina|Paraglomus_family|Paraglomus|Paraglomus_sp. 99.7 379 1 0 1 379 1 532 -1 0 -seq178 AB757862.1.1584_U|18S_rRNA|nucleus||Eukaryota|TSAR|Alveolata|Apicomplexa|Coccidiomorphea|Eimeriida|Eimeriidae|Isospora|Isospora_sp. 96.9 383 12 0 1 383 1 1584 -1 0 -seq181 EU709173.71.1479_U|18S_rRNA|nucleus||Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandona|Sandona_sp. 98.7 389 5 0 1 389 1 1409 -1 0 -seq161 MZ558750.1.2453_U|18S_rRNA|nucleus|clone_|Eukaryota|Archaeplastida|Chlorophyta|Chlorophyta_X|Trebouxiophyceae|Watanabea-Clade|Watanabea-Clade_X|Chloroidium|Chloroidium_orientale 99.2 379 3 0 1 379 1 2453 -1 0 -seq184 AB055193.1.1761_U|18S_rRNA|nucleus|strain_CBS8732|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Occultifur|Occultifur_externus 99.7 383 1 0 1 383 1 1761 -1 0 -seq182 JN635479.1.1258_U|18S_rRNA|nucleus|clone_hay11|Eukaryota|TSAR|Alveolata|Ciliophora|Colpodea|Colpodea_X|Colpodida|Colpodida_X|Colpodida_X_sp. 98.9 378 4 0 1 378 1 1258 -1 0 -seq186 EF468654.1.1782_U|18S_rRNA|nucleus|strain_HC06-28|Eukaryota|TSAR|Stramenopiles|Bigyra|Opalozoa|Opalinata|Blastocystis-Group|Blastocystis|Blastocystis_hominis 100.0 374 0 0 1 374 1 1782 -1 0 -seq185 MG589001.1.1014_U|18S_rRNA|nucleus|strain_HU9268|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Pucciniomycotina|Cystobasidiomycetes|Sporobolomyces|Sporobolomyces_sp. 100.0 382 0 0 1 382 1 1014 -1 0 -seq183 DQ536473.1.3565_G|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Chytridiomycota|Chytridiales|Chytridiaceae|Phlyctochytrium|Phlyctochytrium_planicorne 84.3 381 55 4 1 377 1 3565 -1 0 -seq194 EU709205.65.1542_U|18S_rRNA|nucleus|clone_|Eukaryota|TSAR|Rhizaria|Cercozoa|Filosa-Sarcomonadea|Glissomonadida|Sandonidae|Sandonidae_X|Sandonidae_X_sp. 99.5 389 2 0 1 389 1 1478 -1 0 -seq196 AY821994.1.1627_U|18S_rRNA|nucleus|clone_CV1_B1_42|Eukaryota|Obazoa|Opisthokonta|Fungi|Basidiomycota|Agaricomycotina|Tremellomycetes|Tremellomycetes_X|Tremellomycetes_X_sp. 95.8 380 16 0 1 380 1 1627 -1 0 -seq195 HQ888716.1.1022_U|18S_rRNA|nucleus||Eukaryota|Obazoa|Opisthokonta|Fungi|Blastocladiomycota|Blastocladiomycotina|Blastocladiomycetes|Physoderma|Physoderma_lycopi 99.0 388 3 1 1 388 1 1022 -1 0 From fed106ba0a314ee3a90ce7b88789ad23d913b19d Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 20:15:44 +0100 Subject: [PATCH 007/175] Added dada2 R module. --- .gitignore | 6 +- Manifest.toml | 1621 +++++++++++++++++++++++++++++++++++++++++ Project.toml | 19 + pipelinesteps.txt | 17 + src/dada2.r | 274 +++++++ src/dada2_functions.r | 387 ++++++++++ src/main.jl | 12 +- src/run_cutadapt.jl | 8 +- 8 files changed, 2330 insertions(+), 14 deletions(-) create mode 100644 Manifest.toml create mode 100644 Project.toml create mode 100644 src/dada2.r create mode 100644 src/dada2_functions.r diff --git a/.gitignore b/.gitignore index 2e3069e..8b50f65 100644 --- a/.gitignore +++ b/.gitignore @@ -18,11 +18,7 @@ deps/src/ docs/build/ docs/site/ -# File generated by Pkg, the package manager, based on a corresponding Project.toml -# It records a fixed state of all packages used by the project. As such, it should not be -# committed for packages, but should be committed for applications that require a static -# environment. -Manifest.toml +# Manifest.toml is committed for reproducibility (this is an application, not a library) ### Python ### # Byte-compiled / optimized / DLL files diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..fa79302 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,1621 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.5" +manifest_format = "2.0" +project_hash = "6b1ec8272fc2c5461cd314f6512513860a0b0719" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" +weakdeps = ["ChainRulesCore", "Test"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + +[[deps.AbstractTrees]] +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.5" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "7e35fca2bdfba44d797c53dfe63a51fabf39bfc0" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.4.0" +weakdeps = ["SparseArrays", "StaticArrays"] + + [deps.Adapt.extensions] + AdaptSparseArraysExt = "SparseArrays" + AdaptStaticArraysExt = "StaticArrays" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Arpack]] +deps = ["Arpack_jll", "Libdl", "LinearAlgebra", "Logging"] +git-tree-sha1 = "9b9b347613394885fd1c8c7729bfc60528faa436" +uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" +version = "0.5.4" + +[[deps.Arpack_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "libblastrampoline_jll"] +git-tree-sha1 = "7f54761502ff149a9d492e4acefe9805898e29b3" +uuid = "68821587-b530-5797-8361-c406ea357684" +version = "3.5.2+0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.AxisAlgorithms]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] +git-tree-sha1 = "01b8ccb13d68535d73d2b0c23e39bd23155fb712" +uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" +version = "1.1.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1b96ea4a01afe0ea4090c5c8039690672dd13f2e" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.9+0" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "8d8e0b0f350b8e1c91420b5e64e5de774c2f0f4d" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.16" + +[[deps.Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "a21c5464519504e41e0cbc91f0188e8ca23d7440" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.18.5+1" + +[[deps.CategoricalArrays]] +deps = ["Compat", "DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"] +git-tree-sha1 = "73acb4ed51b1855e1b5ce5c610334363a98d13f1" +uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" +version = "1.0.2" + + [deps.CategoricalArrays.extensions] + CategoricalArraysArrowExt = "Arrow" + CategoricalArraysJSONExt = "JSON" + CategoricalArraysRecipesBaseExt = "RecipesBase" + CategoricalArraysSentinelArraysExt = "SentinelArrays" + CategoricalArraysStatsBaseExt = "StatsBase" + CategoricalArraysStructTypesExt = "StructTypes" + + [deps.CategoricalArrays.weakdeps] + Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" + JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" + SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" + StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" + StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra"] +git-tree-sha1 = "e4c6a16e77171a5f5e25e9646617ab1c276c5607" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.26.0" +weakdeps = ["SparseArrays"] + + [deps.ChainRulesCore.extensions] + ChainRulesCoreSparseArraysExt = "SparseArrays" + +[[deps.Clustering]] +deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"] +git-tree-sha1 = "3e22db924e2945282e70c33b75d4dde8bfa44c94" +uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" +version = "0.15.8" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.8" + +[[deps.ColorSchemes]] +deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] +git-tree-sha1 = "b0fd3f56fa442f81e0a47815c92245acfaaa4e34" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.31.0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "67e11ee83a43eb71ddc950302c53bf33f0690dfe" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.12.1" +weakdeps = ["StyledStrings"] + + [deps.ColorTypes.extensions] + StyledStringsExt = "StyledStrings" + +[[deps.ColorVectorSpace]] +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] +git-tree-sha1 = "8b3b6f87ce8f65a2b4f857528fd8d70086cd72b1" +uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" +version = "0.11.0" +weakdeps = ["SpecialFunctions"] + + [deps.ColorVectorSpace.extensions] + SpecialFunctionsExt = "SpecialFunctions" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "37ea44092930b1811e666c3bc38065d7d87fcc74" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.13.1" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.5.0" + +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "8f06b0cfa4c514c7b9546756dbae91fcfbc92dc9" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.10.3" + +[[deps.Contour]] +git-tree-sha1 = "439e35b0b36e2e5881738abc8857bd92ad6ff9a8" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.6.3" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "d8928e9169ff76c6281f39a659f9bca3a573f24c" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.8.1" + +[[deps.DataStructures]] +deps = ["OrderedCollections"] +git-tree-sha1 = "e357641bb3e0638d353c4b29ea0e40ea644066a6" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.19.3" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.Dbus_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "473e9afc9cf30814eb67ffa5f2db7df82c3ad9fd" +uuid = "ee1fde0b-3d02-5ea6-8484-8dfef6360eab" +version = "1.16.2+0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" +weakdeps = ["ChainRulesCore", "SparseArrays"] + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.Distributions]] +deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "fbcc7610f6d8348428f722ecbe0e6cfe22e672c6" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.123" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.EpollShim_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8a4be429317c42cfae6a7fc03c31bad1970c310d" +uuid = "2702e6a9-849d-5ed8-8c21-79e8b8f9ee43" +version = "0.0.20230411+1" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.11" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "27af30de8b5445644e8ffe3bcb0d72049c089cf1" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.7.3+0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FFMPEG]] +deps = ["FFMPEG_jll"] +git-tree-sha1 = "95ecf07c2eea562b5adbd0696af6db62c0f52560" +uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +version = "0.4.5" + +[[deps.FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "01ba9d15e9eae375dc1eb9589df76b3572acd3f2" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "8.0.1+0" + +[[deps.FFTA]] +deps = ["AbstractFFTs", "DocStringExtensions", "LinearAlgebra", "MuladdMacro", "Primes", "Random", "Reexport"] +git-tree-sha1 = "65e55303b72f4a567a51b174dd2c47496efeb95a" +uuid = "b86e33f2-c0db-4aa1-a6e0-ab43e668529e" +version = "0.3.1" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.24" +weakdeps = ["Mmap", "Test"] + + [deps.FilePathsBase.extensions] + FilePathsBaseMmapExt = "Mmap" + FilePathsBaseTestExt = "Test" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.16.0" +weakdeps = ["PDMats", "SparseArrays", "StaticArrays", "Statistics"] + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStaticArraysExt = "StaticArrays" + FillArraysStatisticsExt = "Statistics" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.5" + +[[deps.Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"] +git-tree-sha1 = "f85dac9a96a01087df6e3a749840015a0ca3817d" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.17.1+0" + +[[deps.Format]] +git-tree-sha1 = "9c68794ef81b08086aeb32eeaf33531668d5f5fc" +uuid = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8" +version = "1.3.7" + +[[deps.FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "2c5512e11c791d1baed2049c5652441b28fc6a31" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.13.4+0" + +[[deps.FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7a214fdac5ed5f59a22c2d9a885a16da1c74bbc7" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.17+0" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GLFW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll", "libdecor_jll", "xkbcommon_jll"] +git-tree-sha1 = "b7bfd56fa66616138dfe5237da4dc13bbd83c67f" +uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" +version = "3.4.1+0" + +[[deps.GR]] +deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Preferences", "Printf", "Qt6Wayland_jll", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "p7zip_jll"] +git-tree-sha1 = "ee0585b62671ce88e48d3409733230b401c9775c" +uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" +version = "0.73.22" + + [deps.GR.extensions] + IJuliaExt = "IJulia" + + [deps.GR.weakdeps] + IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" + +[[deps.GR_jll]] +deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "FreeType2_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt6Base_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "7dd7173f7129a1b6f84e0f03e0890cd1189b0659" +uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" +version = "0.73.22+0" + +[[deps.GettextRuntime_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll"] +git-tree-sha1 = "45288942190db7c5f760f59c04495064eedf9340" +uuid = "b0724c58-0f36-5564-988d-3bb0596ebc4a" +version = "0.22.4+0" + +[[deps.Ghostscript_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Zlib_jll"] +git-tree-sha1 = "38044a04637976140074d0b0621c1edf0eb531fd" +uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" +version = "9.55.1+0" + +[[deps.Glib_jll]] +deps = ["Artifacts", "GettextRuntime_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "24f6def62397474a297bfcec22384101609142ed" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.86.3+0" + +[[deps.Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8a6dbda1fd736d60cc477d99f2e7a042acfa46e8" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.15+0" + +[[deps.Grisu]] +git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" +uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" +version = "1.0.2" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.19" + +[[deps.HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll"] +git-tree-sha1 = "f923f9a774fcf3f5cb761bfa43aeadd689714813" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "8.5.1+0" + +[[deps.HypergeometricFunctions]] +deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.28" + +[[deps.InlineStrings]] +git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.5" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.IntegerMathUtils]] +git-tree-sha1 = "4c1acff2dc6b6967e7e750633c50bc3b8d83e617" +uuid = "18e54dd8-cb9d-406c-a71d-865a43cbb235" +version = "0.1.3" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.Interpolations]] +deps = ["Adapt", "AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] +git-tree-sha1 = "65d505fa4c0d7072990d659ef3fc086eb6da8208" +uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" +version = "0.16.2" + + [deps.Interpolations.extensions] + InterpolationsForwardDiffExt = "ForwardDiff" + InterpolationsUnitfulExt = "Unitful" + + [deps.Interpolations.weakdeps] + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.InvertedIndices]] +git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.1" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLFzf]] +deps = ["REPL", "Random", "fzf_jll"] +git-tree-sha1 = "82f7acdc599b65e0f8ccd270ffa1467c21cb647b" +uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" +version = "0.1.11" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.4.0" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b6893345fd6658c8e475d40155789f4860ac3b21" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "3.1.4+0" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.KernelDensity]] +deps = ["Distributions", "DocStringExtensions", "FFTA", "Interpolations", "StatsBase"] +git-tree-sha1 = "4260cfc991b8885bf747801fb60dd4503250e478" +uuid = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b" +version = "0.6.11" + +[[deps.LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "059aabebaa7c82ccb853dd4a0ee9d17796f7e1bc" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.3+0" + +[[deps.LERC_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "aaafe88dccbd957a8d82f7d05be9b69172e0cee3" +uuid = "88015f11-f218-50d7-93a8-a6af411a945d" +version = "4.0.1+0" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "eb62a3deb62fc6d8822c0c4bef73e4412419c5d8" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.8+0" + +[[deps.LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1c602b1127f4751facb671441ca72715cc95938a" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.3+0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.Latexify]] +deps = ["Format", "Ghostscript_jll", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Requires"] +git-tree-sha1 = "44f93c47f9cd6c7e431f2f2091fcba8f01cd7e8f" +uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" +version = "0.16.10" + + [deps.Latexify.extensions] + DataFramesExt = "DataFrames" + SparseArraysExt = "SparseArrays" + SymEngineExt = "SymEngine" + TectonicExt = "tectonic_jll" + + [deps.Latexify.weakdeps] + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + SymEngine = "123dc426-2d89-5057-bbad-38513e3affd8" + tectonic_jll = "d7dd28d6-a5e6-559c-9131-7eb760cdacc5" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "c8da7e6a91781c41a863611c7e966098d783c57a" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.4.7+0" + +[[deps.Libglvnd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll", "Xorg_libXext_jll"] +git-tree-sha1 = "d36c21b9e7c172a44a10484125024495e2625ac0" +uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" +version = "1.7.1+1" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.18.0+0" + +[[deps.Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "97bbca976196f2a1eb9607131cb108c69ec3f8a6" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.41.3+0" + +[[deps.Libtiff_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "f04133fe05eff1667d2054c53d59f9122383fe05" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.7.2+0" + +[[deps.Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "d0205286d9eceadc518742860bf23f703779a3d6" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.41.3+0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.12.0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.2.0" + +[[deps.MacroTools]] +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.16" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ff69a2b1330bcb730b9ac1ab7dd680176f5896b8" +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.1010+0" + +[[deps.Measures]] +git-tree-sha1 = "b513cedd20d9c914783d8ad83d08120702bf2c77" +uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e" +version = "0.3.3" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.Mocking]] +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "2c140d60d7cb82badf06d8783800d0bcd1a7daa2" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.8.1" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2025.11.4" + +[[deps.MuladdMacro]] +git-tree-sha1 = "cac9cc5499c25554cba55cd3c30543cff5ca4fab" +uuid = "46d2c3a1-f734-5fdb-9937-b9b9aeba4221" +version = "0.2.4" + +[[deps.MultivariateStats]] +deps = ["Arpack", "Distributions", "LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI", "StatsBase"] +git-tree-sha1 = "816620e3aac93e5b5359e4fdaf23ca4525b00ddf" +uuid = "6f286f6a-111f-5878-ab1e-185364afe411" +version = "0.10.3" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "9b8215b1ee9e78a293f99797cd31375471b2bcae" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.1.3" + +[[deps.NearestNeighbors]] +deps = ["AbstractTrees", "Distances", "StaticArrays"] +git-tree-sha1 = "e2c3bba08dd6dedfe17a17889131b885b8c082f0" +uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" +version = "0.4.27" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.Observables]] +git-tree-sha1 = "7438a59546cf62428fc9d1bc94729146d37a7225" +uuid = "510215fc-4207-5dde-b226-833fc4488ee2" +version = "0.5.5" + +[[deps.OffsetArrays]] +git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.17.0" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b6aa4566bb7ae78498a5e68943863fa8b5231b59" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.6+0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.29+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.7+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "1d1aaa7d449b58415f97d2839c318b70ffb525a0" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.6.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.4+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e2bb57a313a74b8104064b7efd01406c0a50d2ff" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.6.1+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.44.0+1" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.37" +weakdeps = ["StatsBase"] + + [deps.PDMats.extensions] + StatsBaseExt = "StatsBase" + +[[deps.Pango_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0662b083e11420952f2e62e17eddae7fc07d5997" +uuid = "36c8627f-9965-5494-a995-c6b170f724f3" +version = "1.57.0+0" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.3" + +[[deps.Pixman_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] +git-tree-sha1 = "db76b1ecd5e9715f3d043cec13b2ec93ce015d53" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.44.2+0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PlotThemes]] +deps = ["PlotUtils", "Statistics"] +git-tree-sha1 = "41031ef3a1be6f5bbbf3e8073f210556daeae5ca" +uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a" +version = "3.3.0" + +[[deps.PlotUtils]] +deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "StableRNGs", "Statistics"] +git-tree-sha1 = "26ca162858917496748aad52bb5d3be4d26a228a" +uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" +version = "1.4.4" + +[[deps.Plots]] +deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "TOML", "UUIDs", "UnicodeFun", "Unzip"] +git-tree-sha1 = "1cc8ad0762e59e713ee3ef28f9b78b2c9f4ca078" +uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +version = "1.41.5" + + [deps.Plots.extensions] + FileIOExt = "FileIO" + GeometryBasicsExt = "GeometryBasics" + IJuliaExt = "IJulia" + ImageInTerminalExt = "ImageInTerminal" + UnitfulExt = "Unitful" + + [deps.Plots.weakdeps] + FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" + GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326" + IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" + ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "07a921781cab75691315adc645096ed5e370cb77" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.3.3" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.1" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "211530a7dc76ab59087f4d4d1fc3f086fbe87594" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "3.2.3" + + [deps.PrettyTables.extensions] + PrettyTablesTypstryExt = "Typstry" + + [deps.PrettyTables.weakdeps] + Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e" + +[[deps.Primes]] +deps = ["IntegerMathUtils"] +git-tree-sha1 = "25cdd1d20cd005b52fc12cb6be3f75faaf59bb9b" +uuid = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae" +version = "0.5.7" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.PtrArrays]] +git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.3.0" + +[[deps.Qt6Base_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Vulkan_Loader_jll", "Xorg_libSM_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_cursor_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "libinput_jll", "xkbcommon_jll"] +git-tree-sha1 = "34f7e5d2861083ec7596af8b8c092531facf2192" +uuid = "c0090381-4147-56d7-9ebc-da0b1113ec56" +version = "6.8.2+2" + +[[deps.Qt6Declarative_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll", "Qt6ShaderTools_jll"] +git-tree-sha1 = "da7adf145cce0d44e892626e647f9dcbe9cb3e10" +uuid = "629bc702-f1f5-5709-abd5-49b8460ea067" +version = "6.8.2+1" + +[[deps.Qt6ShaderTools_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll"] +git-tree-sha1 = "9eca9fc3fe515d619ce004c83c31ffd3f85c7ccf" +uuid = "ce943373-25bb-56aa-8eca-768745ed7b5a" +version = "6.8.2+1" + +[[deps.Qt6Wayland_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll", "Qt6Declarative_jll"] +git-tree-sha1 = "8f528b0851b5b7025032818eb5abbeb8a736f853" +uuid = "e99dba38-086e-5de3-a5b1-6e4c66e897c3" +version = "6.8.2+2" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.11.2" + + [deps.QuadGK.extensions] + QuadGKEnzymeExt = "Enzyme" + + [deps.QuadGK.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + +[[deps.RCall]] +deps = ["CategoricalArrays", "Conda", "DataFrames", "DataStructures", "Dates", "Libdl", "Preferences", "REPL", "Random", "StatsModels", "WinReg"] +git-tree-sha1 = "0ea46f30de5b17d7bd8eaaadb431b0a9ae494a48" +uuid = "6f49c342-dc21-5d91-9882-a32aef131414" +version = "0.14.12" + + [deps.RCall.extensions] + RCallAxisArraysExt = ["AxisArrays"] + + [deps.RCall.weakdeps] + AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" + +[[deps.REPL]] +deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Ratios]] +deps = ["Requires"] +git-tree-sha1 = "1342a47bf3260ee108163042310d26f2be5ec90b" +uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" +version = "0.4.5" +weakdeps = ["FixedPointNumbers"] + + [deps.Ratios.extensions] + RatiosFixedPointNumbersExt = "FixedPointNumbers" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.RecipesPipeline]] +deps = ["Dates", "NaNMath", "PlotUtils", "PrecompileTools", "RecipesBase"] +git-tree-sha1 = "45cf9fd0ca5839d06ef333c8201714e888486342" +uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" +version = "0.6.12" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "1.0.1" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.9.0" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.5.1+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "ebe7e59b37c400f694f52b58c93d26201387da70" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.9" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" +version = "1.11.0" + +[[deps.ShiftedArrays]] +git-tree-sha1 = "503688b59397b3307443af35cd953a13e8005c16" +uuid = "1277b4bf-5013-50f5-be3d-901d8477a67a" +version = "2.0.0" + +[[deps.Showoff]] +deps = ["Dates", "Grisu"] +git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" +uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" +version = "1.0.3" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.2" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.12.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "5acc6a41b3082920f79ca3c759acbcecf18a8d78" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.7.1" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.StableRNGs]] +deps = ["Random"] +git-tree-sha1 = "4f96c596b8c8258cc7d3b19797854d368f243ddc" +uuid = "860ef19b-820b-49d6-a774-d7a799459cd3" +version = "1.0.4" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "eee1b9ad8b29ef0d936e3ec9838c7ec089620308" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.16" +weakdeps = ["ChainRulesCore", "Statistics"] + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.4" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.8.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "aceda6f4e598d331548e04cc6b2124a6148138e3" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.10" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.5.2" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StatsModels]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsAPI", "StatsBase", "StatsFuns", "Tables"] +git-tree-sha1 = "08786db4a1346d17d0a8d952d2e66fd00fa18192" +uuid = "3eaba693-59b7-5ba5-a881-562e759f1c8d" +version = "0.7.9" + +[[deps.StatsPlots]] +deps = ["AbstractFFTs", "Clustering", "DataStructures", "Distributions", "Interpolations", "KernelDensity", "LinearAlgebra", "MultivariateStats", "NaNMath", "Observables", "Plots", "RecipesBase", "RecipesPipeline", "Reexport", "StatsBase", "TableOperations", "Tables", "Widgets"] +git-tree-sha1 = "88cf3587711d9ad0a55722d339a013c4c56c5bbc" +uuid = "f3b207a7-027a-5e70-b257-86293d7955fd" +version = "0.15.8" + +[[deps.StringEncodings]] +deps = ["Libiconv_jll"] +git-tree-sha1 = "b765e46ba27ecf6b44faf70df40c57aa3a547dcb" +uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" +version = "0.3.7" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.2" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "28145feabf717c5d65c1d5e09747ee7b1ff3ed13" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.6.3" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.8.3+2" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TZJData]] +deps = ["Artifacts"] +git-tree-sha1 = "72df96b3a595b7aab1e101eb07d2a435963a97e2" +uuid = "dc5dba14-91b3-4cab-a142-028a31da12f7" +version = "1.5.0+2025b" + +[[deps.TableOperations]] +deps = ["SentinelArrays", "Tables", "Test"] +git-tree-sha1 = "e383c87cf2a1dc41fa30c093b2a19877c83e1bc1" +uuid = "ab02a1b2-a7df-11e8-156e-fb1833f50b87" +version = "1.2.0" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TensorCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" +uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" +version = "0.1.1" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimeZones]] +deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"] +git-tree-sha1 = "d422301b2a1e294e3e4214061e44f338cafe18a2" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "1.22.2" +weakdeps = ["RecipesBase"] + + [deps.TimeZones.extensions] + TimeZonesRecipesBaseExt = "RecipesBase" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.6.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UnicodeFun]] +deps = ["REPL"] +git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" +uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" +version = "0.4.1" + +[[deps.Unzip]] +git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78" +uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d" +version = "0.2.0" + +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + +[[deps.Vulkan_Loader_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Wayland_jll", "Xorg_libX11_jll", "Xorg_libXrandr_jll", "xkbcommon_jll"] +git-tree-sha1 = "2f0486047a07670caad3a81a075d2e518acc5c59" +uuid = "a44049a8-05dd-5a78-86c9-5fde0876e88c" +version = "1.3.243+0" + +[[deps.Wayland_jll]] +deps = ["Artifacts", "EpollShim_jll", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll"] +git-tree-sha1 = "96478df35bbc2f3e1e791bc7a3d0eeee559e60e9" +uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" +version = "1.24.0+0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.Widgets]] +deps = ["Colors", "Dates", "Observables", "OrderedCollections"] +git-tree-sha1 = "e9aeb174f95385de31e70bd15fa066a505ea82b9" +uuid = "cc8bc4a8-27d6-5769-a93b-9d913e69aa62" +version = "0.6.7" + +[[deps.WinReg]] +git-tree-sha1 = "cd910906b099402bcc50b3eafa9634244e5ec83b" +uuid = "1b915085-20d7-51cf-bf83-8f477d6f5128" +version = "1.0.0" + +[[deps.WoodburyMatrices]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "248a7031b3da79a127f14e5dc5f417e26f9f6db7" +uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" +version = "1.1.0" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "9cce64c0fdd1960b597ba7ecda2950b5ed957438" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.8.2+0" + +[[deps.Xorg_libICE_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "a3ea76ee3f4facd7a64684f9af25310825ee3668" +uuid = "f67eecfb-183a-506d-b269-f58e52b52d7c" +version = "1.1.2+0" + +[[deps.Xorg_libSM_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libICE_jll"] +git-tree-sha1 = "9c7ad99c629a44f81e7799eb05ec2746abb5d588" +uuid = "c834827a-8449-5923-a945-d239c165b7dd" +version = "1.2.6+0" + +[[deps.Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "808090ede1d41644447dd5cbafced4731c56bd2f" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.8.13+0" + +[[deps.Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "aa1261ebbac3ccc8d16558ae6799524c450ed16b" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.13+0" + +[[deps.Xorg_libXcursor_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "6c74ca84bbabc18c4547014765d194ff0b4dc9da" +uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" +version = "1.2.4+0" + +[[deps.Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "52858d64353db33a56e13c341d7bf44cd0d7b309" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.6+0" + +[[deps.Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "1a4a26870bf1e5d26cd585e38038d399d7e65706" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.8+0" + +[[deps.Xorg_libXfixes_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "75e00946e43621e09d431d9b95818ee751e6b2ef" +uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" +version = "6.0.2+0" + +[[deps.Xorg_libXi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll", "Xorg_libXfixes_jll"] +git-tree-sha1 = "a376af5c7ae60d29825164db40787f15c80c7c54" +uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" +version = "1.8.3+0" + +[[deps.Xorg_libXinerama_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll"] +git-tree-sha1 = "0ba01bc7396896a4ace8aab67db31403c71628f4" +uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" +version = "1.1.7+0" + +[[deps.Xorg_libXrandr_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "6c174ef70c96c76f4c3f4d3cfbe09d018bcd1b53" +uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" +version = "1.5.6+0" + +[[deps.Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "7ed9347888fac59a618302ee38216dd0379c480d" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.12+0" + +[[deps.Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXau_jll", "Xorg_libXdmcp_jll"] +git-tree-sha1 = "bfcaf7ec088eaba362093393fe11aa141fa15422" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.17.1+0" + +[[deps.Xorg_libxkbfile_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "ed756a03e95fff88d8f738ebc2849431bdd4fd1a" +uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" +version = "1.2.0+0" + +[[deps.Xorg_xcb_util_cursor_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_jll", "Xorg_xcb_util_renderutil_jll"] +git-tree-sha1 = "9750dc53819eba4e9a20be42349a6d3b86c7cdf8" +uuid = "e920d4aa-a673-5f3a-b3d7-f755a4d47c43" +version = "0.1.6+0" + +[[deps.Xorg_xcb_util_image_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "f4fc02e384b74418679983a97385644b67e1263b" +uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b" +version = "0.4.1+0" + +[[deps.Xorg_xcb_util_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll"] +git-tree-sha1 = "68da27247e7d8d8dafd1fcf0c3654ad6506f5f97" +uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5" +version = "0.4.1+0" + +[[deps.Xorg_xcb_util_keysyms_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "44ec54b0e2acd408b0fb361e1e9244c60c9c3dd4" +uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7" +version = "0.4.1+0" + +[[deps.Xorg_xcb_util_renderutil_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "5b0263b6d080716a02544c55fdff2c8d7f9a16a0" +uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e" +version = "0.3.10+0" + +[[deps.Xorg_xcb_util_wm_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "f233c83cad1fa0e70b7771e0e21b061a116f2763" +uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361" +version = "0.4.2+0" + +[[deps.Xorg_xkbcomp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxkbfile_jll"] +git-tree-sha1 = "801a858fc9fb90c11ffddee1801bb06a738bda9b" +uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" +version = "1.4.7+0" + +[[deps.Xorg_xkeyboard_config_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xkbcomp_jll"] +git-tree-sha1 = "00af7ebdc563c9217ecc67776d1bbf037dbcebf4" +uuid = "33bec58e-1273-512f-9401-5d533626f822" +version = "2.44.0+0" + +[[deps.Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "a63799ff68005991f9d9491b6e95bd3478d783cb" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.6.0+0" + +[[deps.YAML]] +deps = ["Base64", "Dates", "Printf", "StringEncodings"] +git-tree-sha1 = "a1c0c7585346251353cddede21f180b96388c403" +uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" +version = "0.4.16" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[[deps.eudev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "c3b0e6196d50eab0c5ed34021aaa0bb463489510" +uuid = "35ca27e7-8b34-5b7f-bca9-bdc33f59eb06" +version = "3.2.14+0" + +[[deps.fzf_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b6a34e0e0960190ac2a4363a1bd003504772d631" +uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" +version = "0.61.1+0" + +[[deps.libaom_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "371cc681c00a3ccc3fbc5c0fb91f58ba9bec1ecf" +uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" +version = "3.13.1+0" + +[[deps.libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "125eedcb0a4a0bba65b657251ce1d27c8714e9d6" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.17.4+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.15.0+0" + +[[deps.libdecor_jll]] +deps = ["Artifacts", "Dbus_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pango_jll", "Wayland_jll", "xkbcommon_jll"] +git-tree-sha1 = "9bf7903af251d2050b467f76bdbe57ce541f7f4f" +uuid = "1183f4f0-6f2a-5f1a-908b-139f9cdfea6f" +version = "0.2.2+0" + +[[deps.libevdev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "56d643b57b188d30cccc25e331d416d3d358e557" +uuid = "2db6ffa8-e38f-5e21-84af-90c45d0032cc" +version = "1.13.4+0" + +[[deps.libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "646634dd19587a56ee2f1199563ec056c5f228df" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "2.0.4+0" + +[[deps.libinput_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "eudev_jll", "libevdev_jll", "mtdev_jll"] +git-tree-sha1 = "91d05d7f4a9f67205bd6cf395e488009fe85b499" +uuid = "36db933b-70db-51c0-b978-0f229ee0e533" +version = "1.28.1+0" + +[[deps.libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "e015f211ebb898c8180887012b938f3851e719ac" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.55+0" + +[[deps.libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll"] +git-tree-sha1 = "11e1772e7f3cc987e9d3de991dd4f6b2602663a5" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.8+0" + +[[deps.mtdev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b4d631fd51f2e9cdd93724ae25b2efc198b059b1" +uuid = "009596ad-96f7-51b1-9f1b-5ce2d5e8a71e" +version = "1.1.7+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" + +[[deps.x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "14cc7083fc6dff3cc44f2bc435ee96d06ed79aa7" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "10164.0.1+0" + +[[deps.x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e7b67590c14d487e734dcb925924c5dc43ec85f3" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "4.1.0+0" + +[[deps.xkbcommon_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] +git-tree-sha1 = "a1fc6507a40bf504527d0d4067d718f8e179b2b8" +uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" +version = "1.13.0+0" diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..88ccc29 --- /dev/null +++ b/Project.toml @@ -0,0 +1,19 @@ +[deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +RCall = "6f49c342-dc21-5d91-9882-a32aef131414" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd" +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" + +[compat] +CSV = "0.10" +DataFrames = "1.8" +RCall = "0.14" +StatsBase = "0.34" +StatsPlots = "0.15" +TimeZones = "1.22" +YAML = "0.4" +julia = "1" diff --git a/pipelinesteps.txt b/pipelinesteps.txt index e170048..b324e15 100644 --- a/pipelinesteps.txt +++ b/pipelinesteps.txt @@ -1,3 +1,4 @@ +Old: sftp raw .fastq.gz files to local S bash fastqc on server `fastqc -t 20 *` S bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` @@ -11,3 +12,19 @@ S output taxonomy.tsv sftp taxonomy.tsv to local L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` L python merge tables `python3 merge_tables.py` + +Current: +sftp raw .fastq.gz files to local +S bash fastqc on server `fastqc -t 20 *` +S bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` +L julia ./src/main.jl + cutadapt + R call for filter and taxonomic assignment +L output .fasta and taxonomy.csv +sftp .fasta to server +S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i fastaseqs.fasta -o output.fasta` +S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global fastaseqs.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` +S output taxonomy.tsv +sftp taxonomy.tsv to local +L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` +L python merge tables `python3 merge_tables.py` \ No newline at end of file diff --git a/src/dada2.r b/src/dada2.r new file mode 100644 index 0000000..968ded7 --- /dev/null +++ b/src/dada2.r @@ -0,0 +1,274 @@ +# DADA2 amplicon sequencing pipeline +# +# Processes paired- or single-end Illumina reads into ASVs and assigns +# taxonomy using a reference database (default: PR2). Options for output +# of bootstraps or counts only, as well as parameters for filterAndTrim(), +# dada(), mergePairs(), and assignTaxonomy(). +# +# Usage: +# Rscript dada2.r +# +# Outputs (in workspace.root/Tables/): +# seqtab_nochim.csv - chimera-free ASV count table (sequences as rows) +# asvs.fasta / asvs.csv - ASV sequences with short identifiers (seq1, seq2 ...) +# taxonomy.csv - taxonomy assignments (optionally with bootstrap confidence values) +# tax_counts.xlsx - combined taxonomy + per-sample counts +# pipeline_stats.csv - read counts at each pipeline stage +# +# Notice: +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). +# +# This work is based on the DADA2 tutorial by Benjamin J. Callahan, et al., +# available at https://benjjneb.github.io/dada2/tutorial.html, with modification +# into a single module. The original material is licensed under the Creative +# Commons Attribution 4.0 International License (CC BY 4.0): +# https://creativecommons.org/licenses/by/4.0/. + +source("./src/dada2_functions.r") + +## Pipeline +main <- function() { + # Load args + args <- commandArgs(trailingOnly = TRUE) + if (length(args) == 0) { + stop("Usage: Rscript dada2_newer.r ") + } + yaml_path <- args[1] + + # Load config + message("Loading and validating config: ", yaml_path) + cfg <- load_config(yaml_path) + cfg <- validate_config(cfg) + verbose <- cfg$verbose %||% TRUE + mode <- cfg$file_patterns$mode %||% "paired" + + message("Setting up workspace: ", cfg$workspace$root) + paths <- setup_workspace(cfg$workspace$root) + + # Find input files + message("Discovering FASTQ files (mode: ", mode, ")") + fastq <- find_fastq_files( + cfg$workspace$input_dir, + cfg$file_patterns$forward, + cfg$file_patterns$reverse, + mode + ) + validate_sample_files(fastq$forward, fastq$reverse, mode) + + # In forward or reverse-only mode, sample names come from whichever side + # is present; the split pattern in config determines what counts as a name. + primary_files <- fastq$forward %||% fastq$reverse + sample_names <- extract_sample_names( + primary_files, + cfg$file_patterns$sample_name_split, + cfg$file_patterns$sample_name_index + ) + message(" Found ", length(sample_names), " samples") + + # Single-sample fallback: + # dada() returns a bare dada object (not a list) when given a single file. + # This breaks makeSequenceTable(), mergePairs(), and sapply() downstream. + # Workaround: duplicate the file paths so the pipeline sees 2 samples, then + # drop the duplicate row from all results before writing outputs. + # ASV calls are unaffected because dada() processes each file independently. + single_sample <- length(sample_names) == 1 + if (single_sample) { + warning("Only 1 sample found. Duplicating it to work around dada() returning ", + "a bare object for single-file input. The duplicate will be dropped ", + "from all outputs.") + fastq$forward <- rep(fastq$forward, 2) + if (!is.null(fastq$reverse)) fastq$reverse <- rep(fastq$reverse, 2) + sample_names <- c(sample_names, paste0(sample_names, "_dup")) + } + + # Quality assessment (pre-filter) + # Inspect quality_unfiltered.pdf to choose truncLen and maxEE values. + message("Plotting unfiltered quality profiles") + plot_quality_profiles( + fastq$forward, + fastq$reverse, + file.path(paths$Figures, "quality_unfiltered.pdf") + ) + + # Filter and trim + # Removes low-quality bases and reads. Reads that pass filtering are written + # to Filtered/; empty samples (0 reads) are silently skipped by DADA2 later. + message("Filtering and trimming reads") + filtered <- make_filtered_paths(sample_names, paths$Filtered, mode) + + # For single-end modes, route the relevant files into the fwd slots and + # pass NULL for rev so run_filter_trim() calls the correct filterAndTrim form. + in_fwd <- if (mode != "reverse") fastq$forward else fastq$reverse + out_fwd <- if (mode != "reverse") filtered$forward else filtered$reverse + in_rev_arg <- if (mode == "paired") fastq$reverse else NULL + out_rev_arg <- if (mode == "paired") filtered$reverse else NULL + + filter_stats <- run_filter_trim(in_fwd, in_rev_arg, out_fwd, out_rev_arg, + cfg$filter_trim, verbose) + + message("Plotting filtered quality profiles") + plot_quality_profiles( + filtered$forward, + filtered$reverse, + file.path(paths$Figures, "quality_filtered.pdf") + ) + + # Error model + # set.seed() should be called before learnErrors() + message("Learning error rates") + set.seed(cfg$dada$seed %||% 123L) + + fwd_errors <- if (mode != "reverse") { + learnErrors(filtered$forward, + nbases = cfg$dada$nbases, MAX_CONSIST = cfg$dada$max_consist, + verbose = verbose) + } else NULL + + rev_errors <- if (mode != "forward") { + learnErrors(filtered$reverse, + nbases = cfg$dada$nbases, MAX_CONSIST = cfg$dada$max_consist, + verbose = verbose) + } else NULL + + # Inspect error_rates.pdf: the fitted line should follow the observed points. + plot_error_rates(fwd_errors, rev_errors, + file.path(paths$Figures, "error_rates.pdf")) + + # Denoising + # dada() applies the error model to resolve ASV's from sequencing errors. + # pool = "pseudo" shares information across samples to improve detection of + # rare variants. + message("Denoising reads") + dada_fwd <- if (!is.null(fwd_errors)) { + dada(filtered$forward, err = fwd_errors, pool = cfg$dada$pool_method, + verbose = verbose) + } else NULL + + dada_rev <- if (!is.null(rev_errors)) { + dada(filtered$reverse, err = rev_errors, pool = cfg$dada$pool_method, + verbose = verbose) + } else NULL + + # Merge and sequence table + # Paired reads are joined at the overlapping region. Longer overlap and + # lower maxMismatch = higher confidence merges but fewer total merges. + message("Building sequence table") + if (mode == "paired") { + merged <- mergePairs( + dada_fwd, filtered$forward, + dada_rev, filtered$reverse, + minOverlap = cfg$merge$min_overlap, + maxMismatch = cfg$merge$max_mismatch, + trimOverhang = cfg$merge$trim_overhang, + verbose = verbose + ) + seq_table <- makeSequenceTable(merged) + } else { + merged <- NULL + seq_table <- makeSequenceTable(dada_fwd %||% dada_rev) + } + + # Plot the full length distribution before any length filtering so that + # off-target bands are visible when choosing band_size_min / band_size_max. + plot_length_distribution(seq_table, + file.path(paths$Figures, "length_distribution.pdf")) + + # Length filtering: retains only ASVs matching the expected amplicon size. + # Set band_size_min / band_size_max to null in config to skip this step. + band_min <- cfg$asv$band_size_min + band_max <- cfg$asv$band_size_max + if (!is.null(band_min) && !is.null(band_max)) { + message(" Filtering by length: ", band_min, "-", band_max, " bp") + seq_table <- filter_by_length(seq_table, band_min, band_max) + # Second plot confirms off-target lengths were successfully removed. + plot_length_distribution(seq_table, + file.path(paths$Figures, "length_distribution_filtered.pdf")) + } + + # Chimera removal + # "consensus" removes sequences identified as chimeric in the majority + # of samples in which they appear. + message("Removing chimeras") + seq_table_nochim <- removeBimeraDenovo(seq_table, method = cfg$asv$denovo_method, + verbose = verbose) + + nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 + message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), + "% | Retained: ", round(nochim_pct, 2), "%") + + # Drop duplicate row (if using single-sample fallback) + if (single_sample) { + filter_stats <- filter_stats[1, , drop = FALSE] + if (!is.null(dada_fwd)) dada_fwd <- dada_fwd[1] + if (!is.null(dada_rev)) dada_rev <- dada_rev[1] + if (!is.null(merged)) merged <- merged[1] + seq_table_nochim <- seq_table_nochim[1, , drop = FALSE] + sample_names <- sample_names[1] + } + + # Pipeline stats + message("Computing pipeline stats") + stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, + seq_table_nochim, sample_names, mode) + write.csv(stats, file.path(paths$Tables, "pipeline_stats.csv"), quote = FALSE) + if (verbose) print(stats) + + # Write core outputs + write_seq_table(seq_table_nochim, paths$Tables, + cfg$output$seq_table_prefix %||% "seqtab_nochim") + + message("Writing output tables") + # write_fasta() returns index (SeqName - Sequence), which is the join key + # used in write_taxa_table() and write_combined_table() below. + index <- write_fasta(seq_table_nochim, paths$Tables, + cfg$output$fasta_prefix %||% "asvs") + + # combined_input() defaults to index (SeqName + Sequence + counts only), + # which is the alternative output mode. It is replaced with the full + # taxa_df when taxonomy is run and combined_mode = "regular". + combined_input <- index + + # Taxonomy + if (!(cfg$taxonomy$skip %||% FALSE)) { + message("Assigning taxonomy") + db_path <- fetch_taxonomy_db(cfg$taxonomy$uri, paths$Taxonomy) + taxa_result <- run_assign_taxonomy(seq_table_nochim, db_path, + cfg$taxonomy, verbose) + taxa_df <- write_taxa_table( + taxa_result$tax, taxa_result$boot, index, + paths$Tables, + cfg$output$taxa_prefix %||% "taxonomy", + cfg$output$bootstraps %||% "combined" + ) + if ((cfg$output$combined_mode %||% "regular") == "regular") { + combined_input <- taxa_df + } + } else { + message(" Skipping taxonomy (taxonomy.skip = true)") + } + + # Checkpoint saved after taxonomy so the full environment (including taxa) + # can be restored with load() without re-running the pipeline. + checkpoint <- file.path(paths$Analysis, "checkpoint.RData") + save.image(checkpoint) + message(" Checkpoint saved: ", checkpoint) + + write_combined_table(combined_input, seq_table_nochim, paths$Tables, + cfg$output$combined_filename %||% "tax_counts.xlsx") + + message("Pipeline complete.") + message("Outputs:") + message(" ", file.path(paths$Tables, paste0(cfg$output$seq_table_prefix %||% "seqtab_nochim", ".csv"))) + message(" ", file.path(paths$Tables, paste0(cfg$output$fasta_prefix %||% "asvs", ".fasta"))) + message(" ", file.path(paths$Tables, paste0(cfg$output$fasta_prefix %||% "asvs", ".csv"))) + message(" ", file.path(paths$Tables, paste0(cfg$output$taxa_prefix %||% "taxonomy", ".csv"))) + message(" ", file.path(paths$Tables, cfg$output$combined_filename %||% "tax_counts.xlsx")) + message(" ", file.path(paths$Tables, "pipeline_stats.csv")) + message(" ", checkpoint) +} + +# Run pipeline +if (sys.nframe() == 0L) main() \ No newline at end of file diff --git a/src/dada2_functions.r b/src/dada2_functions.r new file mode 100644 index 0000000..d282e41 --- /dev/null +++ b/src/dada2_functions.r @@ -0,0 +1,387 @@ +# DADA2 amplicon sequencing functions +# +# Functions for dada2.r main workflow +# +# Notice: +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). +# +# This work is based on the DADA2 tutorial by Benjamin J. Callahan, et al., +# available at https://benjjneb.github.io/dada2/tutorial.html, with modification +# into a single module. The original material is licensed under the Creative +# Commons Attribution 4.0 International License (CC BY 4.0): +# https://creativecommons.org/licenses/by/4.0/. + +library(dada2) +library(openxlsx) +library(tidyverse) +library(yaml) + +# Null-coalescing operator: returns x if not NULL, otherwise y. +# Used to apply config defaults. +`%||%` <- function(x, y) if (!is.null(x)) x else y + +## Configuration +load_config <- function(yaml_path) { + if (!file.exists(yaml_path)) { + stop("Config file not found: ", yaml_path) + } + yaml.load_file(yaml_path) +} + +validate_config <- function(cfg) { + required <- c("workspace", "file_patterns", "filter_trim", + "dada", "merge", "asv", "taxonomy", "output") + missing_sections <- setdiff(required, names(cfg)) + if (length(missing_sections) > 0) { + stop("Missing required config sections: ", + paste(missing_sections, collapse = ", ")) + } + + if (is.null(cfg$workspace$root)) { + stop("workspace.root is required") + } + if (is.null(cfg$workspace$input_dir)) { + stop("workspace.input_dir is required") + } + if (!dir.exists(cfg$workspace$input_dir)) { + stop("workspace.input_dir does not exist: ", cfg$workspace$input_dir) + } + + mode <- cfg$file_patterns$mode %||% "paired" + if (!mode %in% c("paired", "forward", "reverse")) { + stop("file_patterns.mode must be one of: paired, forward, reverse") + } + + boot_mode <- cfg$output$bootstraps %||% "combined" + if (!boot_mode %in% c("none", "combined", "separate")) { + stop("output.bootstraps must be one of: none, combined, separate") + } + + # taxonomy.uri is only required when taxonomy assignment is not skipped + if (!(cfg$taxonomy$skip %||% FALSE) && is.null(cfg$taxonomy$uri)) { + stop("taxonomy.uri is required when taxonomy.skip is not true") + } + + combined_mode <- cfg$output$combined_mode %||% "regular" + if (!combined_mode %in% c("regular", "alternative")) { + stop("output.combined_mode must be one of: regular, alternative") + } + + invisible(cfg) +} + +# Creates the standard directory tree under root and returns paths as a named +# list so downstream functions can reference them by: +# directory <- paths$Directory (e.g. paths$Tables). +setup_workspace <- function(root) { + dirs <- list( + Tables = file.path(root, "Tables"), + Analysis = file.path(root, "Analysis"), + Taxonomy = file.path(root, "Taxonomy"), + Figures = file.path(root, "Figures"), + Filtered = file.path(root, "Filtered") + ) + for (d in dirs) dir.create(d, showWarnings = FALSE, recursive = TRUE) + dirs +} + +# Finds FASTQ files matching forward/reverse patterns. Files are sorted so that +# each forward file always corresponds its reverse file by name. +find_fastq_files <- function(input_dir, fwd_pattern, rev_pattern, mode) { + fwd <- NULL + rev <- NULL + if (mode %in% c("paired", "forward")) { + fwd <- sort(list.files(input_dir, pattern = fwd_pattern, full.names = TRUE)) + } + if (mode %in% c("paired", "reverse")) { + rev <- sort(list.files(input_dir, pattern = rev_pattern, full.names = TRUE)) + } + list(forward = fwd, reverse = rev) +} + +# Checks that at least one sample was found and that forward/reverse counts +# match in paired mode. With one sample, dada() would return a bare object +# instead of a list so duplication is performed in main() by the file paths +# before the pipeline runs. +validate_sample_files <- function(fwd, rev, mode) { + if (mode %in% c("paired", "forward")) { + if (length(fwd) < 1) { + stop("No forward FASTQ files found. ", + "Check workspace.input_dir and file_patterns.forward.") + } + missing_fwd <- fwd[!file.exists(fwd)] + if (length(missing_fwd) > 0) { + stop("Forward files not found:\n ", paste(missing_fwd, collapse = "\n ")) + } + } + if (mode %in% c("paired", "reverse")) { + if (length(rev) < 1) { + stop("No reverse FASTQ files found. ", + "Check workspace.input_dir and file_patterns.reverse.") + } + missing_rev <- rev[!file.exists(rev)] + if (length(missing_rev) > 0) { + stop("Reverse files not found:\n ", paste(missing_rev, collapse = "\n ")) + } + } + if (mode == "paired" && length(fwd) != length(rev)) { + stop("Forward file count (", length(fwd), ") does not match ", + "reverse file count (", length(rev), ").") + } + invisible(NULL) +} + +# Extracts sample names. (e.g. "Sample1_R1_trimmed.fastq.gz" split by "_" +# at index 1 gives "Sample1"). +extract_sample_names <- function(fwd_files, split_char, split_index) { + sapply(strsplit(basename(fwd_files), split_char), `[`, split_index) +} + +# Constructs output paths for filtered reads. +make_filtered_paths <- function(sample_names, filtered_dir, mode = "paired") { + fwd <- NULL + rev <- NULL + if (mode != "reverse") { + fwd <- file.path(filtered_dir, paste0(sample_names, "_R1_filt.fastq.gz")) + names(fwd) <- sample_names + } + if (mode != "forward") { + rev <- file.path(filtered_dir, paste0(sample_names, "_R2_filt.fastq.gz")) + names(rev) <- sample_names + } + list(forward = fwd, reverse = rev) +} + +## Plotting + +# Saves per-base quality score profiles for up to 3 forward and 3 reverse +# samples. Inspect before and after filtering to guide truncLen / maxEE choices. +plot_quality_profiles <- function(fwd_files, rev_files, output_pdf) { + pdf(output_pdf, width = 8, height = 6) + if (!is.null(fwd_files) && length(fwd_files) > 0) { + plotQualityProfile(fwd_files[seq_len(min(3L, length(fwd_files)))]) + } + if (!is.null(rev_files) && length(rev_files) > 0) { + plotQualityProfile(rev_files[seq_len(min(3L, length(rev_files)))]) + } + dev.off() + invisible(NULL) +} + +# Plots the learned substitution error rates against quality scores. +# The fitted line should track the observed points closely; poor fit suggests +# the error model did not converge. If so, try increasing nbases or max_consist. +plot_error_rates <- function(fwd_errors, rev_errors, output_pdf) { + pdf(output_pdf, width = 8, height = 6) + if (!is.null(fwd_errors)) plotErrors(fwd_errors, nominalQ = TRUE) + if (!is.null(rev_errors)) plotErrors(rev_errors, nominalQ = TRUE) + dev.off() + invisible(NULL) +} + +# Plots the distribution of merged ASV lengths. Inspect to confirm the +# dominant peak corresponds to the expected amplicon size and to set +# band_size_min / band_size_max for off-target removal. +plot_length_distribution <- function(seq_table, output_pdf) { + pdf(output_pdf, width = 8, height = 6) + plot(table(nchar(getSequences(seq_table))), + xlab = "Merged read length (bp)", + ylab = "Count", + main = "ASV length distribution") + dev.off() + invisible(NULL) +} + +## Filter and trim + +# Wrapper around filterAndTrim that handles both paired and single-end modes. +# Key parameters: +# truncQ - truncate reads at the first base with Phred quality ≤ this value +# maxEE - maximum expected errors per read (calculated from Phred scores); +# lower = stricter, recommended 2–5 for typical Illumina data +# matchIDs - (paired only) discard read pairs where one read was filtered out, +# ensuring F and R files remain perfectly synchronised +run_filter_trim <- function(fwd_in, rev_in, fwd_out, rev_out, params, verbose) { + trunc_len <- unlist(params$trunc_len) + max_ee <- unlist(params$max_ee) + + if (!is.null(rev_in)) { + filterAndTrim( + fwd_in, fwd_out, + rev_in, rev_out, + truncQ = params$trunc_q, + truncLen = trunc_len, + maxEE = max_ee, + minLen = params$min_len, + maxN = params$max_n, + matchIDs = params$match_ids, + rm.phix = params$rm_phix, + verbose = verbose + ) + } else { + # Single-end: use only the first element of trunc_len / maxEE + filterAndTrim( + fwd_in, fwd_out, + truncQ = params$trunc_q, + truncLen = trunc_len[1], + maxEE = max_ee[1], + minLen = params$min_len, + maxN = params$max_n, + rm.phix = params$rm_phix, + verbose = verbose + ) + } +} + +## Sequence table + +# Retains only ASVs whose length falls within [band_min, band_max]. +# Amplicons outside this range are typically non-specific or artefactual +# (e.g. primer dimers, chimeras that survived removal, off-target loci). +filter_by_length <- function(seq_table, band_min, band_max) { + lengths <- nchar(colnames(seq_table)) + seq_table[, lengths >= band_min & lengths <= band_max, drop = FALSE] +} + +## Pipeline stats + +# Builds a per-sample read-count table showing reads retained at each step: +# input -> filtered -> denoised (F/R) -> merged -> nochim. +# Large drops at any stage can indicate a problem: +# filtered - overly strict truncLen / maxEE +# denoised - poor error model fit +# merged - insufficient overlap, mismatched truncation lengths +# nochim - high chimera rate +compute_pipeline_stats <- function(filter_stats, dada_fwd, dada_rev, merged, + seq_table_nochim, sample_names, mode) { + get_n <- function(x) sum(getUniques(x)) + + track <- as.data.frame(filter_stats) + colnames(track) <- c("input", "filtered") + + if (mode %in% c("paired", "forward") && !is.null(dada_fwd)) { + track$denoisedF <- sapply(dada_fwd, get_n) + } + if (mode %in% c("paired", "reverse") && !is.null(dada_rev)) { + track$denoisedR <- sapply(dada_rev, get_n) + } + if (mode == "paired" && !is.null(merged)) { + track$merged <- sapply(merged, get_n) + } + track$nochim <- rowSums(seq_table_nochim) + + rownames(track) <- sample_names + track +} + +## Taxonomy + +# Downloads the reference database if not already present. +# The file is stored in the project's Taxonomy/ folder so it can be reused +# across runs without re-downloading. +fetch_taxonomy_db <- function(uri, local_dir) { + local_path <- file.path(local_dir, basename(uri)) + if (!file.exists(local_path)) { + message(" Downloading taxonomy database: ", uri) + download.file(uri, local_path, mode = "wb") + } + local_path +} + +# Assigns taxonomy to ASVs using a naive Bayesian classifier. +# outputBootstraps is always TRUE so bootstrap confidence values (0–100 per +# rank) are available for downstream filtering regardless of minBoot. +# minBoot sets the threshold at which assignments are returned; 0 returns all +# assignments. +run_assign_taxonomy <- function(seq_table, db_path, params, verbose) { + result <- assignTaxonomy( + seq_table, + db_path, + multithread = params$multithread, + minBoot = params$min_boot, + outputBootstraps = TRUE, + verbose = verbose, + taxLevels = params$levels + ) + list(tax = result$tax, boot = result$boot) +} + +## Output + +# Writes the chimera-free ASV count table as a CSV with sequences as row names +# and samples as columns. +write_seq_table <- function(seq_table, tables_dir, prefix) { + output_path <- file.path(tables_dir, paste0(prefix, ".csv")) + write.csv(t(seq_table), output_path, quote = FALSE) + invisible(NULL) +} + +# Writes ASV sequences to a FASTA file and a companion CSV mapping short +# identifiers (seq1, seq2, ...) to full sequences. Returns the index +# data frame, which is used as the join key in later tables. +write_fasta <- function(seq_table, tables_dir, prefix) { + sequences <- colnames(seq_table) + n <- length(sequences) + seq_names <- sprintf("seq%d", seq_len(n)) + + fasta_lines <- c(rbind(paste0(">", seq_names), sequences)) + writeLines(fasta_lines, file.path(tables_dir, paste0(prefix, ".fasta"))) + + index <- data.frame(SeqName = seq_names, Sequence = sequences, + stringsAsFactors = FALSE) + write.csv(index, file.path(tables_dir, paste0(prefix, ".csv")), + quote = FALSE, row.names = FALSE) + + index +} + +# Writes taxonomy assignments to CSV. Bootstrap confidence values (0–100 per +# taxonomic rank) are handled based on bootstrap_mode: +# none - taxonomy columns only +# combined - taxonomy and bootstrap columns in one file (suffix: _boot) +# separate - taxonomy and bootstraps written to two separate files +# Returns taxa_df (SeqName + Sequence + taxonomy), used by write_combined_table(). +write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, + prefix, bootstrap_mode) { + taxa_df <- as.data.frame(tax_matrix, stringsAsFactors = FALSE) + taxa_df$Sequence <- rownames(taxa_df) + taxa_df <- dplyr::left_join(index, taxa_df, by = "Sequence") + + if (bootstrap_mode == "combined") { + boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) + colnames(boot_df) <- paste0(colnames(boot_df), "_boot") + boot_df$Sequence <- rownames(boot_matrix) + combined_df <- dplyr::left_join(taxa_df, boot_df, by = "Sequence") + write.csv(combined_df, file.path(tables_dir, paste0(prefix, ".csv")), + quote = FALSE, row.names = FALSE) + } else { + write.csv(taxa_df, file.path(tables_dir, paste0(prefix, ".csv")), + quote = FALSE, row.names = FALSE) + if (bootstrap_mode == "separate") { + boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) + boot_df$Sequence <- rownames(boot_matrix) + boot_df <- dplyr::left_join(index, boot_df, by = "Sequence") + write.csv(boot_df, + file.path(tables_dir, paste0(prefix, "_bootstraps.csv")), + quote = FALSE, row.names = FALSE) + } + } + + taxa_df +} + +# Joins taxonomy (or index for alternative mode) with per-sample counts and +# writes an Excel workbook. +write_combined_table <- function(taxa_df, seq_table, tables_dir, filename) { + seq_t <- as.data.frame(t(seq_table), stringsAsFactors = FALSE) + seq_t <- tibble::rownames_to_column(seq_t, "Sequence") + combined <- dplyr::left_join(taxa_df, seq_t, by = "Sequence") + output_path <- file.path(tables_dir, filename) + openxlsx::write.xlsx(combined, output_path, overwrite = TRUE, + asTable = FALSE, sheetName = "1.sampling", + firstRow = TRUE, zoom = 90, keepNA = TRUE) + invisible(NULL) +} diff --git a/src/main.jl b/src/main.jl index a8ee96e..ae719d2 100644 --- a/src/main.jl +++ b/src/main.jl @@ -1,7 +1,7 @@ include("run_cutadapt.jl") include("merge_and_filter_taxa.jl") -using CSV +using CSV, RCall using .Cutadapt, .TaxonomyTableTools ## Instantiate @@ -16,8 +16,16 @@ vespav = "./vsearch/taxonomy_vespa_pool_fwdonly.tsv" multid = "./DADA2/tax_counts_fasta_multi_pool.csv" vespad = "./DADA2/tax_counts_fasta_vespa_pool_fwdonly.csv" +dada2_config_path = "./inputs/dada2_config.yml" + ## Main #cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) -CSV.write("protist_filtered_multi_pool.csv", filter_table(merge_taxonomy_counts(multiv, multid),"./inputs/protist_filter_multiplex.yml")) +#R"system(paste('Rscript', './src/dada2.r', $dada2_config_path))" + +#CSV.write("protist_filtered_vespa_pool.csv", filter_table(merge_taxonomy_counts(vespav, vespad),"./inputs/protist_filter.yml")) +#CSV.write("protist_filtered_vespa_pool.csv", filter_table(merge_taxonomy_counts(vespav, vespad),"./inputs/protist_filter.yml")) + +#CSV.write("merged_multi_pool.csv", merge_taxonomy_counts(multiv, multid)) +#CSV.write("merged_vespa_pool.csv", merge_taxonomy_counts(vespav, vespad)) \ No newline at end of file diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index 217b99c..1f31844 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -159,10 +159,4 @@ export cutadapt cutadapt_dir ) end -end - -primers_path = "./inputs/primers.yml" -primer_pairs = ["TarEuk", "Meta2"] -fastq_input_dir = "./inputs/fastq/" -fastq_output_dir = "./cutadapt/fastq/" -optional_args = "-m 200 --discard-untrimmed" \ No newline at end of file +end \ No newline at end of file From 377d95d35cdfb0b1718822388e951eff9c786ee3 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 22:12:02 +0100 Subject: [PATCH 008/175] Reorganise directory structure. --- .gitignore | 5 ++- config/dada2.yml | 61 ++++++++++++++++++++++++++++++++++++ config/primers.yml | 27 ++++++++++++++++ config/protist_filter.yml | 34 ++++++++++++++++++++ src/main.jl | 54 +++++++++++++++++++++---------- src/merge_and_filter_taxa.jl | 10 +++--- src/run_cutadapt.jl | 16 +++++----- src/run_dada2.jl | 18 +++++++++++ 8 files changed, 191 insertions(+), 34 deletions(-) create mode 100644 config/dada2.yml create mode 100644 config/primers.yml create mode 100644 config/protist_filter.yml create mode 100644 src/run_dada2.jl diff --git a/.gitignore b/.gitignore index 8b50f65..a9c42f1 100644 --- a/.gitignore +++ b/.gitignore @@ -250,7 +250,7 @@ rsconnect/ ### Other ### # Things with ambiguous intellectual property internal -inputs/fastq/ +data/fastq/ cutadapt/output_* ### Lock files ### @@ -259,5 +259,4 @@ cutadapt/output_* ### Bits I don't want to share yet ### hide.* -DADA2 -vsearch \ No newline at end of file +output/ \ No newline at end of file diff --git a/config/dada2.yml b/config/dada2.yml new file mode 100644 index 0000000..13b01a6 --- /dev/null +++ b/config/dada2.yml @@ -0,0 +1,61 @@ +workspace: + root: "/path/to/project" + input_dir: "/path/to/fastqs" + +file_patterns: + forward: "_R1_trimmed.fastq.gz" + reverse: "_R2_trimmed.fastq.gz" + sample_name_split: "_" + sample_name_index: 1 + mode: "paired" # paired | forward | reverse + +filter_trim: + trunc_q: 2 + trunc_len: [220, 220] # [forward, reverse]; first value used for single-end + max_ee: [3, 3] + min_len: 175 + max_n: 0 + match_ids: true + rm_phix: true + +dada: + seed: 123 + nbases: 200000000 + max_consist: 15 + pool_method: "pseudo" # none | pseudo | true + +merge: + min_overlap: 20 + max_mismatch: 0 + trim_overhang: true + +asv: + band_size_min: 250 # null to skip length filtering + band_size_max: 256 + denovo_method: "consensus" + +taxonomy: + skip: false # true to skip taxonomy and use alternative output mode + uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" + multithread: true + min_boot: 0 + levels: + - "Domain" + - "Supergroup" + - "Division" + - "Subdivision" + - "Class" + - "Order" + - "Family" + - "Genus" + - "Species" + +output: + bootstraps: "combined" # none | combined | separate + combined_mode: "regular" # regular (taxonomy + counts) | alternative (SeqName + Sequence + counts, no taxonomy) + seq_table_prefix: "seqtab_nochim" + fasta_prefix: "asvs" + taxa_prefix: "taxonomy" + combined_filename: "tax_counts.xlsx" + +verbose: true diff --git a/config/primers.yml b/config/primers.yml new file mode 100644 index 0000000..53a0027 --- /dev/null +++ b/config/primers.yml @@ -0,0 +1,27 @@ +Forward: + TarEukF: "CCAGCASCYGCGGTAATTCC" + ArchF: "YGCGGTAAYTCCAGCTC" + VESPAF: "AGCAGCCGCGGTAATTCC" + +Reverse: + TarEukR: "ACTTTCGTTCTTGATYRA" + Meta1R: "CYCCTACYYTMGYYCTKGA" + Meta2R: "DCTKTCGTYCTTGATYRA" + VESPAR: "TCCGTCAATTYCTTNAASTTTC" + +Pairs: + - TarEuk: + - TarEukF + - TarEukR + - Meta1: + - TarEukF + - Meta1R + - Meta2: + - TarEukF + - Meta2R + - Arch: + - ArchF + - TarEukR + - VESPA: + - VESPAF + - VESPAR \ No newline at end of file diff --git a/config/protist_filter.yml b/config/protist_filter.yml new file mode 100644 index 0000000..e2d8daf --- /dev/null +++ b/config/protist_filter.yml @@ -0,0 +1,34 @@ +# Mappings from Division -> Supergroup +mappings: + Rhizaria: Rhizaria + Alveolata: Alveolata + Stramenopiles: Stramenopiles + Hemimastigophora: Hemimastigophora + Discoba: Discoba + Metamonada: Metamonada + Telonemia: Telonemia + Ancyromonadida: Ancyromonadida + +# Taxonomic filters: exclude rows where these columns contain these substrings +filters: + - column: Domain + pattern: Bacteria + - column: Domain + pattern: Archaea + - column: Domain + pattern: Eukaryota:plas + - column: Domain + pattern: Eukaryota:mito + - column: Supergroup + pattern: TSAR:chro + - column: Subdivision + pattern: Metazoa + - column: Subdivision + pattern: Fungi + - column: Division + pattern: Rhodophyta + - column: Class + pattern: Embryophyceae + +# Optional: whether to remove empty/unassigned domains by default +remove_empty_domain: true \ No newline at end of file diff --git a/src/main.jl b/src/main.jl index ae719d2..22df383 100644 --- a/src/main.jl +++ b/src/main.jl @@ -1,31 +1,51 @@ include("run_cutadapt.jl") +include("run_dada2.jl") include("merge_and_filter_taxa.jl") -using CSV, RCall -using .Cutadapt, .TaxonomyTableTools +using CSV +using .Cutadapt, .TaxonomyTableTools, .DADA2 -## Instantiate -primers_path = "./inputs/primers.yml" +## Instantiate filesystem +# Root directories +data_dir = "./data" +config_dir = "./config" +output_dir = "./output" + +# Cutadapt paths +fastq_input_dir = joinpath(data_dir, "fastq") +cutadapt_dir = joinpath(output_dir, "cutadapt") +primers_config = joinpath(config_dir, "primers.yml") + +## Instantiate parameters +# Cutadapt parameters primer_pairs = ["TarEuk", "Meta2"] -fastq_input_dir = "./inputs/fastq/" -cutadapt_dir = "./cutadapt/" optional_args = "-m 200 --discard-untrimmed" -multiv = "./vsearch/taxonomy_multi_pool.tsv" -vespav = "./vsearch/taxonomy_vespa_pool_fwdonly.tsv" -multid = "./DADA2/tax_counts_fasta_multi_pool.csv" -vespad = "./DADA2/tax_counts_fasta_vespa_pool_fwdonly.csv" +# DADA2 parameters +dada2_config_dir = joinpath(config_dir, "dada2_config.yml") + +# Merge and filter (DADA2-VSEARCH) parameters +multiv = joinpath(output_dir, "vsearch/taxonomy_multi_pool.tsv") +vespav = joinpath(output_dir, "vsearch/taxonomy_vespa_pool_fwdonly.tsv") +multid = joinpath(output_dir, "dada2/tax_counts_fasta_multi_pool.csv") +vespad = joinpath(output_dir, "dada2/tax_counts_fasta_vespa_pool_fwdonly.csv") + +protist_filter = joinpath(config_dir, "protist_filter.yml") + +merged_outfile_multi = joinpath(output_dir, "merged_multi.csv") +merged_outfile_vespa = joinpath(output_dir, "merged_vespa.csv") -dada2_config_path = "./inputs/dada2_config.yml" +filtered_outfile_multi = joinpath(output_dir, "protist_filtered_vespa.csv") +filtered_outfile_vespa = joinpath(output_dir, "protist_filtered_vespa.csv") ## Main -#cutadapt(primer_pairs, primers_path, fastq_input_dir, cutadapt_dir, optional_args = optional_args) +#cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, optional_args = optional_args) -#R"system(paste('Rscript', './src/dada2.r', $dada2_config_path))" +#dada2(dada_config_dir) -#CSV.write("protist_filtered_vespa_pool.csv", filter_table(merge_taxonomy_counts(vespav, vespad),"./inputs/protist_filter.yml")) -#CSV.write("protist_filtered_vespa_pool.csv", filter_table(merge_taxonomy_counts(vespav, vespad),"./inputs/protist_filter.yml")) +#CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) +#CSV.write(merged_outfile_vespa, merge_taxonomy_counts(vespav, vespad)) -#CSV.write("merged_multi_pool.csv", merge_taxonomy_counts(multiv, multid)) -#CSV.write("merged_vespa_pool.csv", merge_taxonomy_counts(vespav, vespad)) \ No newline at end of file +#CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(vespav, vespad), protist_filter)) +#CSV.write(filtered_outfile_vespa, filter_table(merge_taxonomy_counts(vespav, vespad), protist_filter)) \ No newline at end of file diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index a3f6233..f13efaf 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -148,9 +148,7 @@ export merge_taxonomy_counts, filter_table """ filter_table(merged_df; filters_yaml_path, remove_empty_domain_override) - Applies protist filtering to a merged DataFrame produced by merge_taxonomy_counts. - - Applies protist filtering to a merged DataFrame using rules defined in a YAML file. + Applies protist filtering to a merged DataFrame produced by merge_taxonomy_counts using rules defined in a YAML file. ## Arguments: - `merged_df`: Input DataFrame with taxonomy columns. @@ -178,7 +176,7 @@ export merge_taxonomy_counts, filter_table @info "Filtering table using configuration from $filters_yaml_path" - # === Load mappings === + # Load mappings mapping_config = get(config, "mappings", Dict()) if !isempty(mapping_config) @assert mapping_config isa AbstractDict "mappings must be a dictionary in YAML" @@ -194,7 +192,7 @@ export merge_taxonomy_counts, filter_table ] end - # === Load remove_empty_domain flag === + # Load remove_empty_domain flag remove_empty = get(config, "remove_empty_domain", true) if remove_empty_domain_override !== nothing remove_empty = remove_empty_domain_override @@ -209,7 +207,7 @@ export merge_taxonomy_counts, filter_table df) end - # === Load and apply filters === + # Load and apply filters raw_filters = get(config, "filters", []) @assert raw_filters isa Vector "filters must be a list in YAML" diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index 1f31844..4323078 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -136,20 +136,20 @@ export cutadapt This would execute `cutadapt` with arguments `-g CCAGCASCYGCGGTAATTCC -G ACTTTCGTTCTTGATYRA -G DCTKTCGTYCTTGATYRA`. ## Arguments - - `primer_pairs` (default: [""]): Specify primers in string array format. - - `primers_path` (default: "./inputs/primers.yml"): Specify path of YAML file for primers. - - `fastq_in_dir` (default: "./inputs/fastq/"): Specify path of *_*_L001_R1_001.fastq.gz and *_*_L001_R2_001.fastq.gz files for cutadapt. - - `cutadapt_dir` (default: "./cutadapt/"): Specify an output directory for trimmed .fastq.gz and logs. + - `primer_pairs`: Specify primers in string array format. + - `primers_path`: Specify path of YAML file for primers. + - `fastq_in_dir`: Specify path of *_*_L001_R1_001.fastq.gz and *_*_L001_R2_001.fastq.gz files for cutadapt. + - `cutadapt_dir`: Specify an output directory for trimmed .fastq.gz and logs. ## Keyword Arguments - `optional_args` (optional, default: "-m 200 --discard-untrimmed"): Specify additional arguments passed to `cutadapt` command. """ function cutadapt( - primer_pairs = [""], - primers_path = "./inputs/primers.yml", - fastq_in_dir = "./inputs/fastq/", - cutadapt_dir = "./cutadapt/"; + primer_pairs, + primers_path, + fastq_in_dir, + cutadapt_dir; optional_args = "-m 200 --discard-untrimmed" ) run_cutadapt( diff --git a/src/run_dada2.jl b/src/run_dada2.jl new file mode 100644 index 0000000..3c6c8c0 --- /dev/null +++ b/src/run_dada2.jl @@ -0,0 +1,18 @@ +module DADA2 + +export dada2 + + using RCall + + """ + dada2(dada2_config_dir) + Most adorable little wrapper that simply passes the dada2 config to R to run through its pipeline. + + ## Arguments: + - `dada2_config_dir`: Path to DADA2 YAML config. + """ + function dada2(dada2_config_dir) + R"system(paste('Rscript', './src/dada2.r', $dada2_config_dir))" + end + +end \ No newline at end of file From 8ad82ebaad40ff4302a262f7edc9ec1a4a8378af Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 22:19:29 +0100 Subject: [PATCH 009/175] Update docstrings --- src/merge_and_filter_taxa.jl | 13 +++++++++---- src/run_dada2.jl | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index f13efaf..4600c6c 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -68,13 +68,17 @@ export merge_taxonomy_counts, filter_table end """ - merge_taxonomy_counts(taxonomy_vsearch_path, counts_csv_path) + merge_taxonomy_counts(taxonomy_vsearch_path, counts_csv_path) Reads taxonomy TSV from vsearch and counts CSV from DADA2 and returns a merged DataFrame. + + ## Arguments: + - `taxonomy_vsearch_path` (): Path to vsearch output file. + - `counts_csv_path`: Path to DADA2 idtax output file. """ function merge_taxonomy_counts( - taxonomy_vsearch_path::AbstractString = "./vsearch/taxonomy.tsv", - counts_csv_path::AbstractString = "./DADA2/tax_counts_fasta.csv") + taxonomy_vsearch_path, + counts_csv_path) @info("Merging taxonomy counts.") @@ -147,7 +151,8 @@ export merge_taxonomy_counts, filter_table end """ - filter_table(merged_df; filters_yaml_path, remove_empty_domain_override) + filter_table(merged_df; filters_yaml_path, remove_empty_domain_override) + Applies protist filtering to a merged DataFrame produced by merge_taxonomy_counts using rules defined in a YAML file. ## Arguments: diff --git a/src/run_dada2.jl b/src/run_dada2.jl index 3c6c8c0..32ebfd5 100644 --- a/src/run_dada2.jl +++ b/src/run_dada2.jl @@ -5,8 +5,9 @@ export dada2 using RCall """ - dada2(dada2_config_dir) - Most adorable little wrapper that simply passes the dada2 config to R to run through its pipeline. + dada2(dada2_config_dir) + + Most adorable little wrapper that simply passes the dada2 config to R to run through its pipeline. Requires dada2.r in src. ## Arguments: - `dada2_config_dir`: Path to DADA2 YAML config. From 5d6b2fa87c9e8e774f3b6b33ee099d22b52cf9fb Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 22:26:15 +0100 Subject: [PATCH 010/175] Minor fixes. --- src/main.jl | 6 +++--- src/run_cutadapt.jl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main.jl b/src/main.jl index 22df383..bb83bc7 100644 --- a/src/main.jl +++ b/src/main.jl @@ -22,7 +22,7 @@ primer_pairs = ["TarEuk", "Meta2"] optional_args = "-m 200 --discard-untrimmed" # DADA2 parameters -dada2_config_dir = joinpath(config_dir, "dada2_config.yml") +dada2_config_dir = joinpath(config_dir, "dada2.yml") # Merge and filter (DADA2-VSEARCH) parameters multiv = joinpath(output_dir, "vsearch/taxonomy_multi_pool.tsv") @@ -35,14 +35,14 @@ protist_filter = joinpath(config_dir, "protist_filter.yml") merged_outfile_multi = joinpath(output_dir, "merged_multi.csv") merged_outfile_vespa = joinpath(output_dir, "merged_vespa.csv") -filtered_outfile_multi = joinpath(output_dir, "protist_filtered_vespa.csv") +filtered_outfile_multi = joinpath(output_dir, "protist_filtered_multi.csv") filtered_outfile_vespa = joinpath(output_dir, "protist_filtered_vespa.csv") ## Main #cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, optional_args = optional_args) -#dada2(dada_config_dir) +#dada2(dada2_config_dir) #CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) #CSV.write(merged_outfile_vespa, merge_taxonomy_counts(vespav, vespad)) diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index 4323078..c345cd2 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -63,7 +63,7 @@ export cutadapt # Messy filesystem stuff time = chop("$(now(localzone()))", tail = 13) - fastq_out_dir = cutadapt_dir * "output_$time/" + fastq_out_dir = cutadapt_dir * "$time/" log_dir = fastq_out_dir * "/logs/" stats_path = joinpath(log_dir, "cutadapt_primer_trimming_stats.txt") summary_path = joinpath(log_dir, "cutadapt_trimmed_percentage.txt") From b930778cb8bd70a9f7349fa5d9a2d88f94f5fd03 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 22:29:39 +0100 Subject: [PATCH 011/175] Remove pipelinesteps.txt, update .gitignore. --- .gitignore | 3 ++- pipelinesteps.txt | 30 ------------------------------ 2 files changed, 2 insertions(+), 31 deletions(-) delete mode 100644 pipelinesteps.txt diff --git a/.gitignore b/.gitignore index a9c42f1..12ab42e 100644 --- a/.gitignore +++ b/.gitignore @@ -259,4 +259,5 @@ cutadapt/output_* ### Bits I don't want to share yet ### hide.* -output/ \ No newline at end of file +output/ +pipelinesteps.txt \ No newline at end of file diff --git a/pipelinesteps.txt b/pipelinesteps.txt deleted file mode 100644 index b324e15..0000000 --- a/pipelinesteps.txt +++ /dev/null @@ -1,30 +0,0 @@ -Old: -sftp raw .fastq.gz files to local -S bash fastqc on server `fastqc -t 20 *` -S bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` -L bash cutadapt to trim .fastq.gz files `python3 cutadapt_test.sh` -L R filter and taxonomic assignment -L output .fasta and taxonomy.csv -sftp .fasta to server -S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i fastaseqs.fasta -o output.fasta` -S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global fastaseqs.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` -S output taxonomy.tsv -sftp taxonomy.tsv to local -L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` -L python merge tables `python3 merge_tables.py` - -Current: -sftp raw .fastq.gz files to local -S bash fastqc on server `fastqc -t 20 *` -S bash multiqc on server `/mnt/mokosz/home/novakjiri9/.local/bin/multiqc .` -L julia ./src/main.jl - cutadapt - R call for filter and taxonomic assignment -L output .fasta and taxonomy.csv -sftp .fasta to server -S bash cd-hit-est on server for demultiplex (optional) `cd-hit-est -c 1 -i fastaseqs.fasta -o output.fasta` -S bash vsearch `~/software/vsearch/vsearch-2.26.1/bin/vsearch --usearch_global fastaseqs.fasta --db ~/database/PR2/5.0.0/pr2_version_5.0.0_SSU_taxo_long.fasta --blast6out taxonomy.tsv --id 0.75 --query_cov 0.8` -S output taxonomy.tsv -sftp taxonomy.tsv to local -L python modify taxa `python3 modify_taxonomy.py -i taxonomy.tsv -o taxonomy_table.tsv` -L python merge tables `python3 merge_tables.py` \ No newline at end of file From a317df64753a7afa3ccefe3f421a6351aceff8f0 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Wed, 18 Feb 2026 22:54:17 +0100 Subject: [PATCH 012/175] Added license. --- LICENSE | 661 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. From 0d2ae50cb28ee1e76959aaf4a810b2890a1f3209 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 19 Feb 2026 02:06:03 +0100 Subject: [PATCH 013/175] Added install scripts and README.md. --- .gitignore | 4 + README.md | 291 ++++++++++++++++++++++ config/dada2.yml | 2 +- config/tools.example.yml | 28 +++ install.jl | 461 +++++++++++++++++++++++++++++++++++ install.sh | 83 +++++++ src/main.jl | 32 ++- src/merge_and_filter_taxa.jl | 16 +- src/run_cutadapt.jl | 17 +- 9 files changed, 923 insertions(+), 11 deletions(-) create mode 100644 README.md create mode 100644 config/tools.example.yml create mode 100644 install.jl create mode 100755 install.sh diff --git a/.gitignore b/.gitignore index 12ab42e..4b1a8b3 100644 --- a/.gitignore +++ b/.gitignore @@ -257,6 +257,10 @@ cutadapt/output_* .~lock.* .csv# +### Downloaded binaries and tool paths +bin/ +config/tools.yml + ### Bits I don't want to share yet ### hide.* output/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..734c291 --- /dev/null +++ b/README.md @@ -0,0 +1,291 @@ +# MetaManifold + +A Julia pipeline for amplicon metabarcoding from raw paired-end Illumina reads to filtered, taxonomy-annotated ASV tables. + +## Overview + +This project aims to wrap a standard amplicon sequencing workflow into a single, configurable project. It handles multiplex primer trimming amplicon denoising, taxonomy assignment, and taxonomic filtering. + +**Pipeline stages** + +| Step | Tool | Output | +|------|------|--------| +| Primer trimming | cutadapt | Trimmed FASTQ pairs per sample | +| Quality assessment | FastQC / MultiQC | HTML QC reports | +| Amplicon denoising | DADA2 (R) | ASV count table, FASTA, taxonomy | +| Taxonomy assignment | vsearch | Per-ASV taxonomy TSV | +| Clustering | cd-hit-est | Clustered ASV representatives | +| Merge + filter | Julia | Combined taxonomy/count table, protist-filtered output | + +## Prerequisites + +- **Julia** ≥ 1.0 - installed automatically by `install.sh` if missing +- **R** ≥ 4.0 - must be installed before running `install.sh` + - Ubuntu/Debian: `sudo apt install r-base` + - macOS: `brew install r` or [CRAN package](https://cran.r-project.org/bin/macosx/) + +## Installation + +```bash +git clone https://github.com/JoshuaJewell/MetaManifold.git +cd MetaManifold +bash install.sh +``` + +`install.sh` will check for Julia and R, install Julia and R dependencies, locate or download each external tool (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est). Tool paths can be configured manually in `config/tools.yml`, including by SSH if you wish to use a server-hosted binary. + +To update: +```bash +bash install.sh --update +``` + +### Tool paths + +`install.sh` generates `config/tools.yml`. You can edit it manually, for example, to point a tool at a remote server: + +```yaml +vsearch: + path: "user@bioserver:/home/user/software/vsearch" +``` + +When a remote SSH path is set, the pipeline routes that tool's invocations through +`ssh`. `config/tools.example.yml` contains full config format. + +## Configuration + +All configuration lives in `config/`: + +| File | Purpose | +|------|---------| +| `primers.yml` | Primer sequences and pair definitions | +| `dada2.yml` | DADA2 pipeline parameters (truncation, error model, taxonomy DB, etc.) | +| `protist_filter.yml` | Taxonomic filtering rules for protist output | +| `tools.yml` | Tool paths (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est) | + +### Defining primer pairs `primers.yml` + + Maps primer names to sequences and defines which forward/reverse sequences constitute a pair: + +```yaml +Forward: + PrimerF: "CCAGCASCYGCGGTAATTCC" + +Reverse: + Primer1R: "ACTTTCGTTCTTGATYRA" + Primer2R: "DCTKTCGTYCTTGATYRA" + +Pairs: + - PrimerPair1: + - PrimerF + - Primer1R + - PrimerPair2: + - PrimerF + - Primer2R +``` + +This allows you to referenece one or multiple primer pairs within the pipeline by name. This allows you to store all primer pairs you often work on in one place and then refer to them in whichever combinations you need as and when. Shared primers across pairs (same forward in two pairs, as the above example) are automatically deduplicated in the cutadapt invocation since otherwise it complains a bit. + +### Configuring DADA2 (`dada2.yml`) + +> **Performance tip:** For large datasets and reference databases, consider installing the [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides acceleration for CPU and Nvidia CUDA GPUs for taxonomy assignment, with no changes to the API or configuration required. This fork is experimental, so if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. + +```yaml +workspace: + root: "./output/dada2/" # output directories are created here + input_dir: "/path/to/fastqs" # trimmed FASTQ input (cutadapt output) + +file_patterns: + forward: "_R1_trimmed.fastq.gz" + reverse: "_R2_trimmed.fastq.gz" + sample_name_split: "_" # character to split filenames on + sample_name_index: 1 # which element is the sample name (1-based) + mode: "paired" # paired | forward | reverse + +##Filter and trim - corresponds to DADA2's `filterAndTrim()`: + +filter_trim: + trunc_q: 2 # truncate reads at first base with quality less than this + trunc_len: [220, 220] # truncate F and R reads to this length; single value for single-end + max_ee: [3, 3] # maximum errors permitted in F and R reads + min_len: 175 # discard reads shorter than this after truncation + max_n: 0 # discard reads containing any ambiguous bases + match_ids: true # require F/R read ID's to match + rm_phix: true # remove PhiX spike-in reads + +##Denoising - corresponds to `learnErrors()` and `dada()`: + +dada: + seed: 123 # random seed for reproducibility + nbases: 200000000 # bases used to learn the error model + max_consist: 15 # error model convergence iterations + pool_method: "pseudo" # none | pseudo | true + # pseudo improves sensitivity for rare variants across samples + +## Merging - `mergePairs()`, paired mode only: + +merge: + min_overlap: 20 # minimum overlap between F and R reads + max_mismatch: 0 # mismatches permitted in the overlap region + trim_overhang: true + +## ASV length filtering and chimera removal: + +asv: + band_size_min: 250 # retain only ASVs within this length range (null to skip) + band_size_max: 256 + denovo_method: "consensus" # chimera removal method: consensus | pooled | per-sample + +## Taxonomy - `assignTaxonomy()` against a reference database: + +taxonomy: + skip: false # set true to skip taxonomy and output SeqName + Sequence + counts only + uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" + # URL or local path to a DADA2-formatted reference FASTA + multithread: true + min_boot: 0 # minimum bootstrap confidence to retain an assignment (0–100) + levels: # taxonomic ranks in the reference database (must match its headers) + - "Domain" + - "Supergroup" + - "Division" + - "Subdivision" + - "Class" + - "Order" + - "Family" + - "Genus" + - "Species" + +## Output + +output: + bootstraps: "combined" # none | combined (appended columns) | separate (second sheet) + combined_mode: "regular" # regular: taxonomy + counts | alternative: SeqName + Sequence + counts + seq_table_prefix: "seqtab_nochim" + fasta_prefix: "asvs" + taxa_prefix: "taxonomy" + combined_filename: "tax_counts.xlsx" +``` + +### Configuring taxonomic filtering (`protist_filter.yml`) + +The filter file controls the `filter_table()` step, which removes non-target taxa +and remaps Supergroup labels for consistency with PR2 division names. + +> **Database compatibility:** The default config is tuned specifically for [PR2](https://pr2-database.org/). Column names (`Supergroup`, `Division`, `Subdivision`) and pattern strings (`Eukaryota:plas`, `TSAR:chro`, etc.) reflect PR2's rank structure and nomenclature. If you use a different reference database, you will need to update both the column names here and the `levels` list in `dada2.yml` to match that database's ranks. + +**Division → Supergroup remapping** + +PR2 assigns a `Supergroup` label that can be coarser than `Division` for some lineages. The `mappings` block overrides `Supergroup` with the `Division` value where they should be treated as equivalent: + +```yaml +mappings: + Rhizaria: Rhizaria + Alveolata: Alveolata + Stramenopiles: Stramenopiles + # add further Division: Supergroup pairs as needed +``` + +**Exclusion filters** + +Each entry specifies a column and a substring. Rows where that column contains the substring (partial match) are removed: + +```yaml +filters: + - column: Domain + pattern: Bacteria + - column: Domain + pattern: Eukaryota:plas + - column: Domain + pattern: Eukaryota:mito + - column: Subdivision + pattern: Metazoa + - column: Subdivision + pattern: Fungi + # add further column/pattern pairs to exclude additional lineages +``` + +## Usage + +Edit `src/main.jl` to set your parameters, then uncomment or add pipeline steps you want to run, e.g.: + +```julia +# Remove primers from Illumina reads +cutadapt(primer_pairs, primers_config, fastq_input_dir, + cutadapt_dir, optional_args = optional_args) + +# Run dada2 denoising, chimera removal, etc. in accordance with config +dada2(dada2_config) + +# Use vsearch local alignment to create a taxonomy table +vsearch(fasta) + +# Merge vsearch and idtax tables by ASV +merged = merge_taxonomy_counts(vsearch_tsv, dada2_csv) + +# Output table of all ID'd reads and table of reads for protists +CSV.write(merged_outfile_name, merged) +CSV.write(filtered_outfile_name, filter_table(merged, protist_filter_path)) +``` + +Run from the project root: + +```bash +julia --project=. src/main.jl +``` + +### Input data + +Place paired-end FASTQ files in `data/fastq/` following Illumina naming: + +``` +SampleName_*_L001_R1_001.fastq.gz +SampleName_*_L001_R2_001.fastq.gz +``` + +### Output structure + +``` +output/ +├── cutadapt/ # Trimmed FASTQ pairs and per-run logs +├── dada2/ # ASV count table, FASTA, taxonomy CSV, pipeline stats +└── vsearch/ # Taxonomy TSV files +merged_multi.csv # Merged DADA2 + vsearch results +protist_filtered.csv # After taxonomic filtering +``` + +## Third-party tools + +This project orchestrates the following tools. Each is fetched from its upstream source by `install.sh` and is subject to its own license - no third-party binaries are included in this repository. + +| Tool | License | Source | +|------|---------|--------| +| [cutadapt](https://github.com/marcelm/cutadapt) | MIT | PyPI | +| [FastQC](https://github.com/s-andrews/FastQC) | GPL v3 | Babraham Bioinformatics | +| [MultiQC](https://github.com/MultiQC/MultiQC) | GPL v3 | PyPI | +| [vsearch](https://github.com/torognes/vsearch) | GPL v3 | GitHub Releases | +| [cd-hit](https://github.com/weizhongli/cdhit) | GPL v2+ | GitHub Releases / apt | +| [DADA2](https://benjjneb.github.io/dada2/) ([optimised fork](https://github.com/JoshuaJewell/dada2)) | LGPL v3 | Bioconductor | + +## Acknowledgements + +This pipeline draws on the following prior work: + +- **Frédéric Mahé** - [Fred's metabarcoding pipeline](https://github.com/frederic-mahe/swarm/wiki/Fred's-metabarcoding-pipeline) informed the overall workflow architecture: the sequencing of primer trimming, vsearch-based taxonomy assignment, and final table merge/filter stages. Fred's pipeline uses swarm + OTUs where MetaManifold uses DADA2 + ASVs. +- **Benjamin J. Callahan _et al._** - [DADA2 tutorial](https://benjjneb.github.io/dada2/tutorial.html), used under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), which the `dada2.r` module is based on. + +The following colleagues at the **Department of Parasitology, Charles University** (Faculty of Science, BIOCEV, Vestec, Czech Republic) contributed to this work: + +- **Mgr. Jiří Novák** (supervisor) - scripts from which several modules and configurations were adapted: + - `run_cutadapt.jl` + - `merge_and_filter_taxa.jl` + - `dada2.yml` + - `protist_filter.yml` +- **Bc. Ekaterina Kandaurova** - designed the primer pairs in `primers.yml`. +- **doc. Mgr. Vladimír Hampl** - provided laboratory access and resources. + +## License + +Copyright © 2026 Joshua Benjamin Jewell. + +Source code is licensed under the [GNU Affero General Public License v3.0](LICENSE). + +This documentation (README.md) is licensed under [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). \ No newline at end of file diff --git a/config/dada2.yml b/config/dada2.yml index 13b01a6..64c53ee 100644 --- a/config/dada2.yml +++ b/config/dada2.yml @@ -1,5 +1,5 @@ workspace: - root: "/path/to/project" + root: "./output/dada2/" input_dir: "/path/to/fastqs" file_patterns: diff --git a/config/tools.example.yml b/config/tools.example.yml new file mode 100644 index 0000000..d0ada89 --- /dev/null +++ b/config/tools.example.yml @@ -0,0 +1,28 @@ +# Template for tool path configuration. Copy to config/tools.yml and edit as +# needed, or run install.sh to generate it automatically. +# +# Path options: +# null - use the managed install in bin/ (download by install.sh) +# "/path/to/tool" - absolute local path +# "tool" - bare name; looked up in PATH at runtime +# "user@host:/path" - remote SSH path; the pipeline wraps the call with ssh +# +# Note: config/tools.yml is gitignored (machine-specific). Commit +# config/tools.example.yml instead. + +cutadapt: + path: null + +fastqc: + path: null + +multiqc: + path: null + +vsearch: + path: null + # Remote example (vsearch on a compute server): + # path: "user@bioserver:/home/user/software/vsearch" + +cd_hit_est: + path: null diff --git a/install.jl b/install.jl new file mode 100644 index 0000000..89df766 --- /dev/null +++ b/install.jl @@ -0,0 +1,461 @@ +#!/usr/bin/env julia +# +# Dependency installer for MetabarcodingPipeline +# +# Installs Julia deps, checks/downloads external CLI tools, and installs +# required R packages. Writes resolved tool paths to config/tools.yml. +# +# Usage via install.sh, or directly by: +# julia --project=. install.jl [--update] +# +# Options: +# --update Re-check tool versions and update managed binaries in bin/ + +using Pkg +@info "Installing Julia package dependencies..." +Pkg.instantiate() + +using YAML +import Downloads + +## Instantiate +const UPDATE_MODE = "--update" in ARGS +const PROJECT_ROOT = @__DIR__ +const BIN_DIR = joinpath(PROJECT_ROOT, "bin") +const CONFIG_DIR = joinpath(PROJECT_ROOT, "config") +const TOOLS_CONFIG = joinpath(CONFIG_DIR, "tools.yml") + +const OS_TYPE = Sys.islinux() ? "linux" : + Sys.isapple() ? "macos" : + error("Unsupported OS. Only Linux and macOS are supported.") + +const ARCH_STR = Sys.ARCH == :x86_64 ? "x86_64" : + Sys.ARCH == :aarch64 ? "aarch64" : + string(Sys.ARCH) + +# Canonical binary name for tools whose config key differs from the binary name. +const BINARY_NAMES = Dict("cd_hit_est" => "cd-hit-est") +bin_name(key::String) = get(BINARY_NAMES, key, key) + +mkpath(BIN_DIR) + +## Config loading / saving +function load_tools_config()::Dict{String,Any} + isfile(TOOLS_CONFIG) || return Dict{String,Any}() + data = YAML.load_file(TOOLS_CONFIG) + data isa Dict ? data : Dict{String,Any}() +end + +function write_tools_config(config::Dict) + mkpath(CONFIG_DIR) + open(TOOLS_CONFIG, "w") do io + for key in sort(collect(keys(config))) + val = config[key] + path = val isa Dict ? get(val, "path", nothing) : val + println(io, "$key:") + if path === nothing + println(io, " path: null") + else + # Quote the path to handle spaces and special characters + println(io, " path: \"$path\"") + end + end + end + @info "Config written to $TOOLS_CONFIG" +end + +## Checking for tool +# Returns true if the binary at `path` is callable (local or remote SSH). +function check_tool(path::String)::Bool + if occursin('@', path) + # Remote SSH path: user@host:/path/to/binary + colon_idx = findfirst(':', path) + colon_idx === nothing && return false + user_host = path[1:colon_idx-1] + bin_path = path[colon_idx+1:end] + try + run(pipeline( + `ssh -o BatchMode=yes -o ConnectTimeout=5 $user_host test -x $bin_path`; + stdout=devnull, stderr=devnull + )) + return true + catch + return false + end + else + # Local path or bare name (PATH lookup) + resolved = isfile(path) ? path : Sys.which(path) + resolved === nothing && return false + return Sys.isexecutable(resolved) + end +end + +"""Return the full path to `name` if it is in PATH, otherwise nothing.""" +function find_in_path(name::String) + Sys.which(name) +end + +## Interactive prompts +function prompt_yn(question::String, default_yes::Bool = true)::Bool + hint = default_yes ? "[Y/n]" : "[y/N]" + print(" $question $hint: ") + answer = strip(readline()) + isempty(answer) && return default_yes + return lowercase(answer) in ("y", "yes") +end + +function prompt_path(label::String)::Union{String,Nothing} + print(" $label: ") + p = strip(readline()) + isempty(p) ? nothing : p +end + +## Download helpers +# Fetch URL content as a String. Returns nothing on failure. +function fetch_string(url::String)::Union{String,Nothing} + try + buf = IOBuffer() + Downloads.download(url, buf; headers=["User-Agent" => "MetabarcodingPipeline-installer"]) + String(take!(buf)) + catch e + @warn "Could not fetch $url: $e" + nothing + end +end + +# Return the first capture group of `pattern` in `s`, or nothing. +function first_match(pattern::Regex, s::String)::Union{String,Nothing} + m = match(pattern, s) + m === nothing ? nothing : m[1] +end + +# Download `url` to `dest`. +function download_to(url::String, dest::String) + @info "Downloading $(basename(url))..." + Downloads.download(url, dest) +end + +# Recursively check `dir` and return the first file named `filename`, or nothing. +function find_file_in_dir(dir::String, filename::String)::Union{String,Nothing} + for (root, _dirs, files) in walkdir(dir) + idx = findfirst(==(filename), files) + idx !== nothing && return joinpath(root, files[idx]) + end + nothing +end + +# Tool download functions + +function download_vsearch()::String + @info "Fetching latest vsearch release info from GitHub..." + json = fetch_string("https://api.github.com/repos/torognes/vsearch/releases/latest") + json === nothing && error("Cannot reach GitHub API. Check your internet connection.") + + # Find a download URL matching OS and architecture + pattern = Regex( + "\"browser_download_url\":\\s*\"(https://[^\"]*vsearch[^\"]*$(OS_TYPE)[^\"]*$(ARCH_STR)[^\"]*.tar.gz)\"" + ) + url = first_match(pattern, json) + + # Fallback to any tar.gz for this OS + if url === nothing + url = first_match( + Regex("\"browser_download_url\":\\s*\"(https://[^\"]*vsearch[^\"]*$(OS_TYPE)[^\"]*.tar.gz)\""), + json + ) + end + + url === nothing && error( + "No vsearch binary found for $OS_TYPE/$ARCH_STR in the latest release.\n" * + "Check https://github.com/torognes/vsearch/releases for available builds." + ) + + tarball = joinpath(BIN_DIR, "vsearch_download.tar.gz") + download_to(url, tarball) + run(`tar -xzf $tarball -C $BIN_DIR`) + rm(tarball) + + bin = find_file_in_dir(BIN_DIR, "vsearch") + bin === nothing && error("vsearch binary not found after extraction.") + + dest = joinpath(BIN_DIR, "vsearch") + bin != dest && mv(bin, dest; force=true) + chmod(dest, 0o755) + dest +end + +function download_fastqc()::String + # Pin to a known-good version; update periodically. + version = "0.12.1" + url = "https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v$(version).zip" + + zipfile = joinpath(BIN_DIR, "fastqc.zip") + download_to(url, zipfile) + run(`unzip -q -o $zipfile -d $BIN_DIR`) + rm(zipfile) + + bin = joinpath(BIN_DIR, "FastQC", "fastqc") + isfile(bin) || error("fastqc not found after extraction. Expected at $bin") + chmod(bin, 0o755) + bin +end + +function download_cdhit()::String + # System package manager (no compilation needed) + if OS_TYPE == "linux" && Sys.which("apt-get") !== nothing + @info "Trying apt-get install cd-hit..." + try + run(`sudo apt-get install -y cd-hit`) + found = Sys.which("cd-hit-est") + found !== nothing && return found + catch + @warn "apt-get install failed (no sudo?), falling back to binary download." + end + elseif OS_TYPE == "macos" && Sys.which("brew") !== nothing + @info "Trying brew install cd-hit..." + try + run(`brew install cd-hit`) + found = Sys.which("cd-hit-est") + found !== nothing && return found + catch + @warn "brew install failed, falling back to binary download." + end + end + + # Fallback to download precompiled binary. + if OS_TYPE == "linux" + url = "https://github.com/weizhongli/cdhit/releases/download/V4.8.1/cd-hit-v4.8.1-2019-0228-Linux.tar.gz" + else + error( + "No managed cd-hit download available for macOS.\n" * + "Install via: brew install cd-hit\n" * + "Then enter the path to cd-hit-est when prompted." + ) + end + + @info "Downloading cd-hit v4.8.1 precompiled binary..." + tarball = joinpath(BIN_DIR, "cdhit_download.tar.gz") + download_to(url, tarball) + run(`tar -xzf $tarball -C $BIN_DIR`) + rm(tarball) + + bin = find_file_in_dir(BIN_DIR, "cd-hit-est") + bin === nothing && error("cd-hit-est binary not found after extraction.") + + dest = joinpath(BIN_DIR, "cd-hit-est") + bin != dest && mv(bin, dest; force=true) + chmod(dest, 0o755) + dest +end + +function install_python_tool(name::String)::String + # pipx (recommended on PEP 668 / Debian-managed systems) + if Sys.which("pipx") !== nothing + @info "Installing $name via pipx..." + run(`pipx install $name`) + found = find_in_path(name) + found !== nothing && return found + # pipx installs to ~/.local/bin by default + local_bin = joinpath(homedir(), ".local", "bin", name) + isfile(local_bin) && return local_bin + @warn "Installed $name via pipx but could not locate the binary. Ensure ~/.local/bin is in PATH." + return name + end + + # Fallback to pip --user, then --break-system-packages if blocked + pip_cmd = nothing + for candidate in (`pip3`, `pip`, `python3 -m pip`) + try + run(pipeline(`$candidate --version`; stdout=devnull, stderr=devnull)) + pip_cmd = candidate + break + catch + end + end + pip_cmd === nothing && error( + "Neither pipx nor pip found. Install pipx (recommended) or Python 3 with pip." + ) + + @info "Installing $name via pip..." + success = try + run(`$pip_cmd install --user $name`) + true + catch + false + end + + if !success + # Fallback 2 to PEP 668: externally-managed environment, try --break-system-packages + @warn "pip --user blocked by system policy. Retrying with --break-system-packages..." + run(`$pip_cmd install --user --break-system-packages $name`) + end + + # Locate the installed binary + found = find_in_path(name) + found !== nothing && return found + + local_bin = joinpath(homedir(), ".local", "bin", name) + isfile(local_bin) && return local_bin + + @warn "Installed $name but could not locate the binary. Ensure ~/.local/bin is in PATH." + name +end + +function install_r_packages(packages::Vector{String}) + pkgs_r = join(["\"$p\"" for p in packages], ", ") + snippet = """ + if (!requireNamespace("BiocManager", quietly = TRUE)) + install.packages("BiocManager", repos = "https://cloud.r-project.org") + pkgs <- c($pkgs_r) + missing <- pkgs[!sapply(pkgs, requireNamespace, quietly = TRUE)] + if (length(missing) > 0) { + message("Installing: ", paste(missing, collapse = ", ")) + BiocManager::install(missing, ask = FALSE) + } else { + message("All R packages already installed.") + } + """ + try + run(`Rscript -e $snippet`) + @info "R packages installed successfully." + catch e + @error "R package installation failed: $e" + end +end + +## Per-tool resolution + +# Resolves a single tool interactively. Returns the resolved path string, or nothing +# if the user chose to skip. When `install_fn` is provided, offers an auto-install +# option as the first choice. +function resolve_tool( + key::String, + display_name::String, + existing_config::Dict, + install_fn::Union{Function,Nothing} = nothing +)::Union{String,Nothing} + println() + println(" ─── $display_name ─────────────────────────────────────────") + + bin = bin_name(key) + + # Check existing config (skip in update mode for managed installs) + existing = get(existing_config, key, nothing) + existing_path = existing isa Dict ? get(existing, "path", nothing) : nothing + + if existing_path !== nothing && !UPDATE_MODE + println(" Configured path: $existing_path") + if check_tool(string(existing_path)) + prompt_yn(" Use this?") && return string(existing_path) + else + println(" Warning: configured path does not appear to be callable.") + end + end + + # Check PATH + path_result = find_in_path(bin) + if path_result !== nothing + println(" Found in PATH: $path_result") + prompt_yn(" Use this?") && return path_result + end + + # Build option list + options = String[] + if install_fn !== nothing + push!(options, "Install/download automatically to bin/") + end + push!(options, "Enter a path manually (local: /path/to/$bin or remote: user@host:/path/to/$bin)") + push!(options, "Skip (configure later in config/tools.yml)") + + for (i, opt) in enumerate(options) + println(" $i) $opt") + end + + print(" Choice [1]: ") + raw = strip(readline()) + choice = isempty(raw) ? 1 : something(tryparse(Int, raw), 1) + + if install_fn !== nothing + if choice == 1 + try + return install_fn() + catch e + @error "Auto-install failed: $e" + println(" Falling back to manual entry.") + choice = 2 + end + end + # After potential fallback, re-map remaining choices + if choice == 2 + return prompt_path(" Enter path") + end + return nothing # Skip + else + if choice == 1 + return prompt_path(" Enter path") + end + return nothing # Skip + end +end + +# Main + +function main() + println() + println("╔═══════════════════════════════════════════╗") + println("║ MetabarcodingPipeline — Install Script ║") + println("╚═══════════════════════════════════════════╝") + UPDATE_MODE && println(" Mode: UPDATE") + println() + + config = load_tools_config() + resolved = Dict{String,Any}() + + # cutadapt + path = resolve_tool("cutadapt", "cutadapt", config, + () -> install_python_tool("cutadapt")) + resolved["cutadapt"] = Dict("path" => path) + + # fastqc + path = resolve_tool("fastqc", "FastQC", config, + () -> download_fastqc()) + resolved["fastqc"] = Dict("path" => path) + + # multiqc + path = resolve_tool("multiqc", "MultiQC", config, + () -> install_python_tool("multiqc")) + resolved["multiqc"] = Dict("path" => path) + + # vsearch + path = resolve_tool("vsearch", "vsearch", config, + () -> download_vsearch()) + resolved["vsearch"] = Dict("path" => path) + + # cd-hit-est + path = resolve_tool("cd_hit_est", "cd-hit-est", config, + () -> download_cdhit()) + resolved["cd_hit_est"] = Dict("path" => path) + + # R packages + println() + println(" ─── R packages ─────────────────────────────────────────────") + r_packages = ["dada2", "openxlsx", "tidyverse", "yaml"] + if prompt_yn(" Install/check R packages (dada2, openxlsx, tidyverse, yaml)?") + install_r_packages(r_packages) + end + + # Write config + write_tools_config(resolved) + + println() + println("Installation complete.") + println() + println("To set remote SSH paths or adjust any tool locations, edit:") + println(" $TOOLS_CONFIG") + println() + println("Example remote path (SSH):") + println(" vsearch:") + println(" path: \"user@bioserver:/home/user/software/vsearch\"") +end + +main() \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..acb44c6 --- /dev/null +++ b/install.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Bootstrap installer for MetabarcodingPipeline +# +# Checks for Julia and R, installs Julia via juliaup if missing, +# then hands off to install.jl for all further dependency setup. +# +# Usage: +# bash install.sh [--update] +# +# Options: +# --update Re-check tool versions and update managed binaries in bin/ + +set -euo pipefail + +# OS detection + +OS="$(uname -s)" +case "$OS" in + Linux*) OS_TYPE="Linux" ;; + Darwin*) OS_TYPE="macOS" ;; + *) + echo "Unsupported OS: $OS" + echo "This installer supports Linux and macOS only." + exit 1 + ;; +esac + +echo "Detected OS: $OS_TYPE" +echo "" + +# Julia check and install + +if command -v julia &>/dev/null; then + echo "Found Julia: $(julia --version)" +else + echo "Julia not found. Installing via juliaup..." + curl -fsSL https://install.julialang.org | sh -s -- --yes + + # juliaup installs to ~/.juliaup; source the env file if present + if [ -f "$HOME/.juliaup/env" ]; then + # shellcheck disable=SC1091 + source "$HOME/.juliaup/env" + fi + export PATH="$HOME/.juliaup/bin:$PATH" + + if command -v julia &>/dev/null; then + echo "Julia installed: $(julia --version)" + else + echo "" + echo "Julia was installed but is not yet in PATH." + echo "Please open a new terminal and re-run this script, or install Julia" + echo "manually from https://julialang.org/downloads/ and try again." + exit 1 + fi +fi + +# R check and politely ask user to do it for us + +if command -v Rscript &>/dev/null; then + echo "Found R: $(Rscript --version 2>&1 | head -1)" +else + echo "" + echo "R is not installed. Please install R for your system and re-run this script." + echo "" + if [ "$OS_TYPE" = "Linux" ]; then + echo " Ubuntu/Debian: https://cran.r-project.org/bin/linux/ubuntu/" + echo " Fedora/RHEL: https://cran.r-project.org/bin/linux/fedora/" + echo " Quick install: sudo apt install r-base (Debian/Ubuntu)" + elif [ "$OS_TYPE" = "macOS" ]; then + echo " macOS pkg: https://cran.r-project.org/bin/macosx/" + echo " Homebrew: brew install r" + fi + echo "" + exit 1 +fi + +# Dependency installs by Julia + +echo "" +echo "Running install.jl..." +echo "" +julia --project=. install.jl "$@" diff --git a/src/main.jl b/src/main.jl index bb83bc7..9da707a 100644 --- a/src/main.jl +++ b/src/main.jl @@ -1,10 +1,39 @@ +#!/usr/bin/env julia + include("run_cutadapt.jl") include("run_dada2.jl") include("merge_and_filter_taxa.jl") using CSV +using YAML using .Cutadapt, .TaxonomyTableTools, .DADA2 +## Tools loading (to move to new module at some point) +""" + load_tools(config_path) -> Dict{String,String} + +Read config/tools.yml and return a Dict of tool name => resolved path. +If the file does not exist, returns an empty Dict so tools fall back to PATH. +Paths with `@` are SSH remote paths (user@host:/path), the calling module is +responsible for routing those calls via SSH. +""" +function load_tools(config_path = joinpath(@__DIR__, "..", "config", "tools.yml")) + isfile(config_path) || return Dict{String,String}() + data = YAML.load_file(config_path) + tools = Dict{String,String}() + for (name, info) in data + path = info isa Dict ? get(info, "path", nothing) : nothing + path !== nothing && (tools[name] = string(path)) + end + tools +end + +# Return the configured binary path for `tool_key`, or `default` if not set. +tool_bin(tools, tool_key, default = tool_key) = get(tools, tool_key, default) + +# Load resolved tool paths from config/tools.yml (empty Dict if not present) +tools = load_tools() + ## Instantiate filesystem # Root directories data_dir = "./data" @@ -40,7 +69,8 @@ filtered_outfile_vespa = joinpath(output_dir, "protist_filtered_vespa.csv") ## Main -#cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, optional_args = optional_args) +#cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, +# optional_args = optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) #dada2(dada2_config_dir) diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index 4600c6c..f10d5cb 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -195,6 +195,12 @@ export merge_taxonomy_counts, filter_table ismissing(d) ? s : (haskey(mapping, string(d)) ? mapping[string(d)] : s) for (d, s) in zip(df.Division, df.Supergroup) ] + elseif !isempty(mapping) + missing_cols = filter(c -> c ∉ names(df), ["Division", "Supergroup"]) + @warn "Supergroup remapping skipped: column(s) not present in data: " * + "$(join(missing_cols, ", ")). The mappings block in your filter config " * + "may be tuned for a different reference database (e.g. PR2). " * + "Available columns: $(join(names(df), ", "))" end # Load remove_empty_domain flag @@ -204,12 +210,16 @@ export merge_taxonomy_counts, filter_table end # Optionally remove blank/unassigned Domain entries - if remove_empty && ("Domain" in names(df)) + if remove_empty && "Domain" in names(df) df = filter(row -> !ismissing(row.Domain) && !isempty(strip(string(row.Domain))) && lowercase(strip(string(row.Domain))) != "blank", df) + elseif remove_empty + @warn "remove_empty_domain skipped: 'Domain' column not present in data. " * + "This setting may be tuned for a different reference database (e.g. PR2). " * + "Available columns: $(join(names(df), ", "))" end # Load and apply filters @@ -224,7 +234,9 @@ export merge_taxonomy_counts, filter_table @assert typeof(pattern) <: AbstractString "filter pattern must be a string" if !(colname in names(df)) - @warn "Skipping filter: column '$colname' not found in data." + @warn "Skipping filter: column '$colname' not present in data. " * + "This filter may be tuned for a different reference database (e.g. PR2). " * + "Available columns: $(join(names(df), ", "))" continue end diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index c345cd2..8b82ae8 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -58,7 +58,7 @@ export cutadapt return chop(args) end - function run_cutadapt(primer_args, optional_args, fastq_in_dir, cutadapt_dir) + function run_cutadapt(primer_args, optional_args, fastq_in_dir, cutadapt_dir, cutadapt_bin) samples = [] # Messy filesystem stuff @@ -88,7 +88,7 @@ export cutadapt outputR2 = fastq_out_dir * sample * "_R2_trimmed.fastq.gz" @info("On sample $i/$nsamples ($sample).") - cutadapt_cmd = "cutadapt $primer_args $optional_args -o $outputR1 -p $outputR2 $inputR1 $inputR2" + cutadapt_cmd = "$cutadapt_bin $primer_args $optional_args -o $outputR1 -p $outputR2 $inputR1 $inputR2" open(stats_path, "a") do io run(pipeline(`bash -lc $cutadapt_cmd`; stdout=io, stderr=io)) @@ -143,20 +143,23 @@ export cutadapt ## Keyword Arguments - `optional_args` (optional, default: "-m 200 --discard-untrimmed"): Specify additional arguments passed to `cutadapt` command. + - `cutadapt_bin` (optional, default: "cutadapt"): Path to the cutadapt binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. """ function cutadapt( primer_pairs, - primers_path, - fastq_in_dir, - cutadapt_dir; - optional_args = "-m 200 --discard-untrimmed" + primers_path, + fastq_in_dir, + cutadapt_dir; + optional_args = "-m 200 --discard-untrimmed", + cutadapt_bin = "cutadapt" ) run_cutadapt( get_primer_args(primer_pairs, primers_path), optional_args, fastq_in_dir, - cutadapt_dir + cutadapt_dir, + cutadapt_bin ) end end \ No newline at end of file From 5913667df28042e0d7eebcf65bfba54ca1cdebcb Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 19 Feb 2026 10:39:29 +0100 Subject: [PATCH 014/175] Convert dada2.r orchestrator to use a real language, making the world a better place one less R module at a time. --- src/dada2.jl | 594 ++++++++++++++++++++++++++++++++++++++++++ src/dada2_functions.r | 214 +++------------ src/main.jl | 2 +- src/run_dada2.jl | 4 +- 4 files changed, 637 insertions(+), 177 deletions(-) create mode 100644 src/dada2.jl diff --git a/src/dada2.jl b/src/dada2.jl new file mode 100644 index 0000000..a024d13 --- /dev/null +++ b/src/dada2.jl @@ -0,0 +1,594 @@ +module DADA2 + +# DADA2 amplicon sequencing pipeline — Julia orchestrator +# +# The pipeline is split into six independently callable stages so that +# intermediate outputs (quality profiles, error rate plots, length +# distributions, pipeline stats) can be reviewed and parameters adjusted +# before committing to the next step. Each stage saves its R objects to a +# per-stage checkpoint in Analysis/ and loads what it needs from the previous +# stage's checkpoint, making every stage re-runnable across Julia sessions. +# +# Stage order: +# prefilter_qc → filter_trim → learn_errors → +# denoise → chimera_removal → assign_taxonomy +# +# Call dada2() to run all stages in sequence without stopping. +# +# Notice: +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). +# +# This work is based on the DADA2 tutorial by Benjamin J. Callahan, et al., +# available at https://benjjneb.github.io/dada2/tutorial.html, with modification +# into a single module. The original material is licensed under the Creative +# Commons Attribution 4.0 International License (CC BY 4.0): +# https://creativecommons.org/licenses/by/4.0/. + +export dada2, prefilter_qc, filter_trim, learn_errors, denoise, + chimera_removal, assign_taxonomy + + using Logging, RCall, YAML + + # Helpers + + # Returns a function that routes progress messages to @info (CLI) or a + # Channel{String} (Genie SSE). Pass the result as `emit` in stage functions. + _emitter(::Nothing) = msg -> @info msg + _emitter(ch::Channel{String}) = msg -> put!(ch, msg) + + # Config + function validate_config(cfg) + required = ["workspace", "file_patterns", "filter_trim", "dada", + "merge", "asv", "taxonomy", "output"] + missing_secs = filter(k -> !haskey(cfg, k), required) + isempty(missing_secs) || + error("Missing required config sections: $(join(missing_secs, ", "))") + + ws = cfg["workspace"] + haskey(ws, "root") || error("workspace.root is required") + haskey(ws, "input_dir") || error("workspace.input_dir is required") + isdir(ws["input_dir"]) || + error("workspace.input_dir does not exist: $(ws["input_dir"])") + + mode = get(cfg["file_patterns"], "mode", "paired") + mode in ("paired", "forward", "reverse") || + error("file_patterns.mode must be one of: paired, forward, reverse") + + boot_mode = get(cfg["output"], "bootstraps", "combined") + boot_mode in ("none", "combined", "separate") || + error("output.bootstraps must be one of: none, combined, separate") + + if !get(cfg["taxonomy"], "skip", false) && !haskey(cfg["taxonomy"], "uri") + error("taxonomy.uri is required when taxonomy.skip is not true") + end + + combined_mode = get(cfg["output"], "combined_mode", "regular") + combined_mode in ("regular", "alternative") || + error("output.combined_mode must be one of: regular, alternative") + end + + # File discovery + function find_fastq_files(input_dir, fwd_pattern, rev_pattern, mode) + all_files = sort(readdir(input_dir, join=true)) + fwd = mode in ("paired", "forward") ? + filter(f -> !isnothing(match(Regex(fwd_pattern), basename(f))), all_files) : String[] + rev = mode in ("paired", "reverse") ? + filter(f -> !isnothing(match(Regex(rev_pattern), basename(f))), all_files) : String[] + fwd, rev + end + + function validate_sample_files(fwd, rev, mode) + if mode in ("paired", "forward") + isempty(fwd) && + error("No forward FASTQ files found. " * + "Check workspace.input_dir and file_patterns.forward.") + for f in fwd + isfile(f) || error("Forward file not found: $f") + end + end + if mode in ("paired", "reverse") + isempty(rev) && + error("No reverse FASTQ files found. " * + "Check workspace.input_dir and file_patterns.reverse.") + for f in rev + isfile(f) || error("Reverse file not found: $f") + end + end + if mode == "paired" && length(fwd) != length(rev) + error("Forward file count ($(length(fwd))) does not match " * + "reverse file count ($(length(rev))).") + end + end + + function extract_sample_names(files, split_char, split_index) + [split(basename(f), split_char)[split_index] for f in files] + end + + # Workspace + function setup_workspace(root) + dirs = Dict( + "Tables" => joinpath(root, "Tables"), + "Analysis" => joinpath(root, "Analysis"), + "Taxonomy" => joinpath(root, "Taxonomy"), + "Figures" => joinpath(root, "Figures"), + "Filtered" => joinpath(root, "Filtered"), + ) + for d in values(dirs) + mkpath(d) + end + dirs + end + + # Pipeline context: + # Shared setup called at the start of every stage: sources R, loads and + # validates config, discovers files, handles the single-sample fallback, and + # computes all path variables. Returns a NamedTuple so stage functions can + # extract what they need without repeating boilerplate. + function _pipeline_context(config_path::String) + functions_r = joinpath(@__DIR__, "dada2_functions.r") + R"source($functions_r)" + + cfg = YAML.load_file(config_path) + validate_config(cfg) + verbose = get(cfg, "verbose", true) + mode = get(cfg["file_patterns"], "mode", "paired") + root = cfg["workspace"]["root"] + dirs = setup_workspace(root) + + fwd_files, rev_files = find_fastq_files( + cfg["workspace"]["input_dir"], + cfg["file_patterns"]["forward"], + cfg["file_patterns"]["reverse"], + mode) + validate_sample_files(fwd_files, rev_files, mode) + + primary_files = isempty(fwd_files) ? rev_files : fwd_files + sample_names = extract_sample_names( + primary_files, + cfg["file_patterns"]["sample_name_split"], + cfg["file_patterns"]["sample_name_index"]) + + # Single-sample fallback: dada() returns a bare object (not a list) for a + # single input file, breaking makeSequenceTable() and sapply() downstream. + # Duplicate the paths so the pipeline sees 2 samples; the extra row is + # dropped in chimera_removal(). ASV calls are unaffected. + single_sample = length(sample_names) == 1 + if single_sample + @warn "Only 1 sample found. Duplicating it to work around dada() " * + "returning a bare object for single-file input. The duplicate " * + "will be dropped from all outputs." + fwd_files = isempty(fwd_files) ? fwd_files : repeat(fwd_files, 2) + rev_files = isempty(rev_files) ? rev_files : repeat(rev_files, 2) + sample_names = [sample_names[1], sample_names[1] * "_dup"] + end + + filtered_dir = dirs["Filtered"] + fwd_out = mode != "reverse" ? + [joinpath(filtered_dir, s * "_R1_filt.fastq.gz") for s in sample_names] : String[] + rev_out = mode != "forward" ? + [joinpath(filtered_dir, s * "_R2_filt.fastq.gz") for s in sample_names] : String[] + + # For single-end modes, route the relevant reads into the forward slots. + in_fwd = mode != "reverse" ? fwd_files : rev_files + out_fwd = mode != "reverse" ? fwd_out : rev_out + in_rev_arg = mode == "paired" ? rev_files : nothing + out_rev_arg = mode == "paired" ? rev_out : nothing + + ckpts = Dict( + "filter" => joinpath(dirs["Analysis"], "ckpt_filter.RData"), + "errors" => joinpath(dirs["Analysis"], "ckpt_errors.RData"), + "denoise" => joinpath(dirs["Analysis"], "ckpt_denoise.RData"), + "chimera" => joinpath(dirs["Analysis"], "ckpt_chimera.RData"), + ) + + (; cfg, verbose, mode, dirs, sample_names, single_sample, + fwd_files, rev_files, fwd_out, rev_out, + in_fwd, out_fwd, in_rev_arg, out_rev_arg, ckpts) + end + + # Pre-filter quality assessment + """ + prefilter_qc(config_path; progress) + + **Stage 1** — Plot unfiltered quality profiles. + + Review `Figures/quality_unfiltered.pdf` to choose `truncLen` and `maxEE` + values in config before running `filter_trim()`. + """ + function prefilter_qc(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + + emit("Plotting unfiltered quality profiles") + fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files + rev_for_plot = isempty(ctx.rev_files) ? nothing : ctx.rev_files + unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") + R"plot_quality_profiles($fwd_for_plot, $rev_for_plot, $unfiltered_pdf)" + emit("Written: $unfiltered_pdf") + nothing + end + + # Filter and trim + """ + filter_trim(config_path; progress) + + **Stage 2** — Filter and trim reads; plot filtered quality profiles. + + Review `Figures/quality_filtered.pdf`. If filtering looks appropriate, + proceed to `learn_errors()`; otherwise adjust `truncLen` / `maxEE` in + config and re-run this stage. + + Saves: `Analysis/ckpt_filter.RData` + """ + function filter_trim(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + ft = ctx.cfg["filter_trim"] + trunc_len = ft["trunc_len"] + max_ee = ft["max_ee"] + verbose = ctx.verbose + in_fwd = ctx.in_fwd + out_fwd = ctx.out_fwd + in_rev = ctx.in_rev_arg + out_rev = ctx.out_rev_arg + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + emit("Filtering and trimming reads") + if ctx.mode == "paired" + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + $in_rev, $out_rev, + truncQ = $(ft["trunc_q"]), + truncLen = $trunc_len, + maxEE = $max_ee, + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + matchIDs = $(ft["match_ids"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + else + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + truncQ = $(ft["trunc_q"]), + truncLen = $(trunc_len[1]), + maxEE = $(max_ee[1]), + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + end + + emit("Plotting filtered quality profiles") + fwd_filt = isempty(fwd_out) ? nothing : fwd_out + rev_filt = isempty(rev_out) ? nothing : rev_out + filtered_pdf = joinpath(ctx.dirs["Figures"], "quality_filtered.pdf") + R"plot_quality_profiles($fwd_filt, $rev_filt, $filtered_pdf)" + + ckpt = ctx.ckpts["filter"] + R"save(filter_stats, file=$ckpt)" + emit("Written: $filtered_pdf") + emit("Checkpoint: $ckpt") + nothing + end + + # Error model + """ + learn_errors(config_path; progress) + + **Stage 3** — Learn substitution error rates and plot diagnostics. + + Review `Figures/error_rates.pdf`: the fitted line should closely follow the + observed points. If not, increase `nbases` or `max_consist` in config and + re-run this stage. + + Saves: `Analysis/ckpt_errors.RData` + """ + function learn_errors(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + seed = get(ctx.cfg["dada"], "seed", 123) + nbases = ctx.cfg["dada"]["nbases"] + max_con = ctx.cfg["dada"]["max_consist"] + verbose = ctx.verbose + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + emit("Learning error rates") + R"set.seed($seed)" + + ctx.mode != "reverse" ? + R"fwd_errors <- learnErrors($fwd_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"fwd_errors <- NULL" + + ctx.mode != "forward" ? + R"rev_errors <- learnErrors($rev_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"rev_errors <- NULL" + + error_pdf = joinpath(ctx.dirs["Figures"], "error_rates.pdf") + R"plot_error_rates(fwd_errors, rev_errors, $error_pdf)" + + ckpt = ctx.ckpts["errors"] + R"save(fwd_errors, rev_errors, file=$ckpt)" + emit("Written: $error_pdf") + emit("Checkpoint: $ckpt") + nothing + end + + # Denoise + """ + denoise(config_path; progress) + + **Stage 4** — Denoise reads, merge pairs (paired mode), build the sequence + table, and plot the raw ASV length distribution. + + Review `Figures/length_distribution.pdf` and set `band_size_min` / + `band_size_max` in config to target the expected amplicon peak. Length + filtering is applied in `chimera_removal()`, so this stage does not need + to be re-run when adjusting the length cutoff. + + Requires: `Analysis/ckpt_errors.RData` + Saves: `Analysis/ckpt_denoise.RData` (unfiltered seq_table) + """ + function denoise(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + verbose = ctx.verbose + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + isfile(ctx.ckpts["errors"]) || + error("Error model checkpoint not found. Run learn_errors() first.") + errors_ckpt = ctx.ckpts["errors"] + R"load($errors_ckpt)" + + emit("Denoising reads") + pool_method = ctx.cfg["dada"]["pool_method"] + + ctx.mode != "reverse" ? + R"dada_fwd <- dada($fwd_out, err=fwd_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_fwd <- NULL" + + ctx.mode != "forward" ? + R"dada_rev <- dada($rev_out, err=rev_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_rev <- NULL" + + emit("Building sequence table") + if ctx.mode == "paired" + min_overlap = ctx.cfg["merge"]["min_overlap"] + max_mismatch = ctx.cfg["merge"]["max_mismatch"] + trim_overhang = ctx.cfg["merge"]["trim_overhang"] + R""" + merged <- mergePairs( + dada_fwd, $fwd_out, + dada_rev, $rev_out, + minOverlap = $min_overlap, + maxMismatch = $max_mismatch, + trimOverhang = $trim_overhang, + verbose = $verbose + ) + seq_table <- makeSequenceTable(merged) + """ + else + R""" + merged <- NULL + seq_table <- makeSequenceTable(if (!is.null(dada_fwd)) dada_fwd else dada_rev) + """ + end + + len_dist_pdf = joinpath(ctx.dirs["Figures"], "length_distribution.pdf") + R"plot_length_distribution(seq_table, $len_dist_pdf)" + + ckpt = ctx.ckpts["denoise"] + R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" + emit("Written: $len_dist_pdf") + emit("Checkpoint: $ckpt") + nothing + end + + # Chimera removal + """ + chimera_removal(config_path; progress) + + **Stage 5** — Optionally filter ASVs by length, remove chimeras, compute + pipeline statistics, and write core output files. + + Length filtering (`band_size_min` / `band_size_max`) is applied here from + the saved unfiltered seq_table, so this stage can be re-run with different + length cutoffs without re-running `denoise()`. + + Review `Tables/pipeline_stats.csv` for unexpected read loss at any stage + before committing to the (potentially long) `assign_taxonomy()` step. + + Requires: `Analysis/ckpt_filter.RData`, `Analysis/ckpt_denoise.RData` + Saves: `Analysis/ckpt_chimera.RData` + """ + function chimera_removal(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + verbose = ctx.verbose + mode = ctx.mode + + isfile(ctx.ckpts["filter"]) || + error("Filter checkpoint not found. Run filter_trim() first.") + isfile(ctx.ckpts["denoise"]) || + error("Denoise checkpoint not found. Run denoise() first.") + filter_ckpt = ctx.ckpts["filter"] + denoise_ckpt = ctx.ckpts["denoise"] + R"load($filter_ckpt)" + R"load($denoise_ckpt)" + + # Length filtering applied from the saved unfiltered seq_table. + # Re-running this stage with new band sizes does not require re-denoising. + band_min = get(ctx.cfg["asv"], "band_size_min", nothing) + band_max = get(ctx.cfg["asv"], "band_size_max", nothing) + if !isnothing(band_min) && !isnothing(band_max) + emit("Filtering by length: $band_min-$band_max bp") + R"seq_table <- filter_by_length(seq_table, $band_min, $band_max)" + len_filt_pdf = joinpath(ctx.dirs["Figures"], "length_distribution_filtered.pdf") + R"plot_length_distribution(seq_table, $len_filt_pdf)" + emit("Written: $len_filt_pdf") + end + + emit("Removing chimeras") + denovo_method = ctx.cfg["asv"]["denovo_method"] + R""" + seq_table_nochim <- removeBimeraDenovo(seq_table, method=$denovo_method, verbose=$verbose) + nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 + message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), + "% | Retained: ", round(nochim_pct, 2), "%") + """ + + # Drop duplicate sample row (single-sample fallback) + final_names = ctx.sample_names + if ctx.single_sample + R"filter_stats <- filter_stats[1, , drop=FALSE]" + mode != "reverse" && R"dada_fwd <- dada_fwd[1]" + mode != "forward" && R"dada_rev <- dada_rev[1]" + mode == "paired" && R"merged <- merged[1]" + R"seq_table_nochim <- seq_table_nochim[1, , drop=FALSE]" + final_names = [ctx.sample_names[1]] + end + + emit("Computing pipeline stats") + stats_csv = joinpath(ctx.dirs["Tables"], "pipeline_stats.csv") + R""" + stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, + seq_table_nochim, $final_names, $mode) + write.csv(stats, $stats_csv, quote=FALSE) + if ($verbose) print(stats) + """ + + emit("Writing core output tables") + seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") + fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") + tables_dir = ctx.dirs["Tables"] + R"write_seq_table(seq_table_nochim, $tables_dir, $seq_prefix)" + R"index <- write_fasta(seq_table_nochim, $tables_dir, $fasta_prefix)" + + ckpt = ctx.ckpts["chimera"] + R"save(seq_table_nochim, index, file=$ckpt)" + emit("Written: $stats_csv") + emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") + emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") + emit("Checkpoint: $ckpt") + nothing + end + + # Taxonomy + """ + assign_taxonomy(config_path; progress) + + **Stage 6** — Assign taxonomy to ASVs and write the combined output table. + + This is typically the longest step. Set `taxonomy.skip = true` in config to + skip assignment and output sequence/count data only. + + Requires: `Analysis/ckpt_chimera.RData` + Saves: `Analysis/checkpoint.RData` (full R environment snapshot) + """ + function assign_taxonomy(config_path::String; progress=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path) + verbose = ctx.verbose + + isfile(ctx.ckpts["chimera"]) || + error("Chimera checkpoint not found. Run chimera_removal() first.") + chimera_ckpt = ctx.ckpts["chimera"] + R"load($chimera_ckpt)" + + seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") + fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") + taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") + combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.xlsx") + boot_mode = get(ctx.cfg["output"], "bootstraps", "combined") + comb_mode = get(ctx.cfg["output"], "combined_mode", "regular") + tables_dir = ctx.dirs["Tables"] + + R"combined_input <- index" + + if !get(ctx.cfg["taxonomy"], "skip", false) + emit("Assigning taxonomy") + tax_uri = ctx.cfg["taxonomy"]["uri"] + tax_dir = ctx.dirs["Taxonomy"] + multithread = get(ctx.cfg["taxonomy"], "multithread", true) + min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) + tax_levels = ctx.cfg["taxonomy"]["levels"] + R""" + db_path <- fetch_taxonomy_db($tax_uri, $tax_dir) + taxa_result <- run_assign_taxonomy( + seq_table_nochim, db_path, + list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), + $verbose) + taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, + $tables_dir, $taxa_prefix, $boot_mode) + """ + comb_mode == "regular" && R"combined_input <- taxa_df" + else + emit("Skipping taxonomy (taxonomy.skip = true)") + end + + checkpoint = joinpath(ctx.dirs["Analysis"], "checkpoint.RData") + R"save.image($checkpoint)" + emit("Checkpoint: $checkpoint") + + R"write_combined_table(combined_input, seq_table_nochim, $tables_dir, $combined_file)" + + emit("Pipeline complete. Outputs:") + emit(" $(joinpath(tables_dir, seq_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, fasta_prefix * ".fasta"))") + emit(" $(joinpath(tables_dir, fasta_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, taxa_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, combined_file))") + emit(" $(joinpath(tables_dir, "pipeline_stats.csv"))") + emit(" $checkpoint") + nothing + end + + # Run pipeline + """ + dada2(config_path; progress) + + Run the complete DADA2 pipeline from raw reads to a taxonomy-annotated ASV + table, calling all six stages in sequence: + + prefilter_qc → filter_trim → learn_errors → + denoise → chimera_removal → assign_taxonomy + + For interactive use — reviewing intermediate outputs or adjusting parameters + between steps — call the individual stage functions directly instead. + + ## Outputs + Written to `workspace.root/Tables/`: + - `seqtab_nochim.csv` — chimera-free ASV count table + - `asvs.fasta` / `asvs.csv` — ASV sequences with short identifiers + - `taxonomy.csv` — taxonomy assignments (with optional bootstraps) + - `tax_counts.xlsx` — combined taxonomy + per-sample counts + - `pipeline_stats.csv` — read counts at each pipeline stage + + Written to `workspace.root/Analysis/`: + - `ckpt_filter.RData` — filter_stats + - `ckpt_errors.RData` — fwd_errors, rev_errors + - `ckpt_denoise.RData` — dada objects, merged, unfiltered seq_table + - `ckpt_chimera.RData` — seq_table_nochim, index + - `checkpoint.RData` — full R environment snapshot + """ + function dada2(config_path::String; progress=nothing) + prefilter_qc(config_path; progress) + filter_trim(config_path; progress) + learn_errors(config_path; progress) + denoise(config_path; progress) + chimera_removal(config_path; progress) + assign_taxonomy(config_path; progress) + end + +end \ No newline at end of file diff --git a/src/dada2_functions.r b/src/dada2_functions.r index d282e41..38b482c 100644 --- a/src/dada2_functions.r +++ b/src/dada2_functions.r @@ -1,6 +1,6 @@ # DADA2 amplicon sequencing functions # -# Functions for dada2.r main workflow +# R wrappers and functions for dada2.jl main workflow # # Notice: # @@ -17,143 +17,10 @@ library(dada2) library(openxlsx) library(tidyverse) -library(yaml) +# library(yaml) -# Null-coalescing operator: returns x if not NULL, otherwise y. -# Used to apply config defaults. -`%||%` <- function(x, y) if (!is.null(x)) x else y - -## Configuration -load_config <- function(yaml_path) { - if (!file.exists(yaml_path)) { - stop("Config file not found: ", yaml_path) - } - yaml.load_file(yaml_path) -} - -validate_config <- function(cfg) { - required <- c("workspace", "file_patterns", "filter_trim", - "dada", "merge", "asv", "taxonomy", "output") - missing_sections <- setdiff(required, names(cfg)) - if (length(missing_sections) > 0) { - stop("Missing required config sections: ", - paste(missing_sections, collapse = ", ")) - } - - if (is.null(cfg$workspace$root)) { - stop("workspace.root is required") - } - if (is.null(cfg$workspace$input_dir)) { - stop("workspace.input_dir is required") - } - if (!dir.exists(cfg$workspace$input_dir)) { - stop("workspace.input_dir does not exist: ", cfg$workspace$input_dir) - } - - mode <- cfg$file_patterns$mode %||% "paired" - if (!mode %in% c("paired", "forward", "reverse")) { - stop("file_patterns.mode must be one of: paired, forward, reverse") - } - - boot_mode <- cfg$output$bootstraps %||% "combined" - if (!boot_mode %in% c("none", "combined", "separate")) { - stop("output.bootstraps must be one of: none, combined, separate") - } - - # taxonomy.uri is only required when taxonomy assignment is not skipped - if (!(cfg$taxonomy$skip %||% FALSE) && is.null(cfg$taxonomy$uri)) { - stop("taxonomy.uri is required when taxonomy.skip is not true") - } - - combined_mode <- cfg$output$combined_mode %||% "regular" - if (!combined_mode %in% c("regular", "alternative")) { - stop("output.combined_mode must be one of: regular, alternative") - } - - invisible(cfg) -} - -# Creates the standard directory tree under root and returns paths as a named -# list so downstream functions can reference them by: -# directory <- paths$Directory (e.g. paths$Tables). -setup_workspace <- function(root) { - dirs <- list( - Tables = file.path(root, "Tables"), - Analysis = file.path(root, "Analysis"), - Taxonomy = file.path(root, "Taxonomy"), - Figures = file.path(root, "Figures"), - Filtered = file.path(root, "Filtered") - ) - for (d in dirs) dir.create(d, showWarnings = FALSE, recursive = TRUE) - dirs -} - -# Finds FASTQ files matching forward/reverse patterns. Files are sorted so that -# each forward file always corresponds its reverse file by name. -find_fastq_files <- function(input_dir, fwd_pattern, rev_pattern, mode) { - fwd <- NULL - rev <- NULL - if (mode %in% c("paired", "forward")) { - fwd <- sort(list.files(input_dir, pattern = fwd_pattern, full.names = TRUE)) - } - if (mode %in% c("paired", "reverse")) { - rev <- sort(list.files(input_dir, pattern = rev_pattern, full.names = TRUE)) - } - list(forward = fwd, reverse = rev) -} - -# Checks that at least one sample was found and that forward/reverse counts -# match in paired mode. With one sample, dada() would return a bare object -# instead of a list so duplication is performed in main() by the file paths -# before the pipeline runs. -validate_sample_files <- function(fwd, rev, mode) { - if (mode %in% c("paired", "forward")) { - if (length(fwd) < 1) { - stop("No forward FASTQ files found. ", - "Check workspace.input_dir and file_patterns.forward.") - } - missing_fwd <- fwd[!file.exists(fwd)] - if (length(missing_fwd) > 0) { - stop("Forward files not found:\n ", paste(missing_fwd, collapse = "\n ")) - } - } - if (mode %in% c("paired", "reverse")) { - if (length(rev) < 1) { - stop("No reverse FASTQ files found. ", - "Check workspace.input_dir and file_patterns.reverse.") - } - missing_rev <- rev[!file.exists(rev)] - if (length(missing_rev) > 0) { - stop("Reverse files not found:\n ", paste(missing_rev, collapse = "\n ")) - } - } - if (mode == "paired" && length(fwd) != length(rev)) { - stop("Forward file count (", length(fwd), ") does not match ", - "reverse file count (", length(rev), ").") - } - invisible(NULL) -} - -# Extracts sample names. (e.g. "Sample1_R1_trimmed.fastq.gz" split by "_" -# at index 1 gives "Sample1"). -extract_sample_names <- function(fwd_files, split_char, split_index) { - sapply(strsplit(basename(fwd_files), split_char), `[`, split_index) -} - -# Constructs output paths for filtered reads. -make_filtered_paths <- function(sample_names, filtered_dir, mode = "paired") { - fwd <- NULL - rev <- NULL - if (mode != "reverse") { - fwd <- file.path(filtered_dir, paste0(sample_names, "_R1_filt.fastq.gz")) - names(fwd) <- sample_names - } - if (mode != "forward") { - rev <- file.path(filtered_dir, paste0(sample_names, "_R2_filt.fastq.gz")) - names(rev) <- sample_names - } - list(forward = fwd, reverse = rev) -} +# Config loading/validation, workspace setup, and file discovery are now +# handled by dada2.jl, passing resolved paths and values directly to R. ## Plotting @@ -197,44 +64,41 @@ plot_length_distribution <- function(seq_table, output_pdf) { ## Filter and trim -# Wrapper around filterAndTrim that handles both paired and single-end modes. -# Key parameters: -# truncQ - truncate reads at the first base with Phred quality ≤ this value -# maxEE - maximum expected errors per read (calculated from Phred scores); -# lower = stricter, recommended 2–5 for typical Illumina data -# matchIDs - (paired only) discard read pairs where one read was filtered out, -# ensuring F and R files remain perfectly synchronised -run_filter_trim <- function(fwd_in, rev_in, fwd_out, rev_out, params, verbose) { - trunc_len <- unlist(params$trunc_len) - max_ee <- unlist(params$max_ee) +# filterAndTrim() is now called directly from dada2.jl, which +# handles modes and parameters without this intermediate wrapper. +# I might delete it when I feel more destructive... - if (!is.null(rev_in)) { - filterAndTrim( - fwd_in, fwd_out, - rev_in, rev_out, - truncQ = params$trunc_q, - truncLen = trunc_len, - maxEE = max_ee, - minLen = params$min_len, - maxN = params$max_n, - matchIDs = params$match_ids, - rm.phix = params$rm_phix, - verbose = verbose - ) - } else { - # Single-end: use only the first element of trunc_len / maxEE - filterAndTrim( - fwd_in, fwd_out, - truncQ = params$trunc_q, - truncLen = trunc_len[1], - maxEE = max_ee[1], - minLen = params$min_len, - maxN = params$max_n, - rm.phix = params$rm_phix, - verbose = verbose - ) - } -} +# run_filter_trim <- function(fwd_in, rev_in, fwd_out, rev_out, params, verbose) { +# trunc_len <- unlist(params$trunc_len) +# max_ee <- unlist(params$max_ee) +# +# if (!is.null(rev_in)) { +# filterAndTrim( +# fwd_in, fwd_out, +# rev_in, rev_out, +# truncQ = params$trunc_q, +# truncLen = trunc_len, +# maxEE = max_ee, +# minLen = params$min_len, +# maxN = params$max_n, +# matchIDs = params$match_ids, +# rm.phix = params$rm_phix, +# verbose = verbose +# ) +# } else { +# # Single-end: use only the first element of trunc_len / maxEE +# filterAndTrim( +# fwd_in, fwd_out, +# truncQ = params$trunc_q, +# truncLen = trunc_len[1], +# maxEE = max_ee[1], +# minLen = params$min_len, +# maxN = params$max_n, +# rm.phix = params$rm_phix, +# verbose = verbose +# ) +# } +# } ## Sequence table @@ -331,7 +195,7 @@ write_fasta <- function(seq_table, tables_dir, prefix) { writeLines(fasta_lines, file.path(tables_dir, paste0(prefix, ".fasta"))) index <- data.frame(SeqName = seq_names, Sequence = sequences, - stringsAsFactors = FALSE) + stringsAsFactors = FALSE) write.csv(index, file.path(tables_dir, paste0(prefix, ".csv")), quote = FALSE, row.names = FALSE) diff --git a/src/main.jl b/src/main.jl index 9da707a..48af38a 100644 --- a/src/main.jl +++ b/src/main.jl @@ -1,7 +1,7 @@ #!/usr/bin/env julia include("run_cutadapt.jl") -include("run_dada2.jl") +include("dada2.jl") include("merge_and_filter_taxa.jl") using CSV diff --git a/src/run_dada2.jl b/src/run_dada2.jl index 32ebfd5..56352b3 100644 --- a/src/run_dada2.jl +++ b/src/run_dada2.jl @@ -1,4 +1,6 @@ -module DADA2 +module OldDADA2 + +# This is now obsolete, I guess...how sad, it was quite cute. export dada2 From 8f562e8c16f8f63404eeba5232343b47753331bd Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 19 Feb 2026 10:51:26 +0100 Subject: [PATCH 015/175] Update .gitignore --- .gitignore | 11 +++++++---- .vscode/settings.json | 2 -- 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 4b1a8b3..1cde258 100644 --- a/.gitignore +++ b/.gitignore @@ -243,17 +243,20 @@ po/*~ # RStudio Connect folder rsconnect/ -### R.Bookdown Stack ### +### R.Bookdown Stack # R package: bookdown caching files /*_files/ -### Other ### +# vscode +.vscode + +### Other # Things with ambiguous intellectual property internal data/fastq/ cutadapt/output_* -### Lock files ### +### Lock files .~lock.* .csv# @@ -261,7 +264,7 @@ cutadapt/output_* bin/ config/tools.yml -### Bits I don't want to share yet ### +### Bits I don't want to share yet hide.* output/ pipelinesteps.txt \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 7a73a41..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file From 2c424933743630310f38ce1de0ffb0c7f61d9903 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 19 Feb 2026 15:32:38 +0100 Subject: [PATCH 016/175] Bugfixes and directory restructure. --- .gitignore | 3 +- config/dada2.yml | 11 +++++--- src/dada2.jl | 64 +++++++++++++++++++++++++++---------------- src/dada2_functions.r | 8 +++--- src/main.jl | 3 +- src/run_cutadapt.jl | 2 +- 6 files changed, 57 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 1cde258..e8b929e 100644 --- a/.gitignore +++ b/.gitignore @@ -260,8 +260,9 @@ cutadapt/output_* .~lock.* .csv# -### Downloaded binaries and tool paths +### Tool paths, downloaded binaries and databases bin/ +databases/ config/tools.yml ### Bits I don't want to share yet diff --git a/config/dada2.yml b/config/dada2.yml index 64c53ee..572a725 100644 --- a/config/dada2.yml +++ b/config/dada2.yml @@ -1,6 +1,6 @@ workspace: root: "./output/dada2/" - input_dir: "/path/to/fastqs" + input_dir: "./output/cutadapt/output_2026-01-31T19:51" file_patterns: forward: "_R1_trimmed.fastq.gz" @@ -30,14 +30,17 @@ merge: trim_overhang: true asv: - band_size_min: 250 # null to skip length filtering - band_size_max: 256 + band_size_min: 200 # null to skip length filtering + band_size_max: 430 denovo_method: "consensus" +databases_dir: "./databases" # shared cache for taxonomy/vsearch databases; can be absolute or relative to working directory + taxonomy: skip: false # true to skip taxonomy and use alternative output mode uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" - multithread: true + # uri: "/path/to/local/database.fasta.gz" # use a local file instead of downloading + multithread: 48 min_boot: 0 levels: - "Domain" diff --git a/src/dada2.jl b/src/dada2.jl index a024d13..a4cd024 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -30,6 +30,7 @@ module DADA2 export dada2, prefilter_qc, filter_trim, learn_errors, denoise, chimera_removal, assign_taxonomy + import Downloads using Logging, RCall, YAML # Helpers @@ -110,11 +111,10 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, # Workspace function setup_workspace(root) dirs = Dict( - "Tables" => joinpath(root, "Tables"), - "Analysis" => joinpath(root, "Analysis"), - "Taxonomy" => joinpath(root, "Taxonomy"), - "Figures" => joinpath(root, "Figures"), - "Filtered" => joinpath(root, "Filtered"), + "Tables" => joinpath(root, "Tables"), + "Checkpoints" => joinpath(root, "Checkpoints"), + "Figures" => joinpath(root, "Figures"), + "Filtered" => joinpath(root, "Filtered"), ) for d in values(dirs) mkpath(d) @@ -178,10 +178,10 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, out_rev_arg = mode == "paired" ? rev_out : nothing ckpts = Dict( - "filter" => joinpath(dirs["Analysis"], "ckpt_filter.RData"), - "errors" => joinpath(dirs["Analysis"], "ckpt_errors.RData"), - "denoise" => joinpath(dirs["Analysis"], "ckpt_denoise.RData"), - "chimera" => joinpath(dirs["Analysis"], "ckpt_chimera.RData"), + "filter" => joinpath(dirs["Checkpoints"], "ckpt_filter.RData"), + "errors" => joinpath(dirs["Checkpoints"], "ckpt_errors.RData"), + "denoise" => joinpath(dirs["Checkpoints"], "ckpt_denoise.RData"), + "chimera" => joinpath(dirs["Checkpoints"], "ckpt_chimera.RData"), ) (; cfg, verbose, mode, dirs, sample_names, single_sample, @@ -409,8 +409,8 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Review `Tables/pipeline_stats.csv` for unexpected read loss at any stage before committing to the (potentially long) `assign_taxonomy()` step. - Requires: `Analysis/ckpt_filter.RData`, `Analysis/ckpt_denoise.RData` - Saves: `Analysis/ckpt_chimera.RData` + Requires: `Checkpoints/ckpt_filter.RData`, `Checkpoints/ckpt_denoise.RData` + Saves: `Checkpoints/ckpt_chimera.RData` """ function chimera_removal(config_path::String; progress=nothing) emit = _emitter(progress) @@ -493,8 +493,8 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, This is typically the longest step. Set `taxonomy.skip = true` in config to skip assignment and output sequence/count data only. - Requires: `Analysis/ckpt_chimera.RData` - Saves: `Analysis/checkpoint.RData` (full R environment snapshot) + Requires: `Checkpoints/ckpt_chimera.RData` + Saves: `Checkpoints/checkpoint.RData` (full R environment snapshot) """ function assign_taxonomy(config_path::String; progress=nothing) emit = _emitter(progress) @@ -519,14 +519,32 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, if !get(ctx.cfg["taxonomy"], "skip", false) emit("Assigning taxonomy") tax_uri = ctx.cfg["taxonomy"]["uri"] - tax_dir = ctx.dirs["Taxonomy"] multithread = get(ctx.cfg["taxonomy"], "multithread", true) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) tax_levels = ctx.cfg["taxonomy"]["levels"] + + db_path = if isfile(tax_uri) + # taxonomy.uri is a local path — use it directly + emit("Using local taxonomy database: $tax_uri") + tax_uri + else + # taxonomy.uri is a URL — download to shared project databases dir + db_dir = abspath(get(ctx.cfg, "databases_dir", "./databases")) + mkpath(db_dir) + cached = joinpath(db_dir, basename(tax_uri)) + if !isfile(cached) + emit("Downloading taxonomy database: $tax_uri") + Downloads.download(tax_uri, cached) + emit("Written: $cached") + else + emit("Using cached taxonomy database: $cached") + end + cached + end + R""" - db_path <- fetch_taxonomy_db($tax_uri, $tax_dir) taxa_result <- run_assign_taxonomy( - seq_table_nochim, db_path, + seq_table_nochim, $db_path, list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), $verbose) taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, @@ -537,7 +555,7 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, emit("Skipping taxonomy (taxonomy.skip = true)") end - checkpoint = joinpath(ctx.dirs["Analysis"], "checkpoint.RData") + checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") R"save.image($checkpoint)" emit("Checkpoint: $checkpoint") @@ -575,7 +593,7 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `tax_counts.xlsx` — combined taxonomy + per-sample counts - `pipeline_stats.csv` — read counts at each pipeline stage - Written to `workspace.root/Analysis/`: + Written to `workspace.root/Checkpoints/`: - `ckpt_filter.RData` — filter_stats - `ckpt_errors.RData` — fwd_errors, rev_errors - `ckpt_denoise.RData` — dada objects, merged, unfiltered seq_table @@ -583,11 +601,11 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `checkpoint.RData` — full R environment snapshot """ function dada2(config_path::String; progress=nothing) - prefilter_qc(config_path; progress) - filter_trim(config_path; progress) - learn_errors(config_path; progress) - denoise(config_path; progress) - chimera_removal(config_path; progress) + #prefilter_qc(config_path; progress) + #filter_trim(config_path; progress); R"gc()" + #learn_errors(config_path; progress); R"gc()" + #denoise(config_path; progress); R"gc()" + #chimera_removal(config_path; progress); R"gc()" assign_taxonomy(config_path; progress) end diff --git a/src/dada2_functions.r b/src/dada2_functions.r index 38b482c..f27a31b 100644 --- a/src/dada2_functions.r +++ b/src/dada2_functions.r @@ -29,10 +29,10 @@ library(tidyverse) plot_quality_profiles <- function(fwd_files, rev_files, output_pdf) { pdf(output_pdf, width = 8, height = 6) if (!is.null(fwd_files) && length(fwd_files) > 0) { - plotQualityProfile(fwd_files[seq_len(min(3L, length(fwd_files)))]) + print(plotQualityProfile(fwd_files[seq_len(min(3L, length(fwd_files)))])) } if (!is.null(rev_files) && length(rev_files) > 0) { - plotQualityProfile(rev_files[seq_len(min(3L, length(rev_files)))]) + print(plotQualityProfile(rev_files[seq_len(min(3L, length(rev_files)))])) } dev.off() invisible(NULL) @@ -43,8 +43,8 @@ plot_quality_profiles <- function(fwd_files, rev_files, output_pdf) { # the error model did not converge. If so, try increasing nbases or max_consist. plot_error_rates <- function(fwd_errors, rev_errors, output_pdf) { pdf(output_pdf, width = 8, height = 6) - if (!is.null(fwd_errors)) plotErrors(fwd_errors, nominalQ = TRUE) - if (!is.null(rev_errors)) plotErrors(rev_errors, nominalQ = TRUE) + if (!is.null(fwd_errors)) print(plotErrors(fwd_errors, nominalQ = TRUE)) + if (!is.null(rev_errors)) print(plotErrors(rev_errors, nominalQ = TRUE)) dev.off() invisible(NULL) } diff --git a/src/main.jl b/src/main.jl index 48af38a..b50178b 100644 --- a/src/main.jl +++ b/src/main.jl @@ -43,6 +43,7 @@ output_dir = "./output" # Cutadapt paths fastq_input_dir = joinpath(data_dir, "fastq") cutadapt_dir = joinpath(output_dir, "cutadapt") +vsearch_dir = joinpath(output_dir, "vsearch") primers_config = joinpath(config_dir, "primers.yml") ## Instantiate parameters @@ -72,7 +73,7 @@ filtered_outfile_vespa = joinpath(output_dir, "protist_filtered_vespa.csv") #cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, # optional_args = optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) -#dada2(dada2_config_dir) +dada2(dada2_config_dir) #CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) #CSV.write(merged_outfile_vespa, merge_taxonomy_counts(vespav, vespad)) diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index 8b82ae8..ce96c9c 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -112,7 +112,7 @@ export cutadapt """ function cutadapt(primer_pairs, primers_path, fastq_in_dir, fastq_out_dir, optional_args) - Requires `cutadapt` installed and in PATH. Runs `cutadapt` command with as many primer pairs as necessary (useful for multiplex) and any optional parameters. Primer aguments are determined from YAML file in format: + Requires `cutadapt` installed. Runs `cutadapt` command with as many primer pairs as necessary (useful for multiplex) and any optional parameters. Primer aguments are determined from YAML file in format: ``` YAML Forward: PrimerF: "CCAGCASCYGCGGTAATTCC" From ac930bdcda1ae4b1c8869c791e45359de1fad0d2 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 19 Feb 2026 18:03:08 +0100 Subject: [PATCH 017/175] Added VSEARCH local alignment taxonomic assignment. Pipeline tested and working cutadapt -> vsearch-dada merge on default settings. --- config/dada2.yml | 2 +- src/assign_taxonomy_worker.r | 38 ++++++++++++++++++++++++++++++++ src/dada2.jl | 28 ++++++++++++++++++------ src/main.jl | 39 +++++++++++++++++---------------- src/merge_and_filter_taxa.jl | 42 ++++++++++++++++++++++++------------ src/run_cutadapt.jl | 12 +++++------ src/run_vsearch.jl | 29 +++++++++++++++++++++++++ 7 files changed, 144 insertions(+), 46 deletions(-) create mode 100644 src/assign_taxonomy_worker.r mode change 100644 => 100755 src/main.jl create mode 100644 src/run_vsearch.jl diff --git a/config/dada2.yml b/config/dada2.yml index 572a725..67ca14b 100644 --- a/config/dada2.yml +++ b/config/dada2.yml @@ -40,7 +40,7 @@ taxonomy: skip: false # true to skip taxonomy and use alternative output mode uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" # uri: "/path/to/local/database.fasta.gz" # use a local file instead of downloading - multithread: 48 + multithread: 8 # higher values increase memory use significantly (mclapply forks one process per thread) min_boot: 0 levels: - "Domain" diff --git a/src/assign_taxonomy_worker.r b/src/assign_taxonomy_worker.r new file mode 100644 index 0000000..56348eb --- /dev/null +++ b/src/assign_taxonomy_worker.r @@ -0,0 +1,38 @@ +# Worker script for assignTaxonomy(). +# +# Invoked as a subprocess by assign_taxonomy() in dada2.jl. On Unix this +# process is launched with `nice -n 10` so that the CPU-intensive naive +# Bayesian classification does not monopolise all cores and lock up the +# system. Memory for the reference database is also freed when this process +# exits rather than accumulating in the Julia/RCall session. +# +# Usage (internal - do not call directly): +# Rscript assign_taxonomy_worker.r \ +# \ +# + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 8L) { + stop("assign_taxonomy_worker.r: expected 8 arguments") +} + +functions_r <- args[[1L]] +chimera_ckpt <- args[[2L]] +db_path <- args[[3L]] +taxa_ckpt <- args[[4L]] +multithread <- as.integer(args[[5L]]) +min_boot <- as.numeric(args[[6L]]) +levels <- strsplit(args[[7L]], ",", fixed = TRUE)[[1L]] +verbose <- as.logical(args[[8L]]) + +source(functions_r) +load(chimera_ckpt) # provides seq_table_nochim + +taxa_result <- run_assign_taxonomy(seq_table_nochim, db_path, + list(multithread = multithread, + min_boot = min_boot, + levels = levels), + verbose) + +save(taxa_result, file = taxa_ckpt) +message("Taxonomy assignment complete. Saved to ", taxa_ckpt) diff --git a/src/dada2.jl b/src/dada2.jl index a4cd024..1ca6633 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -503,6 +503,19 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, isfile(ctx.ckpts["chimera"]) || error("Chimera checkpoint not found. Run chimera_removal() first.") + + # Drop all data objects accumulated from prior stages before the + # memory-intensive taxonomy subprocess runs. Named globals in R's + # environment are reachable and gc() won't collect them; rm() them + # explicitly so they don't inflate the subprocess's memory footprint. + # lsf.str() returns function names; setdiff keeps those intact. + R""" + .data_objs <- setdiff(ls(), lsf.str()) + if (length(.data_objs) > 0L) rm(list = .data_objs) + rm(.data_objs) + gc() + """ + chimera_ckpt = ctx.ckpts["chimera"] R"load($chimera_ckpt)" @@ -519,7 +532,7 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, if !get(ctx.cfg["taxonomy"], "skip", false) emit("Assigning taxonomy") tax_uri = ctx.cfg["taxonomy"]["uri"] - multithread = get(ctx.cfg["taxonomy"], "multithread", true) + multithread = get(ctx.cfg["taxonomy"], "multithread", 4) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) tax_levels = ctx.cfg["taxonomy"]["levels"] @@ -550,13 +563,14 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, $tables_dir, $taxa_prefix, $boot_mode) """ + R"gc()" comb_mode == "regular" && R"combined_input <- taxa_df" else emit("Skipping taxonomy (taxonomy.skip = true)") end checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") - R"save.image($checkpoint)" + R"save(seq_table_nochim, index, combined_input, file=$checkpoint)" emit("Checkpoint: $checkpoint") R"write_combined_table(combined_input, seq_table_nochim, $tables_dir, $combined_file)" @@ -601,11 +615,11 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `checkpoint.RData` — full R environment snapshot """ function dada2(config_path::String; progress=nothing) - #prefilter_qc(config_path; progress) - #filter_trim(config_path; progress); R"gc()" - #learn_errors(config_path; progress); R"gc()" - #denoise(config_path; progress); R"gc()" - #chimera_removal(config_path; progress); R"gc()" + prefilter_qc(config_path; progress) + filter_trim(config_path; progress); R"gc()" + learn_errors(config_path; progress); R"gc()" + denoise(config_path; progress); R"gc()" + chimera_removal(config_path; progress); R"gc()" assign_taxonomy(config_path; progress) end diff --git a/src/main.jl b/src/main.jl old mode 100644 new mode 100755 index b50178b..a1a326b --- a/src/main.jl +++ b/src/main.jl @@ -3,10 +3,11 @@ include("run_cutadapt.jl") include("dada2.jl") include("merge_and_filter_taxa.jl") +include("run_vsearch.jl") using CSV using YAML -using .Cutadapt, .TaxonomyTableTools, .DADA2 +using .Cutadapt, .TaxonomyTableTools, .DADA2, .VSEARCH ## Tools loading (to move to new module at some point) """ @@ -43,40 +44,42 @@ output_dir = "./output" # Cutadapt paths fastq_input_dir = joinpath(data_dir, "fastq") cutadapt_dir = joinpath(output_dir, "cutadapt") -vsearch_dir = joinpath(output_dir, "vsearch") primers_config = joinpath(config_dir, "primers.yml") +# DADA2/VSEARCH paths +fasta_outfile = joinpath(output_dir, "dada2/Tables/asvs.fasta") +reference_database = "./databases/pr2_version_5.0.0_SSU_dada2.fasta.gz" +vsearch_dir = joinpath(output_dir, "vsearch") + ## Instantiate parameters # Cutadapt parameters primer_pairs = ["TarEuk", "Meta2"] -optional_args = "-m 200 --discard-untrimmed" +cutadapt_optional_args = "-m 200 --discard-untrimmed" # DADA2 parameters dada2_config_dir = joinpath(config_dir, "dada2.yml") +# VSEARCH parameters +vsearch_optional_args = "--id 0.75 --query_cov 0.8" + # Merge and filter (DADA2-VSEARCH) parameters -multiv = joinpath(output_dir, "vsearch/taxonomy_multi_pool.tsv") -vespav = joinpath(output_dir, "vsearch/taxonomy_vespa_pool_fwdonly.tsv") -multid = joinpath(output_dir, "dada2/tax_counts_fasta_multi_pool.csv") -vespad = joinpath(output_dir, "dada2/tax_counts_fasta_vespa_pool_fwdonly.csv") +multiv = joinpath(output_dir, "vsearch/taxonomy.tsv") +multid = joinpath(output_dir, "dada2/Tables/taxonomy.csv") protist_filter = joinpath(config_dir, "protist_filter.yml") -merged_outfile_multi = joinpath(output_dir, "merged_multi.csv") -merged_outfile_vespa = joinpath(output_dir, "merged_vespa.csv") - -filtered_outfile_multi = joinpath(output_dir, "protist_filtered_multi.csv") -filtered_outfile_vespa = joinpath(output_dir, "protist_filtered_vespa.csv") +merged_outfile_multi = joinpath(output_dir, "merged/merged_multi.csv") +filtered_outfile_multi = joinpath(output_dir, "merged/protist_filtered_multi.csv") ## Main -#cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, -# optional_args = optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) +cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, optional_args = cutadapt_optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) dada2(dada2_config_dir) -#CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) -#CSV.write(merged_outfile_vespa, merge_taxonomy_counts(vespav, vespad)) +vsearch(fasta_outfile, reference_database, vsearch_dir, optional_args = vsearch_optional_args, vsearch_bin = tool_bin(tools, "vsearch")) + +mkpath(dirname(merged_outfile_multi)) -#CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(vespav, vespad), protist_filter)) -#CSV.write(filtered_outfile_vespa, filter_table(merge_taxonomy_counts(vespav, vespad), protist_filter)) \ No newline at end of file +CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) +CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)) \ No newline at end of file diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index f10d5cb..e42374e 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -35,24 +35,28 @@ export merge_taxonomy_counts, filter_table "Domain","Supergroup","Division","Subdivision","Class","Order", "Family","Genus","Species"] out_rows = Vector{Vector{String}}(undef, nrow(df)) - + for (i, r) in enumerate(eachrow(df)) - parts = split(r.sseqtax, '|') - # Always create full row with proper positioning - data = Vector{String}(undef, length(header)) + data = fill("", length(header)) data[1] = string(r.SeqName) data[2] = string(r.Pident) - - # Fill taxonomy fields (positions 3 to end) - for j in 1:min(length(parts), length(header)-2) - data[j+2] = parts[j] - end - - # Fill remaining fields with empty strings - for j in (length(parts)+3):length(header) - data[j] = "" + + tax_str = r.sseqtax + if occursin('|', tax_str) + # Full PR2 format: Accession|rRNA|Organellum|specimen|Domain|...|Species + parts = split(tax_str, '|') + for j in 1:min(length(parts), length(header) - 2) + data[j + 2] = parts[j] + end + else + # Taxonomy-only format: Domain;Supergroup;...;Species[;] + # No accession/rRNA/Organellum/specimen — those stay as empty strings + parts = filter(!isempty, split(tax_str, ';')) + for j in 1:min(length(parts), length(header) - 6) + data[j + 6] = parts[j] + end end - + out_rows[i] = data end return header, out_rows @@ -143,6 +147,16 @@ export merge_taxonomy_counts, filter_table rename!(df_counts_prepared, seq_id_col => "SeqName") end + # Rename any columns in the DADA2 file that clash with vsearch taxonomy + # columns (e.g. Domain, Supergroup, … when counts_csv_path is taxonomy.csv) + overlap = filter(!=(Symbol("SeqName")), + intersect(Symbol.(names(df_taxonomy)), + Symbol.(names(df_counts_prepared)))) + if !isempty(overlap) + rename!(df_counts_prepared, + [String(c) => String(c) * "_dada2" for c in overlap]) + end + # Left join instead of outerjoin to keep all taxonomy rows merged_df = leftjoin(df_taxonomy, df_counts_prepared, on="SeqName") sort!(merged_df, "SeqName", by=seqnum) diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index ce96c9c..0c1cfe7 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -63,8 +63,8 @@ export cutadapt # Messy filesystem stuff time = chop("$(now(localzone()))", tail = 13) - fastq_out_dir = cutadapt_dir * "$time/" - log_dir = fastq_out_dir * "/logs/" + fastq_out_dir = joinpath(cutadapt_dir, time) + log_dir = joinpath(fastq_out_dir, "logs") stats_path = joinpath(log_dir, "cutadapt_primer_trimming_stats.txt") summary_path = joinpath(log_dir, "cutadapt_trimmed_percentage.txt") stats_basename = basename(stats_path) @@ -81,11 +81,11 @@ export cutadapt nsamples = length(samples) for (i, sample) in enumerate(samples) - inputR1 = fastq_in_dir * sample * "_*_L001_R1_001.fastq.gz" - inputR2 = fastq_in_dir * sample * "_*_L001_R2_001.fastq.gz" + inputR1 = joinpath(fastq_in_dir, sample * "_*_L001_R1_001.fastq.gz") + inputR2 = joinpath(fastq_in_dir, sample * "_*_L001_R2_001.fastq.gz") - outputR1 = fastq_out_dir * sample * "_R1_trimmed.fastq.gz" - outputR2 = fastq_out_dir * sample * "_R2_trimmed.fastq.gz" + outputR1 = joinpath(fastq_out_dir, sample * "_R1_trimmed.fastq.gz") + outputR2 = joinpath(fastq_out_dir, sample * "_R2_trimmed.fastq.gz") @info("On sample $i/$nsamples ($sample).") cutadapt_cmd = "$cutadapt_bin $primer_args $optional_args -o $outputR1 -p $outputR2 $inputR1 $inputR2" diff --git a/src/run_vsearch.jl b/src/run_vsearch.jl new file mode 100644 index 0000000..86eb53d --- /dev/null +++ b/src/run_vsearch.jl @@ -0,0 +1,29 @@ +module VSEARCH + +export vsearch + + using Logging + + """ + vsearch(fasta_in_dir, reference_database; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") + + Requires `vsearch` installed. Runs `vsearch` command with any optional parameters to perform taxonomy assignment by local alignment against specified database. + + ## Arguments + - `fasta_in_dir`: Specify path of fasta files output by DADA2 pipeline. + - `reference_database` (default: "./databases"): Specify path of reference database. + + ## Keyword Arguments + - `optional_args` (optional, default: "--id 0.75 --query_cov 0.8"): Specify additional arguments passed to `vsearch` command. + - `vsearch_bin` (optional, default: "vsearch"): Path to the vsearch binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. + + """ + function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") + outfile = joinpath(vsearch_dir, "taxonomy.tsv") + + cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" + run(`bash -lc $cmd`) + + end + +end \ No newline at end of file From edb681a4080aee8834929aa40a1789ae0a7512e3 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 16:29:18 +0100 Subject: [PATCH 018/175] Added databases.jl to handle databases due to different requirements for DADA2 and VSEARCH --- config/dada2.yml | 15 +++-- src/dada2.jl | 146 +++++++++++++++++++++++++++++++------------- src/databases.jl | 89 +++++++++++++++++++++++++++ src/main.jl | 44 +++++++------ src/run_cutadapt.jl | 24 +++----- src/run_vsearch.jl | 1 + 6 files changed, 242 insertions(+), 77 deletions(-) create mode 100644 src/databases.jl diff --git a/config/dada2.yml b/config/dada2.yml index 67ca14b..4d14965 100644 --- a/config/dada2.yml +++ b/config/dada2.yml @@ -34,13 +34,20 @@ asv: band_size_max: 430 denovo_method: "consensus" -databases_dir: "./databases" # shared cache for taxonomy/vsearch databases; can be absolute or relative to working directory +databases: + dir: "./databases" # shared cache directory; absolute or relative to working directory + pr2: + dada2: + uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" + local: ~ # set to a local path to skip download + vsearch: + uri: "https://github.com/pr2database/pr2database/releases/download/v5.1.0.0/pr2_version_5.1.0_SSU_taxo_long.fasta.gz" + local: ~ # set to a local path to skip download taxonomy: + database: pr2 # key into databases: section above skip: false # true to skip taxonomy and use alternative output mode - uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" - # uri: "/path/to/local/database.fasta.gz" # use a local file instead of downloading - multithread: 8 # higher values increase memory use significantly (mclapply forks one process per thread) + multithread: 4 # higher values increase memory use significantly (mclapply forks one process per thread) min_boot: 0 levels: - "Domain" diff --git a/src/dada2.jl b/src/dada2.jl index 1ca6633..74619d1 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -40,6 +40,67 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, _emitter(::Nothing) = msg -> @info msg _emitter(ch::Channel{String}) = msg -> put!(ch, msg) + # Database resolution helpers + + # Download `uri` to `db_dir/basename(uri)` if not already cached. + # Respects the optional `local:` override in `fmt_info`. + function _download_db_if_needed(key, fmt_info, db_dir, emit) + local_p = get(fmt_info, "local", nothing) + if !isnothing(local_p) + local_p = string(local_p) + if !isempty(local_p) + isfile(local_p) && (emit("[$key] Using local file: $local_p"); return local_p) + @warn "[$key] Configured local path not found: $local_p — falling back to uri" + end + end + uri = fmt_info["uri"] + cached = joinpath(db_dir, basename(uri)) + if isfile(cached) + emit("[$key] Using cached: $cached") + else + emit("[$key] Downloading: $uri") + Downloads.download(uri, cached) + emit("[$key] Saved to: $cached") + end + return cached + end + + # Resolve the DADA2 taxonomy database from config when no external path is + # provided. Supports the `databases: / taxonomy.database:` scheme (preferred) + # and the legacy `taxonomy.uri:` key for backwards compatibility. + function _resolve_taxonomy_db(cfg, emit) + tax_cfg = cfg["taxonomy"] + if haskey(tax_cfg, "database") + db_key = string(tax_cfg["database"]) + db_cfg = get(cfg, "databases", Dict()) + haskey(db_cfg, db_key) || + error("taxonomy.database = \"$db_key\" not found in databases: section") + fmt_cfg = get(db_cfg[db_key], "dada2", nothing) + isnothing(fmt_cfg) && + error("databases.$db_key.dada2 is not configured") + db_dir = abspath(get(db_cfg, "dir", "./databases")) + mkpath(db_dir) + return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) + end + # Legacy: taxonomy.uri + tax_uri = tax_cfg["uri"] + if isfile(tax_uri) + emit("Using local taxonomy database: $tax_uri") + return tax_uri + end + db_dir = abspath(get(cfg, "databases_dir", "./databases")) + mkpath(db_dir) + cached = joinpath(db_dir, basename(tax_uri)) + if !isfile(cached) + emit("Downloading taxonomy database: $tax_uri") + Downloads.download(tax_uri, cached) + emit("Written: $cached") + else + emit("Using cached taxonomy database: $cached") + end + return cached + end + # Config function validate_config(cfg) required = ["workspace", "file_patterns", "filter_trim", "dada", @@ -62,8 +123,12 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, boot_mode in ("none", "combined", "separate") || error("output.bootstraps must be one of: none, combined, separate") - if !get(cfg["taxonomy"], "skip", false) && !haskey(cfg["taxonomy"], "uri") - error("taxonomy.uri is required when taxonomy.skip is not true") + if !get(cfg["taxonomy"], "skip", false) + has_uri = haskey(cfg["taxonomy"], "uri") + has_db = haskey(cfg["taxonomy"], "database") + has_uri || has_db || + error("taxonomy: configure `database` (referencing a databases: entry) " * + "or `uri` when skip is not true") end combined_mode = get(cfg["output"], "combined_mode", "regular") @@ -127,11 +192,25 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, # validates config, discovers files, handles the single-sample fallback, and # computes all path variables. Returns a NamedTuple so stage functions can # extract what they need without repeating boilerplate. - function _pipeline_context(config_path::String) + # + # Optional overrides (used when main.jl manages directory layout): + # input_dir — overrides cfg["workspace"]["input_dir"] + # workspace_root — overrides cfg["workspace"]["root"] + function _pipeline_context(config_path::String; input_dir=nothing, workspace_root=nothing) functions_r = joinpath(@__DIR__, "dada2_functions.r") R"source($functions_r)" - cfg = YAML.load_file(config_path) + cfg = YAML.load_file(config_path) + + # Apply caller-supplied overrides before validation so that validate_config + # sees the final paths (including the isdir check on input_dir). + if !isnothing(input_dir) + cfg["workspace"]["input_dir"] = input_dir + end + if !isnothing(workspace_root) + cfg["workspace"]["root"] = workspace_root + end + validate_config(cfg) verbose = get(cfg, "verbose", true) mode = get(cfg["file_patterns"], "mode", "paired") @@ -198,9 +277,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Review `Figures/quality_unfiltered.pdf` to choose `truncLen` and `maxEE` values in config before running `filter_trim()`. """ - function prefilter_qc(config_path::String; progress=nothing) + function prefilter_qc(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) emit("Plotting unfiltered quality profiles") fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files @@ -223,9 +302,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Saves: `Analysis/ckpt_filter.RData` """ - function filter_trim(config_path::String; progress=nothing) + function filter_trim(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) ft = ctx.cfg["filter_trim"] trunc_len = ft["trunc_len"] max_ee = ft["max_ee"] @@ -293,9 +372,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Saves: `Analysis/ckpt_errors.RData` """ - function learn_errors(config_path::String; progress=nothing) + function learn_errors(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) seed = get(ctx.cfg["dada"], "seed", 123) nbases = ctx.cfg["dada"]["nbases"] max_con = ctx.cfg["dada"]["max_consist"] @@ -339,9 +418,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Requires: `Analysis/ckpt_errors.RData` Saves: `Analysis/ckpt_denoise.RData` (unfiltered seq_table) """ - function denoise(config_path::String; progress=nothing) + function denoise(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) verbose = ctx.verbose fwd_out = ctx.fwd_out rev_out = ctx.rev_out @@ -412,9 +491,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Requires: `Checkpoints/ckpt_filter.RData`, `Checkpoints/ckpt_denoise.RData` Saves: `Checkpoints/ckpt_chimera.RData` """ - function chimera_removal(config_path::String; progress=nothing) + function chimera_removal(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) verbose = ctx.verbose mode = ctx.mode @@ -496,9 +575,9 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, Requires: `Checkpoints/ckpt_chimera.RData` Saves: `Checkpoints/checkpoint.RData` (full R environment snapshot) """ - function assign_taxonomy(config_path::String; progress=nothing) + function assign_taxonomy(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) emit = _emitter(progress) - ctx = _pipeline_context(config_path) + ctx = _pipeline_context(config_path; input_dir, workspace_root) verbose = ctx.verbose isfile(ctx.ckpts["chimera"]) || @@ -531,29 +610,12 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, if !get(ctx.cfg["taxonomy"], "skip", false) emit("Assigning taxonomy") - tax_uri = ctx.cfg["taxonomy"]["uri"] multithread = get(ctx.cfg["taxonomy"], "multithread", 4) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) tax_levels = ctx.cfg["taxonomy"]["levels"] - db_path = if isfile(tax_uri) - # taxonomy.uri is a local path — use it directly - emit("Using local taxonomy database: $tax_uri") - tax_uri - else - # taxonomy.uri is a URL — download to shared project databases dir - db_dir = abspath(get(ctx.cfg, "databases_dir", "./databases")) - mkpath(db_dir) - cached = joinpath(db_dir, basename(tax_uri)) - if !isfile(cached) - emit("Downloading taxonomy database: $tax_uri") - Downloads.download(tax_uri, cached) - emit("Written: $cached") - else - emit("Using cached taxonomy database: $cached") - end - cached - end + db_path = isnothing(taxonomy_db) ? + _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db R""" taxa_result <- run_assign_taxonomy( @@ -614,13 +676,13 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `ckpt_chimera.RData` — seq_table_nochim, index - `checkpoint.RData` — full R environment snapshot """ - function dada2(config_path::String; progress=nothing) - prefilter_qc(config_path; progress) - filter_trim(config_path; progress); R"gc()" - learn_errors(config_path; progress); R"gc()" - denoise(config_path; progress); R"gc()" - chimera_removal(config_path; progress); R"gc()" - assign_taxonomy(config_path; progress) + function dada2(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) + prefilter_qc(config_path; progress, input_dir, workspace_root) + filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" + learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" + denoise(config_path; progress, input_dir, workspace_root); R"gc()" + chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" + assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end end \ No newline at end of file diff --git a/src/databases.jl b/src/databases.jl new file mode 100644 index 0000000..a2443ac --- /dev/null +++ b/src/databases.jl @@ -0,0 +1,89 @@ +module Databases + +# Ensures all databases declared in the config are available locally, +# downloading from their configured URIs as needed. +# +# Usage: +# dbs = ensure_databases("config/dada2.yml") +# dbs["pr2_dada2"] # → resolved local path for DADA2 assignTaxonomy +# dbs["pr2_vsearch"] # → resolved local path for VSEARCH --db +# +# Keys follow the pattern "_", e.g. "pr2_dada2", +# "pr2_vsearch". Format names map to the sub-keys under each database entry +# in the databases: config section. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +import Downloads +using YAML, Logging + +export ensure_databases + +""" + ensure_databases(config_path) -> Dict{String,String} + +Reads the `databases:` section of `config_path`, ensures every declared +database file is present in `databases.dir` (downloading from `uri` if the +file is absent), and returns a Dict mapping `"_"` keys to +resolved absolute local paths. + +Set a `local:` path under any entry to use a pre-existing file directly. +If the `local:` path does not exist, the function warns and falls back to +downloading from `uri`. +""" +function ensure_databases(config_path::String) + cfg = YAML.load_file(config_path) + db_cfg = get(cfg, "databases", nothing) + + if isnothing(db_cfg) || isempty(db_cfg) + @warn "ensure_databases: no `databases:` section found in $config_path" + return Dict{String,String}() + end + + db_dir = abspath(get(db_cfg, "dir", "./databases")) + mkpath(db_dir) + + resolved = Dict{String,String}() + for (db_name, db_info) in db_cfg + db_name == "dir" && continue + !(db_info isa AbstractDict) && continue + for (fmt, fmt_info) in db_info + !(fmt_info isa AbstractDict) && continue + key = "$(db_name)_$(fmt)" + resolved[key] = _resolve_entry(key, fmt_info, db_dir) + end + end + return resolved +end + +function _resolve_entry(key, fmt_info, db_dir) + local_p = get(fmt_info, "local", nothing) + if !isnothing(local_p) + local_p = string(local_p) + if !isempty(local_p) + if isfile(local_p) + @info "[$key] Using local file: $local_p" + return local_p + end + @warn "[$key] Configured local path not found: $local_p — falling back to uri" + end + end + + uri = get(fmt_info, "uri", nothing) + isnothing(uri) && + error("databases entry '$key': no valid local path and no uri is configured") + + cached = joinpath(db_dir, basename(uri)) + if isfile(cached) + @info "[$key] Using cached: $cached" + else + @info "[$key] Downloading: $uri" + Downloads.download(uri, cached) + @info "[$key] Saved to: $cached" + end + return cached +end + +end \ No newline at end of file diff --git a/src/main.jl b/src/main.jl index a1a326b..f497a69 100755 --- a/src/main.jl +++ b/src/main.jl @@ -1,5 +1,6 @@ #!/usr/bin/env julia +include("databases.jl") include("run_cutadapt.jl") include("dada2.jl") include("merge_and_filter_taxa.jl") @@ -7,7 +8,7 @@ include("run_vsearch.jl") using CSV using YAML -using .Cutadapt, .TaxonomyTableTools, .DADA2, .VSEARCH +using .Databases, .Cutadapt, .TaxonomyTableTools, .DADA2, .VSEARCH ## Tools loading (to move to new module at some point) """ @@ -41,15 +42,27 @@ data_dir = "./data" config_dir = "./config" output_dir = "./output" +# Project name - all stage outputs live under output/{project_name}/ +# Re-running with the same project_name overwrites previous results. +project_name = "project" +project_dir = joinpath(output_dir, project_name) + # Cutadapt paths fastq_input_dir = joinpath(data_dir, "fastq") -cutadapt_dir = joinpath(output_dir, "cutadapt") +trimmed_dir = joinpath(project_dir, "cutadapt") primers_config = joinpath(config_dir, "primers.yml") -# DADA2/VSEARCH paths -fasta_outfile = joinpath(output_dir, "dada2/Tables/asvs.fasta") -reference_database = "./databases/pr2_version_5.0.0_SSU_dada2.fasta.gz" -vsearch_dir = joinpath(output_dir, "vsearch") +# DADA2 paths +dada2_dir = joinpath(project_dir, "dada2") + +# VSEARCH paths +vsearch_dir = joinpath(project_dir, "vsearch") +fasta_outfile = joinpath(dada2_dir, "Tables/asvs.fasta") + +# Merge/filter paths (derived from project_dir) +merged_dir = joinpath(project_dir, "merged") +merged_outfile_multi = joinpath(merged_dir, "merged_multi.csv") +filtered_outfile_multi = joinpath(merged_dir, "protist_filtered_multi.csv") ## Instantiate parameters # Cutadapt parameters @@ -57,29 +70,26 @@ primer_pairs = ["TarEuk", "Meta2"] cutadapt_optional_args = "-m 200 --discard-untrimmed" # DADA2 parameters -dada2_config_dir = joinpath(config_dir, "dada2.yml") +dada2_config_path = joinpath(config_dir, "dada2.yml") # VSEARCH parameters vsearch_optional_args = "--id 0.75 --query_cov 0.8" # Merge and filter (DADA2-VSEARCH) parameters -multiv = joinpath(output_dir, "vsearch/taxonomy.tsv") -multid = joinpath(output_dir, "dada2/Tables/taxonomy.csv") +multiv = joinpath(vsearch_dir, "taxonomy.tsv") +multid = joinpath(dada2_dir, "Tables/taxonomy.csv") protist_filter = joinpath(config_dir, "protist_filter.yml") -merged_outfile_multi = joinpath(output_dir, "merged/merged_multi.csv") -filtered_outfile_multi = joinpath(output_dir, "merged/protist_filtered_multi.csv") +# Download/use database +dbs = ensure_databases(dada2_config_path) ## Main +#cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) -cutadapt(primer_pairs, primers_config, fastq_input_dir, cutadapt_dir, optional_args = cutadapt_optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) - -dada2(dada2_config_dir) - -vsearch(fasta_outfile, reference_database, vsearch_dir, optional_args = vsearch_optional_args, vsearch_bin = tool_bin(tools, "vsearch")) +#dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) -mkpath(dirname(merged_outfile_multi)) +vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args, vsearch_bin = tool_bin(tools, "vsearch")) CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)) \ No newline at end of file diff --git a/src/run_cutadapt.jl b/src/run_cutadapt.jl index 0c1cfe7..b64eb07 100644 --- a/src/run_cutadapt.jl +++ b/src/run_cutadapt.jl @@ -3,7 +3,6 @@ module Cutadapt export cutadapt using YAML - using TimeZones using Logging # Prevent duplication of primers. Must be instantiated outside get_primers() loop. Global scope may be an issue. @@ -61,13 +60,10 @@ export cutadapt function run_cutadapt(primer_args, optional_args, fastq_in_dir, cutadapt_dir, cutadapt_bin) samples = [] - # Messy filesystem stuff - time = chop("$(now(localzone()))", tail = 13) - fastq_out_dir = joinpath(cutadapt_dir, time) - log_dir = joinpath(fastq_out_dir, "logs") - stats_path = joinpath(log_dir, "cutadapt_primer_trimming_stats.txt") - summary_path = joinpath(log_dir, "cutadapt_trimmed_percentage.txt") - stats_basename = basename(stats_path) + log_dir = joinpath(cutadapt_dir, "logs") + stats_path = joinpath(log_dir, "cutadapt_primer_trimming_stats.txt") + summary_path = joinpath(log_dir, "cutadapt_trimmed_percentage.txt") + stats_basename = basename(stats_path) summary_basename = basename(summary_path) isdir(log_dir) || mkpath(log_dir) @@ -77,15 +73,15 @@ export cutadapt push!(samples, split(f, '_')[1]) end - @info("cutadapt running at $time with arguments: $primer_args $optional_args.") + @info("cutadapt running with arguments: $primer_args $optional_args.") nsamples = length(samples) for (i, sample) in enumerate(samples) inputR1 = joinpath(fastq_in_dir, sample * "_*_L001_R1_001.fastq.gz") inputR2 = joinpath(fastq_in_dir, sample * "_*_L001_R2_001.fastq.gz") - outputR1 = joinpath(fastq_out_dir, sample * "_R1_trimmed.fastq.gz") - outputR2 = joinpath(fastq_out_dir, sample * "_R2_trimmed.fastq.gz") + outputR1 = joinpath(cutadapt_dir, sample * "_R1_trimmed.fastq.gz") + outputR2 = joinpath(cutadapt_dir, sample * "_R2_trimmed.fastq.gz") @info("On sample $i/$nsamples ($sample).") cutadapt_cmd = "$cutadapt_bin $primer_args $optional_args -o $outputR1 -p $outputR2 $inputR1 $inputR2" @@ -93,7 +89,6 @@ export cutadapt open(stats_path, "a") do io run(pipeline(`bash -lc $cutadapt_cmd`; stdout=io, stderr=io)) end - end samples_str = join(samples, " ") @@ -106,7 +101,8 @@ export cutadapt run(pipeline(`bash -lc $cmd`)) end - @info("cutadapt complete. Output available in $fastq_out_dir.") + @info("cutadapt complete. Output available in $cutadapt_dir.") + return cutadapt_dir end """ @@ -154,7 +150,7 @@ export cutadapt optional_args = "-m 200 --discard-untrimmed", cutadapt_bin = "cutadapt" ) - run_cutadapt( + return run_cutadapt( get_primer_args(primer_pairs, primers_path), optional_args, fastq_in_dir, diff --git a/src/run_vsearch.jl b/src/run_vsearch.jl index 86eb53d..84399a6 100644 --- a/src/run_vsearch.jl +++ b/src/run_vsearch.jl @@ -19,6 +19,7 @@ export vsearch """ function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") + mkpath(vsearch_dir) outfile = joinpath(vsearch_dir, "taxonomy.tsv") cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" From ce7ce6f9eb0d2958935d45e2bc4fe0fd6b555662 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 16:34:45 +0100 Subject: [PATCH 019/175] Remove dead R modules. Good riddance. --- src/assign_taxonomy_worker.r | 38 ----- src/dada2.r | 274 ----------------------------------- 2 files changed, 312 deletions(-) delete mode 100644 src/assign_taxonomy_worker.r delete mode 100644 src/dada2.r diff --git a/src/assign_taxonomy_worker.r b/src/assign_taxonomy_worker.r deleted file mode 100644 index 56348eb..0000000 --- a/src/assign_taxonomy_worker.r +++ /dev/null @@ -1,38 +0,0 @@ -# Worker script for assignTaxonomy(). -# -# Invoked as a subprocess by assign_taxonomy() in dada2.jl. On Unix this -# process is launched with `nice -n 10` so that the CPU-intensive naive -# Bayesian classification does not monopolise all cores and lock up the -# system. Memory for the reference database is also freed when this process -# exits rather than accumulating in the Julia/RCall session. -# -# Usage (internal - do not call directly): -# Rscript assign_taxonomy_worker.r \ -# \ -# - -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 8L) { - stop("assign_taxonomy_worker.r: expected 8 arguments") -} - -functions_r <- args[[1L]] -chimera_ckpt <- args[[2L]] -db_path <- args[[3L]] -taxa_ckpt <- args[[4L]] -multithread <- as.integer(args[[5L]]) -min_boot <- as.numeric(args[[6L]]) -levels <- strsplit(args[[7L]], ",", fixed = TRUE)[[1L]] -verbose <- as.logical(args[[8L]]) - -source(functions_r) -load(chimera_ckpt) # provides seq_table_nochim - -taxa_result <- run_assign_taxonomy(seq_table_nochim, db_path, - list(multithread = multithread, - min_boot = min_boot, - levels = levels), - verbose) - -save(taxa_result, file = taxa_ckpt) -message("Taxonomy assignment complete. Saved to ", taxa_ckpt) diff --git a/src/dada2.r b/src/dada2.r deleted file mode 100644 index 968ded7..0000000 --- a/src/dada2.r +++ /dev/null @@ -1,274 +0,0 @@ -# DADA2 amplicon sequencing pipeline -# -# Processes paired- or single-end Illumina reads into ASVs and assigns -# taxonomy using a reference database (default: PR2). Options for output -# of bootstraps or counts only, as well as parameters for filterAndTrim(), -# dada(), mergePairs(), and assignTaxonomy(). -# -# Usage: -# Rscript dada2.r -# -# Outputs (in workspace.root/Tables/): -# seqtab_nochim.csv - chimera-free ASV count table (sequences as rows) -# asvs.fasta / asvs.csv - ASV sequences with short identifiers (seq1, seq2 ...) -# taxonomy.csv - taxonomy assignments (optionally with bootstrap confidence values) -# tax_counts.xlsx - combined taxonomy + per-sample counts -# pipeline_stats.csv - read counts at each pipeline stage -# -# Notice: -# -# © 2026 Joshua Benjamin Jewell. All rights reserved. -# -# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). -# -# This work is based on the DADA2 tutorial by Benjamin J. Callahan, et al., -# available at https://benjjneb.github.io/dada2/tutorial.html, with modification -# into a single module. The original material is licensed under the Creative -# Commons Attribution 4.0 International License (CC BY 4.0): -# https://creativecommons.org/licenses/by/4.0/. - -source("./src/dada2_functions.r") - -## Pipeline -main <- function() { - # Load args - args <- commandArgs(trailingOnly = TRUE) - if (length(args) == 0) { - stop("Usage: Rscript dada2_newer.r ") - } - yaml_path <- args[1] - - # Load config - message("Loading and validating config: ", yaml_path) - cfg <- load_config(yaml_path) - cfg <- validate_config(cfg) - verbose <- cfg$verbose %||% TRUE - mode <- cfg$file_patterns$mode %||% "paired" - - message("Setting up workspace: ", cfg$workspace$root) - paths <- setup_workspace(cfg$workspace$root) - - # Find input files - message("Discovering FASTQ files (mode: ", mode, ")") - fastq <- find_fastq_files( - cfg$workspace$input_dir, - cfg$file_patterns$forward, - cfg$file_patterns$reverse, - mode - ) - validate_sample_files(fastq$forward, fastq$reverse, mode) - - # In forward or reverse-only mode, sample names come from whichever side - # is present; the split pattern in config determines what counts as a name. - primary_files <- fastq$forward %||% fastq$reverse - sample_names <- extract_sample_names( - primary_files, - cfg$file_patterns$sample_name_split, - cfg$file_patterns$sample_name_index - ) - message(" Found ", length(sample_names), " samples") - - # Single-sample fallback: - # dada() returns a bare dada object (not a list) when given a single file. - # This breaks makeSequenceTable(), mergePairs(), and sapply() downstream. - # Workaround: duplicate the file paths so the pipeline sees 2 samples, then - # drop the duplicate row from all results before writing outputs. - # ASV calls are unaffected because dada() processes each file independently. - single_sample <- length(sample_names) == 1 - if (single_sample) { - warning("Only 1 sample found. Duplicating it to work around dada() returning ", - "a bare object for single-file input. The duplicate will be dropped ", - "from all outputs.") - fastq$forward <- rep(fastq$forward, 2) - if (!is.null(fastq$reverse)) fastq$reverse <- rep(fastq$reverse, 2) - sample_names <- c(sample_names, paste0(sample_names, "_dup")) - } - - # Quality assessment (pre-filter) - # Inspect quality_unfiltered.pdf to choose truncLen and maxEE values. - message("Plotting unfiltered quality profiles") - plot_quality_profiles( - fastq$forward, - fastq$reverse, - file.path(paths$Figures, "quality_unfiltered.pdf") - ) - - # Filter and trim - # Removes low-quality bases and reads. Reads that pass filtering are written - # to Filtered/; empty samples (0 reads) are silently skipped by DADA2 later. - message("Filtering and trimming reads") - filtered <- make_filtered_paths(sample_names, paths$Filtered, mode) - - # For single-end modes, route the relevant files into the fwd slots and - # pass NULL for rev so run_filter_trim() calls the correct filterAndTrim form. - in_fwd <- if (mode != "reverse") fastq$forward else fastq$reverse - out_fwd <- if (mode != "reverse") filtered$forward else filtered$reverse - in_rev_arg <- if (mode == "paired") fastq$reverse else NULL - out_rev_arg <- if (mode == "paired") filtered$reverse else NULL - - filter_stats <- run_filter_trim(in_fwd, in_rev_arg, out_fwd, out_rev_arg, - cfg$filter_trim, verbose) - - message("Plotting filtered quality profiles") - plot_quality_profiles( - filtered$forward, - filtered$reverse, - file.path(paths$Figures, "quality_filtered.pdf") - ) - - # Error model - # set.seed() should be called before learnErrors() - message("Learning error rates") - set.seed(cfg$dada$seed %||% 123L) - - fwd_errors <- if (mode != "reverse") { - learnErrors(filtered$forward, - nbases = cfg$dada$nbases, MAX_CONSIST = cfg$dada$max_consist, - verbose = verbose) - } else NULL - - rev_errors <- if (mode != "forward") { - learnErrors(filtered$reverse, - nbases = cfg$dada$nbases, MAX_CONSIST = cfg$dada$max_consist, - verbose = verbose) - } else NULL - - # Inspect error_rates.pdf: the fitted line should follow the observed points. - plot_error_rates(fwd_errors, rev_errors, - file.path(paths$Figures, "error_rates.pdf")) - - # Denoising - # dada() applies the error model to resolve ASV's from sequencing errors. - # pool = "pseudo" shares information across samples to improve detection of - # rare variants. - message("Denoising reads") - dada_fwd <- if (!is.null(fwd_errors)) { - dada(filtered$forward, err = fwd_errors, pool = cfg$dada$pool_method, - verbose = verbose) - } else NULL - - dada_rev <- if (!is.null(rev_errors)) { - dada(filtered$reverse, err = rev_errors, pool = cfg$dada$pool_method, - verbose = verbose) - } else NULL - - # Merge and sequence table - # Paired reads are joined at the overlapping region. Longer overlap and - # lower maxMismatch = higher confidence merges but fewer total merges. - message("Building sequence table") - if (mode == "paired") { - merged <- mergePairs( - dada_fwd, filtered$forward, - dada_rev, filtered$reverse, - minOverlap = cfg$merge$min_overlap, - maxMismatch = cfg$merge$max_mismatch, - trimOverhang = cfg$merge$trim_overhang, - verbose = verbose - ) - seq_table <- makeSequenceTable(merged) - } else { - merged <- NULL - seq_table <- makeSequenceTable(dada_fwd %||% dada_rev) - } - - # Plot the full length distribution before any length filtering so that - # off-target bands are visible when choosing band_size_min / band_size_max. - plot_length_distribution(seq_table, - file.path(paths$Figures, "length_distribution.pdf")) - - # Length filtering: retains only ASVs matching the expected amplicon size. - # Set band_size_min / band_size_max to null in config to skip this step. - band_min <- cfg$asv$band_size_min - band_max <- cfg$asv$band_size_max - if (!is.null(band_min) && !is.null(band_max)) { - message(" Filtering by length: ", band_min, "-", band_max, " bp") - seq_table <- filter_by_length(seq_table, band_min, band_max) - # Second plot confirms off-target lengths were successfully removed. - plot_length_distribution(seq_table, - file.path(paths$Figures, "length_distribution_filtered.pdf")) - } - - # Chimera removal - # "consensus" removes sequences identified as chimeric in the majority - # of samples in which they appear. - message("Removing chimeras") - seq_table_nochim <- removeBimeraDenovo(seq_table, method = cfg$asv$denovo_method, - verbose = verbose) - - nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 - message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), - "% | Retained: ", round(nochim_pct, 2), "%") - - # Drop duplicate row (if using single-sample fallback) - if (single_sample) { - filter_stats <- filter_stats[1, , drop = FALSE] - if (!is.null(dada_fwd)) dada_fwd <- dada_fwd[1] - if (!is.null(dada_rev)) dada_rev <- dada_rev[1] - if (!is.null(merged)) merged <- merged[1] - seq_table_nochim <- seq_table_nochim[1, , drop = FALSE] - sample_names <- sample_names[1] - } - - # Pipeline stats - message("Computing pipeline stats") - stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, - seq_table_nochim, sample_names, mode) - write.csv(stats, file.path(paths$Tables, "pipeline_stats.csv"), quote = FALSE) - if (verbose) print(stats) - - # Write core outputs - write_seq_table(seq_table_nochim, paths$Tables, - cfg$output$seq_table_prefix %||% "seqtab_nochim") - - message("Writing output tables") - # write_fasta() returns index (SeqName - Sequence), which is the join key - # used in write_taxa_table() and write_combined_table() below. - index <- write_fasta(seq_table_nochim, paths$Tables, - cfg$output$fasta_prefix %||% "asvs") - - # combined_input() defaults to index (SeqName + Sequence + counts only), - # which is the alternative output mode. It is replaced with the full - # taxa_df when taxonomy is run and combined_mode = "regular". - combined_input <- index - - # Taxonomy - if (!(cfg$taxonomy$skip %||% FALSE)) { - message("Assigning taxonomy") - db_path <- fetch_taxonomy_db(cfg$taxonomy$uri, paths$Taxonomy) - taxa_result <- run_assign_taxonomy(seq_table_nochim, db_path, - cfg$taxonomy, verbose) - taxa_df <- write_taxa_table( - taxa_result$tax, taxa_result$boot, index, - paths$Tables, - cfg$output$taxa_prefix %||% "taxonomy", - cfg$output$bootstraps %||% "combined" - ) - if ((cfg$output$combined_mode %||% "regular") == "regular") { - combined_input <- taxa_df - } - } else { - message(" Skipping taxonomy (taxonomy.skip = true)") - } - - # Checkpoint saved after taxonomy so the full environment (including taxa) - # can be restored with load() without re-running the pipeline. - checkpoint <- file.path(paths$Analysis, "checkpoint.RData") - save.image(checkpoint) - message(" Checkpoint saved: ", checkpoint) - - write_combined_table(combined_input, seq_table_nochim, paths$Tables, - cfg$output$combined_filename %||% "tax_counts.xlsx") - - message("Pipeline complete.") - message("Outputs:") - message(" ", file.path(paths$Tables, paste0(cfg$output$seq_table_prefix %||% "seqtab_nochim", ".csv"))) - message(" ", file.path(paths$Tables, paste0(cfg$output$fasta_prefix %||% "asvs", ".fasta"))) - message(" ", file.path(paths$Tables, paste0(cfg$output$fasta_prefix %||% "asvs", ".csv"))) - message(" ", file.path(paths$Tables, paste0(cfg$output$taxa_prefix %||% "taxonomy", ".csv"))) - message(" ", file.path(paths$Tables, cfg$output$combined_filename %||% "tax_counts.xlsx")) - message(" ", file.path(paths$Tables, "pipeline_stats.csv")) - message(" ", checkpoint) -} - -# Run pipeline -if (sys.nframe() == 0L) main() \ No newline at end of file From 1a8630a928fa3b9f8ca4f01681d95cebcf28f0e1 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 20:43:47 +0100 Subject: [PATCH 020/175] Split massive dada2.jl, merged smaller run_*.jl, reorganised directories. --- data/MiSeq_SOP/HMP_MOCK.v35.fasta | 64 +++ data/MiSeq_SOP/mouse.dpw.metadata | 20 + data/MiSeq_SOP/mouse.time.design | 20 + data/MiSeq_SOP/stability.batch | 26 + data/MiSeq_SOP/stability.files | 20 + requirements.txt | 1 - src/{run_cutadapt.jl => call_tools.jl} | 102 +++- src/dada2.jl | 686 ++----------------------- src/dada2/chimera.jl | 76 +++ src/dada2/context.jl | 235 +++++++++ src/{ => dada2}/dada2_functions.r | 4 +- src/dada2/denoise.jl | 156 ++++++ src/dada2/qc.jl | 94 ++++ src/dada2/taxonomy.jl | 87 ++++ src/databases.jl | 6 +- src/main.jl | 46 +- src/merge_and_filter_taxa.jl | 8 +- src/run_dada2.jl | 21 - src/run_vsearch.jl | 30 -- 19 files changed, 958 insertions(+), 744 deletions(-) create mode 100644 data/MiSeq_SOP/HMP_MOCK.v35.fasta create mode 100644 data/MiSeq_SOP/mouse.dpw.metadata create mode 100644 data/MiSeq_SOP/mouse.time.design create mode 100644 data/MiSeq_SOP/stability.batch create mode 100644 data/MiSeq_SOP/stability.files delete mode 100644 requirements.txt rename src/{run_cutadapt.jl => call_tools.jl} (58%) create mode 100644 src/dada2/chimera.jl create mode 100644 src/dada2/context.jl rename src/{ => dada2}/dada2_functions.r (99%) create mode 100644 src/dada2/denoise.jl create mode 100644 src/dada2/qc.jl create mode 100644 src/dada2/taxonomy.jl delete mode 100644 src/run_dada2.jl delete mode 100644 src/run_vsearch.jl diff --git a/data/MiSeq_SOP/HMP_MOCK.v35.fasta b/data/MiSeq_SOP/HMP_MOCK.v35.fasta new file mode 100644 index 0000000..c1748ed --- /dev/null +++ b/data/MiSeq_SOP/HMP_MOCK.v35.fasta @@ -0,0 +1,64 @@ +>A.baumannii.1 +TGGGGAATATTGGACAATGGGGGGAACCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGCCTTATGGTTGTAAAGCACTTTAAGCGAGGAGGAGGCTACTTTAGTTAATACCTAGAGATAGTGGACGTTACTCGCAGAATAAGCACCGGCTAACTCTGTGCCAGCAGCCGCGGTAATACAGAGGGTGCGAGCGTTAATCGGATTTACTGGGCGTAAAGCGTGCGTAGGCGGCTTATTAAGTCGGATGTGAAATCCCCGAGCTTAACTTGGGAATTGCATTCGATACTGGTGAGCTAGAGTATGGGAGAGGATGGTAGAATTCCAGGTGTAGCGGTGAAATGCGTAGAGATCTGGAGGAATACCGATGGCGAAGGCAGCCATCTGGCCTAATACTGACGCTGAGGTACGAAAGCATGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCATGCCGTAAACGATGTCTACTAGCCGTTGGGGCCTTTGAGGCTTTAGTGGCGCAGCTAACGCGATAAGTAGACCGCCTGGGGAGTACGGTC +>A.odontolyticus.1 +TGGGGAATATTGCACAATGGGCGAAAGCCTGATGCAGCGACGCCGCGTGAGGGATGGAGGCCTTCGGGTTGTAAACCTCTTTCGCTCATGGTCAAGCCGCAACTCAAGGTTGTGGTGAGGGTAGTGGGTAAAGAAGCGCCGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGGCGCGAGCGTTGTCCGGAATTATTGGGCGTAAAGGGCTTGTAGGCGGTTGGTCGCGTCTGCCGTGAAATCCTCTGGCTTAACTGGGGGCGTGCGGTGGGTACGGGCTGACTTGAGTGCGGTAGGGGAGACTGGAACTCCTGGTGTAGCGGTGGAATGCGCAGATATCAGGAAGAACACCGGTGGCGAAGGCGGGTCTCTGGGCCGTTACTGACGCTGAGGAGCGAAAGCGTGGGGAGCGAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGTTGGGCACTAGGTGTGGGGGCCACCCGTGGTTTCTGCGCCGTAGCTAACGCTTTAAGTGCCCCGCCTGGGGAGTACGGCC +>B.cereus.1 +TAGGGAATCTTCCGCAATGGACGAAAGTCTGACGGAGCAACGCCGCGTGAGTGATGAAGGCTTTCGGGTCGTAAAACTCTGTTGTTAGGGAAGAACAAGTGCTAGTTGAATAAGCTGGCACCTTGACGGTACCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGCAGGTGGTTTCTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACACTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGAGGGTTTCCGCCCTTTAGTGCTGAAGTTAACGCATTAAGCACTCCGCCTGGGGAGTACGGCC +>B.vulgatus.1 +TGAGGAATATTGGTCAATGGGCGCAGGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG +>B.vulgatus.2 +TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG +>B.vulgatus.4 +TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATATTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACTGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG +>B.vulgatus.5 +TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACTGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG +>B.vulgatus.7 +TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGCATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAGGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG +>C.beijerinckii.1 +TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCCTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGCCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC +>C.beijerinckii.3 +TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCCTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGTCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC +>C.beijerinckii.4 +TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCTTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGCCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC +>D.radiodurans.1 +TTAGGAATCTTCCACAATGGGCGCAAGCCTGATGGAGCGACGCCGCGTGAGGGATGAAGGTTTTCGGATCGTAAACCTCTGAATCTGGGACGAAAGAGCCTTAGGGCAGATGACGGTACCAGAGTAATAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTACCCGGAATCACTGGGCGTAAAGGGCGTGTAGGCGGAAATTTAAGTCTGGTTTTAAAGACCGGGGCTCAACCTCGGGGATGGACTGGATACTGGATTTCTTGACCTCTGGAGAGGTAACTGGAATTCCTGGTGTAGCGGTGGAATGCGTAGATACCAGGAGGAACACCAATGGCGAAGGCAAGTTACTGGACAGAAGGTGACGCTGAGGCGCGAAAGTGTGGGGAGCAAACCGGATTAGATACCCGGGTAGTCCACACCCTAAACGATGTACGTTGGCTAAGCGCAGGATGCTGTGCTTGGCGAAGCTAACGCGATAAACGTACCGCCTGGGAAGTACGGCC +>E.faecalis.1 +TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAACTCTGTTGTTAGAGAAGAACAAGGACGTTAGTAACTGAACGTCCCCTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCCCGGCTCAACCGGGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTGGAGGGTTTCCGCCCTTCAGTGCTGCAGCAAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>E.faecalis.2 +TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAACTCTGTTGTTAGAGAAGAACAAGGACGTTAGTAACTGAACGTCCCCTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCCCGGCTCAACCGGGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTGGAGGGTTTCCGCCCTTCAGTGCTGCAGCAAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>E.coli.1 +TGGGGAATATTGCACAATGGGCGCAAGCCTGATGCAGCCATGCCGCGTGTATGAAGAAGGCCTTCGGGTTGTAAAGTACTTTCAGCGGGGAGGAAGGGAGTAAAGTTAATACCTTTGCTCATTGACGTTACCCGCAGAAGAAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCACGCAGGCGGTTTGTTAAGTCAGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCTGATACTGGCAAGCTTGAGTCTCGTAGAGGGGGGTAGAATTCCAGGTGTAGCGGTGAAATGCGTAGAGATCTGGAGGAATACCGGTGGCGAAGGCGGCCCCCTGGACGAAGACTGACGCTCAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTTGGAGGTTGTGCCCTTGAGGCGTGGCTTCCGGAGCTAACGCGTTAAGTCGACCGCCTGGGGAGTACGGCC +>H.pylori.1 +TAGGGAATATTGCTCAATGGGGGAAACCCTGAAGCAGCAACGCCGCGTGGAGGATGAAGGTTTTAGGATTGTAAACTCCTTTTGTTAGAGAAGATAATGACGGTATCTAACGAATAAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTACTCGGAATCACTGGGCGTAAAGAGCGCGTAGGCGGGATAGTCAGTCAGGTGTGAAATCCTATGGCTTAACCATAGAACTGCATTTGAAACTACTATTCTAGAGTGTGGGAGAGGTAGGTGGAATTCTTGGTGTAGGGGTAAAATCCGTAGAGATCAAGAGGAATACTCATTGCGAAGGCGACCTGCTGGAACATTACTGACGCTGATTGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCCTAAACGATGGATGCTAGTTGTTGGAGGGCTTAGTCTCTCCAGTAATGCAGCTAACGCATTAAGCATCCCGCCTGGGGAGTACGGTC +>L.gasseri.1 +TAGGGAATCTTCCACAATGGACGCAAGTCTGATGGAGCAACGCCGCGTGAGTGAAGAAGGGTTTCGGCTCGTAAAGCTCTGTTGGTAGTGAAGAAAGATAGAGGTAGTAACTGGCCTTTATTTGACGGTAATTACTTAGAAAGTCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGTGCAGGCGGTTCAATAAGTCTGATGTGAAAGCCTTCGGCTCAACCGGAGAATTGCATCAGAAACTGTTGAACTTGAGTGCAGAAGAGGAGAGTGGAACTCCATGTGTAGCGGTGGAATGCGTAGATATATGGAAGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGCAACTGACGCTGAGGCTCGAAAGCATGGGTAGCGAACAGGATTAGATACCCTGGTAGTCCATGCCGTAAACGATGAGTGCTAAGTGTTGGGAGGTTTCCGCCTCTCAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>L.monocytogenes.1 +TAGGGAATCTTCCGCAATGGACGAAAGTCTGACGGAGCAACGCCGCGTGTATGAAGAAGGTTTTCGGATCGTAAAGTACTGTTGTTAGAGAAGAACAAGGATAAGAGTAACTGCTTGTCCCTTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGCGCGCAGGCGGTCTTTTAAGTCTGATGTGAAAGCCCCCGGCTTAACCGGGGAGGGTCATTGGAAACTGGAAGACTGGAGTGCAGAAGAGGAGAGTGGAATTCCACGTGTAGCGGTGAAATGCGTAGATATGTGGAGGAACACCAGTGGCGAAGGCGACTCTCTGGTCTGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>N.meningitidis.1 +TGGGGAATTTTGGACAATGGGCGCAAGCCTGATCCAGCCATGCCGCGTGTCTGAAGAAGGCCTTCGGGTTGTAAAGGACTTTTGTCAGGGAAGAAAAGGCTGTTGCTAATATCAGCGGCTGATGACGGTACCTGAAGAATAAGCACCGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGGTGCGAGCGTTAATCGGAATTACTGGGCGTAAAGCGGGCGCAGACGGTTACTTAAGCAGGATGTGAAATCCCCGGGCTCAACCCGGGAACTGCGTTCTGAACTGGGTGACTCGAGTGTGTCAGAGGGAGGTAGAATTCCACGTGTAGCAGTGAAATGCGTAGAGATGTGGAGGAATACCGATGGCGAAGGCAGCCTCCTGGGACAACACTGACGTTCATGCCCGAAAGCGTGGGTAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCCTAAACGATGTCAATTAGCTGTTGGGCAACCTGATTGCTTGGTAGCGTAGCTAACGCGTGAAATTGACCGCCTGGGGAGTACGGTC +>P.acnes.1 +TGGGGAATATTGCACAATGGGCGGAAGCCTGATGCAGCAACGCCGCGTGCGGGATGACGGCCTTCGGGTTGTAAACCGCTTTCGCCTGTGACGAAGCGTGAGTGACGGTAATGGGTAAAGAAGCACCGGCTAACTACGTGCCAGCAGCCGCGGTGATACGTAGGGTGCGAGCGTTGTCCGGATTTATTGGGCGTAAAGGGCTCGTAGGTGGTTGATCGCGTCGGAAGTGTAATCTTGGGGCTTAACCCTGAGCGTGCTTTCGATACGGGTTGACTTGAGGAAGGTAGGGGAGAATGGAATTCCTGGTGGAGCGGTGGAATGCGCAGATATCAGGAGGAACACCAGTGGCGAAGGCGGTTCTCTGGGCCTTTCCTGACGCTGAGGAGCGAAAGCGTGGGGAGCGAACAGGCTTAGATACCCTGGTAGTCCACGCTGTAAACGGTGGGTACTAGGTGTGGGGTCCATTCCACGGGTTCCGTGCCGTAGCTAACGCTTTAAGTACCCCGCCTGGGGAGTACGGCC +>P.aeruginosa.1 +TGGGGAATATTGGACAATGGGCGAAAGCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGTCTTCGGATTGTAAAGCACTTTAAGTTGGGAGGAAGGGCAGTAAGTTAATACCTTGCTGTTTTGACGTTACCAACAGAATAAGCACCGGCTAACTTCGTGCCAGCAGCCGCGGTAATACGAAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCGCGTAGGTGGTTCAGCAAGTTGGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCCAAAACTACTGAGCTAGAGTACGGTAGAGGGTGGTGGAATTTCCTGTGTAGCGGTGAAATGCGTAGATATAGGAAGGAACACCAGTGGCGAAGGCGACCACCTGGACTGATACTGACACTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTAGCCGTTGGGATCCTTGAGATCTTAGTGGCGCAGCTAACGCGATAAGTCGACCGCCTGGGGAGTACGGCC +>P.aeruginosa.2 +TGGGGAATATTGGACAATGGGCGAAAGCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGTCTTCGGATTGTAAAGCACTTTAAGTTGGGAGGAAGGGCAGTAAGTTAATACCTTGCTGTTTTGACGTTACCAACAGAATAAGCACCGGCTAACTTCGTGCCAGCAGCCGCGGTAATACGAAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCGCGTAGGTGGTTCAGCAAGTTGGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCCAAAACTACTGAGCTAGAGTACGGTAGAGGGTGGTGGAATTTCCTGTGTAGCGGTGAAATGCGTAGATATAGGAAGGAACACCAGTGGCGAAGGCGACCACCTGGACTGATACTGACACTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTAGCCGTTGGGATCCTTGAGATCATAGTGGCGCAGCTAACGCGATAAGTCGACCGCCTGGGGAGTACGGCC +>P.gingivalis.1 +TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTCGCGTGAAGGAAGACTGTCCTAAGGATTGTAAACTTCTTTTATACGGGAATAACGGGCGATACGAGTATTGCATTGAATGTACCGTAAGAATAAGCATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGTTGTTCGGTAAGTCAGCGGTGAAACCTGAGCGCTCAACGTTCAGCCTGCCGTTGAAACTGCCGGGCTTGAGTTCAGCGGCGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCATAGATATCACGAGGAACTCCGATTGCGAAGGCAGCTTGCCATACTGCGACTGACACTGAAGCACGAAGGCGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATTACTAGGAGTTTGCGATATACCGTCAAGCTTCCACAGCGAAAGCGTTAAGTAATCCACCTGGGGAGTACGCCG +>R.sphaeroides.1 +TGGGGAATCTTAGACAATGGGCGCAAGCCTGATCTAGCCATGCCGCGTGATCGATGAAGGCCTTAGGGTTGTAAAGATCTTTCAGGTGGGAAGATAATGACGGTACCACCAGAAGAAGCCCCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGGGCTAGCGTTATTCGGAATTACTGGGCGTAAAGCGCACGTAGGCGGATCGGAAAGTCAGAGGTGAAATCCCAGGGCTCAACCCTGGAACTGCCTTTGAAACTCCCGATCTTGAGGTCGAGAGAGGTGAGTGGAATTCCGAGTGTAGAGGTGAAATTCGTAGATATTCGGAGGAACACCAGTGGCGAAGGCGGCTCACTGGCTCGATACTGACGCTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATGCCAGTCGTCGGGCAGCATGCTGTTCGGTGACACACCTAACGGATTAAGCATTCCGCCTGGGGAGTACGGCC +>S.aureus.1 +TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACATATGTGTAAGTAACTGTGCACATCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.aureus.5 +TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACATATGTGTAAGTAACTGTGCACATCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTTATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.epidermidis.1 +TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTGTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.epidermidis.2 +TAGGGAATCTTCCGCAATGGGCGAAAGCTTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTGTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.epidermidis.5 +TAGGGAATCTTCCGCAATGGGCGAAAGCTTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTCTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGAATCATTGGGCGTAAAGCGCGCGTAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAGGACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.agalactiae.1 +TAGGGAATCTTCGGCAATGGACGGAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTTAGAGAAGAACGTTGGTAGGAGTGGAAAATCTACCAAGTGACGGTAACTAACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTCTTTAAGTCTGAAGTTAAAGGCAGTGGCTTAACCATTGTACGCTTTGGAAACTGGAGGACTTGAGTGCAGAAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCGGTGGCGAAAGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAGGTGTTAGGCCCTTTCCGGGGCTTAGTGCCGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC +>S.mutans.1 +TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTAAGTCAAGAACGTGTGTGAGAGTGGAAAGTTCACACAGTGACGGTAGCTTACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGGGAGCGCAGGCGGTCAGGAAAGTCTGGAGTAAAAGGCTATGGCTCAACCATAGTGTGCTCTGGAAACTGTCTGACTTGAGTGCAGAAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAAGCGGCTCTCTGGTCTGTCACTGACGCTGAGGCTCGAAAGCGTGGGTAGCGAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAGGTGTTAGGCCCTTTCCGGGGCTTAGTGCCGGAGCTAACGCAATAAGCACTCCGCCTGGGGAGTACGACC +>S.pneumoniae.1 +TAGGGAATCTTCGGCAATGGACGGAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTAAGAGAAGAACGAGTGTGAGAGTGGAAAGTTCACACTGTGACGGTATCTTACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTAGATAAGTCTGAAGTTAAAGGCTGTGGCTTAACCATAGTAGGCTTTGGAAACTGTTTAACTTGAGTGCAAGAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCGGTGGCGAAAGCGGCTCTCTGGCTTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGAGTGCTAGGTGTTAGACCCTTTCCGGGGTTTAGTGCCGTAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC diff --git a/data/MiSeq_SOP/mouse.dpw.metadata b/data/MiSeq_SOP/mouse.dpw.metadata new file mode 100644 index 0000000..e754f1c --- /dev/null +++ b/data/MiSeq_SOP/mouse.dpw.metadata @@ -0,0 +1,20 @@ +group dpw +F3D0 0 +F3D1 1 +F3D141 141 +F3D142 142 +F3D143 143 +F3D144 144 +F3D145 145 +F3D146 146 +F3D147 147 +F3D148 148 +F3D149 149 +F3D150 150 +F3D2 2 +F3D3 3 +F3D5 5 +F3D6 6 +F3D7 7 +F3D8 8 +F3D9 9 diff --git a/data/MiSeq_SOP/mouse.time.design b/data/MiSeq_SOP/mouse.time.design new file mode 100644 index 0000000..16a7421 --- /dev/null +++ b/data/MiSeq_SOP/mouse.time.design @@ -0,0 +1,20 @@ +group time +F3D0 Early +F3D1 Early +F3D141 Late +F3D142 Late +F3D143 Late +F3D144 Late +F3D145 Late +F3D146 Late +F3D147 Late +F3D148 Late +F3D149 Late +F3D150 Late +F3D2 Early +F3D3 Early +F3D5 Early +F3D6 Early +F3D7 Early +F3D8 Early +F3D9 Early diff --git a/data/MiSeq_SOP/stability.batch b/data/MiSeq_SOP/stability.batch new file mode 100644 index 0000000..b818716 --- /dev/null +++ b/data/MiSeq_SOP/stability.batch @@ -0,0 +1,26 @@ +pcr.seqs(fasta=silva.bacteria.fasta, start=11894, end=25319, keepdots=F) +rename.file(input=silva.bacteria.pcr.fasta, new=silva.v4.fasta) + +#change the name of the file from stability.files to whatever suits your study +make.file(inputdir=., type=fastq, prefix=stability) +make.contigs(file=current, maxambig=0, maxlength=275, maxhomop=8) +unique.seqs(fasta=stability.trim.contigs.fasta, count=stability.contigs.count_table) +align.seqs(fasta=current, reference=silva.v4.fasta) +screen.seqs(fasta=current, count=current, start=1969, end=11551) +filter.seqs(fasta=current, vertical=T, trump=.) +unique.seqs(fasta=current, count=current) +pre.cluster(fasta=current, count=current, diffs=2) +chimera.vsearch(fasta=current, count=current, dereplicate=t) +classify.seqs(fasta=current, count=current, reference=trainset9_032012.pds.fasta, taxonomy=trainset9_032012.pds.tax) +remove.lineage(fasta=current, count=current, taxonomy=current, taxon=Chloroplast-Mitochondria-unknown-Archaea-Eukaryota) +remove.groups(count=current, fasta=current, taxonomy=current, groups=Mock) +cluster.split(fasta=current, count=current, taxonomy=current, taxlevel=4, cutoff=0.03) +make.shared(list=current, count=current, label=0.03) +classify.otu(list=current, count=current, taxonomy=current, label=0.03) +phylotype(taxonomy=current) +make.shared(list=current, count=current, label=1) +classify.otu(list=current, count=current, taxonomy=current, label=1) +make.shared(count=current) +classify.otu(list=current, count=current, taxonomy=current, label=ASV) +dist.seqs(fasta=current, output=lt) +clearcut(phylip=current) diff --git a/data/MiSeq_SOP/stability.files b/data/MiSeq_SOP/stability.files new file mode 100644 index 0000000..0ff5fec --- /dev/null +++ b/data/MiSeq_SOP/stability.files @@ -0,0 +1,20 @@ +F3D0 F3D0_S188_L001_R1_001.fastq F3D0_S188_L001_R2_001.fastq +F3D141 F3D141_S207_L001_R1_001.fastq F3D141_S207_L001_R2_001.fastq +F3D142 F3D142_S208_L001_R1_001.fastq F3D142_S208_L001_R2_001.fastq +F3D143 F3D143_S209_L001_R1_001.fastq F3D143_S209_L001_R2_001.fastq +F3D144 F3D144_S210_L001_R1_001.fastq F3D144_S210_L001_R2_001.fastq +F3D145 F3D145_S211_L001_R1_001.fastq F3D145_S211_L001_R2_001.fastq +F3D146 F3D146_S212_L001_R1_001.fastq F3D146_S212_L001_R2_001.fastq +F3D147 F3D147_S213_L001_R1_001.fastq F3D147_S213_L001_R2_001.fastq +F3D148 F3D148_S214_L001_R1_001.fastq F3D148_S214_L001_R2_001.fastq +F3D149 F3D149_S215_L001_R1_001.fastq F3D149_S215_L001_R2_001.fastq +F3D150 F3D150_S216_L001_R1_001.fastq F3D150_S216_L001_R2_001.fastq +F3D1 F3D1_S189_L001_R1_001.fastq F3D1_S189_L001_R2_001.fastq +F3D2 F3D2_S190_L001_R1_001.fastq F3D2_S190_L001_R2_001.fastq +F3D3 F3D3_S191_L001_R1_001.fastq F3D3_S191_L001_R2_001.fastq +F3D5 F3D5_S193_L001_R1_001.fastq F3D5_S193_L001_R2_001.fastq +F3D6 F3D6_S194_L001_R1_001.fastq F3D6_S194_L001_R2_001.fastq +F3D7 F3D7_S195_L001_R1_001.fastq F3D7_S195_L001_R2_001.fastq +F3D8 F3D8_S196_L001_R1_001.fastq F3D8_S196_L001_R2_001.fastq +F3D9 F3D9_S197_L001_R1_001.fastq F3D9_S197_L001_R2_001.fastq +Mock Mock_S280_L001_R1_001.fastq Mock_S280_L001_R2_001.fastq diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8ae48c3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -cutadapt \ No newline at end of file diff --git a/src/run_cutadapt.jl b/src/call_tools.jl similarity index 58% rename from src/run_cutadapt.jl rename to src/call_tools.jl index b64eb07..7c9e6b4 100644 --- a/src/run_cutadapt.jl +++ b/src/call_tools.jl @@ -1,10 +1,39 @@ -module Cutadapt +module Tools -export cutadapt +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export cutadapt, vsearch, fastqc_all, fastqc_one using YAML using Logging + ## Tools loading from config + """ + load_tools(config_path) -> Dict{String,String} + + Read config/tools.yml and return a Dict of tool name => resolved path. + If the file does not exist, returns an empty Dict so tools fall back to PATH. + Paths with `@` are SSH remote paths (user@host:/path), the calling module is + responsible for routing those calls via SSH. + """ + function load_tools(config_path = joinpath(@__DIR__, "..", "config", "tools.yml")) + isfile(config_path) || return Dict{String,String}() + data = YAML.load_file(config_path) + tools = Dict{String,String}() + for (name, info) in data + path = info isa Dict ? get(info, "path", nothing) : nothing + path !== nothing && (tools[name] = string(path)) + end + tools + end + + tool_bin(key) = get(_tools, key, key) + + const _tools = load_tools() + + ## cutadapt # Prevent duplication of primers. Must be instantiated outside get_primers() loop. Global scope may be an issue. used_forward = String[] used_reverse = String[] @@ -148,7 +177,7 @@ export cutadapt fastq_in_dir, cutadapt_dir; optional_args = "-m 200 --discard-untrimmed", - cutadapt_bin = "cutadapt" + cutadapt_bin = tool_bin("cutadapt") ) return run_cutadapt( get_primer_args(primer_pairs, primers_path), @@ -158,4 +187,71 @@ export cutadapt cutadapt_bin ) end + + ## FastQC + """ + function fastqc_all(fastq_in_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = "fastqc") + + Requires `fastqc` installed. Runs `fastqc` command with any optional parameters on a whole set. + + ## Arguments + - `fastq_in_dir`: Specify path of fastq files to qc. + + ## Keyword Arguments + - `optional_args` (optional, default: "-t 20 --extract --delete"): Specify additional arguments passed to `fastqc` command. + - `fastqc_bin` (optional, default: "fastqc"): Path to the fastqc binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. + + """ + function fastqc_all(fastq_in_dir, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) + mkpath(fastqc_dir) + + cmd = "$fastqc_bin $fastq_in_dir/*.fastq* -o $fastqc_dir $optional_args" + run(`bash -lc $cmd`) + + end + + """ + function fastqc_one(fastq_in_file; optional_args = "-t 20 --extract --delete", fastqc_bin = "fastqc") + + Requires `fastqc` installed. Runs `fastqc` command with any optional parameters on a whole set. + + ## Arguments + - `fastq_in_file`: Specify path of fastq file to qc. + + ## Keyword Arguments + - `optional_args` (optional, default: "-t 20 --extract --delete"): Specify additional arguments passed to `fastqc` command. + - `fastqc_bin` (optional, default: "fastqc"): Path to the fastqc binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. + + """ + function fastqc_one(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) + mkpath(fastqc_dir) + + cmd = "$fastqc_bin $fastq_in_file -o $fastqc_dir $optional_args" + run(`bash -lc $cmd`) + + end + + ## VSEARCH + """ + vsearch(fasta_in_dir, reference_database; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") + + Requires `vsearch` installed. Runs `vsearch` command with any optional parameters to perform taxonomy assignment by local alignment against specified database. + + ## Arguments + - `fasta_in_dir`: Specify path of fasta files output by DADA2 pipeline. + - `reference_database` (default: "./databases"): Specify path of reference database. + + ## Keyword Arguments + - `optional_args` (optional, default: "--id 0.75 --query_cov 0.8"): Specify additional arguments passed to `vsearch` command. + - `vsearch_bin` (optional, default: "vsearch"): Path to the vsearch binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. + + """ + function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = tool_bin("vsearch")) + mkpath(vsearch_dir) + outfile = joinpath(vsearch_dir, "taxonomy.tsv") + + cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" + run(`bash -lc $cmd`) + + end end \ No newline at end of file diff --git a/src/dada2.jl b/src/dada2.jl index 74619d1..80d430b 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -1,23 +1,21 @@ module DADA2 -# DADA2 amplicon sequencing pipeline — Julia orchestrator +# DADA2 amplicon sequencing pipeline - Julia orchestrator # -# The pipeline is split into six independently callable stages so that -# intermediate outputs (quality profiles, error rate plots, length -# distributions, pipeline stats) can be reviewed and parameters adjusted -# before committing to the next step. Each stage saves its R objects to a -# per-stage checkpoint in Analysis/ and loads what it needs from the previous -# stage's checkpoint, making every stage re-runnable across Julia sessions. +# Stages and their Web UI page groupings: # -# Stage order: -# prefilter_qc → filter_trim → learn_errors → -# denoise → chimera_removal → assign_taxonomy +# QC page: prefilter_qc, filter_trim +# Denoising page: learn_errors, denoise, filter_length, chimera_removal +# Taxonomy page: assign_taxonomy +# +# Each stage is independently callable and saves a checkpoint so the pipeline +# can be resumed or individual stages re-run across Julia sessions. # # Call dada2() to run all stages in sequence without stopping. # # Notice: # -# © 2026 Joshua Benjamin Jewell. All rights reserved. +# (c) 2026 Joshua Benjamin Jewell. All rights reserved. # # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). # @@ -28,661 +26,53 @@ module DADA2 # https://creativecommons.org/licenses/by/4.0/. export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - chimera_removal, assign_taxonomy + filter_length, chimera_removal, assign_taxonomy import Downloads using Logging, RCall, YAML - # Helpers - - # Returns a function that routes progress messages to @info (CLI) or a - # Channel{String} (Genie SSE). Pass the result as `emit` in stage functions. - _emitter(::Nothing) = msg -> @info msg - _emitter(ch::Channel{String}) = msg -> put!(ch, msg) - - # Database resolution helpers - - # Download `uri` to `db_dir/basename(uri)` if not already cached. - # Respects the optional `local:` override in `fmt_info`. - function _download_db_if_needed(key, fmt_info, db_dir, emit) - local_p = get(fmt_info, "local", nothing) - if !isnothing(local_p) - local_p = string(local_p) - if !isempty(local_p) - isfile(local_p) && (emit("[$key] Using local file: $local_p"); return local_p) - @warn "[$key] Configured local path not found: $local_p — falling back to uri" - end - end - uri = fmt_info["uri"] - cached = joinpath(db_dir, basename(uri)) - if isfile(cached) - emit("[$key] Using cached: $cached") - else - emit("[$key] Downloading: $uri") - Downloads.download(uri, cached) - emit("[$key] Saved to: $cached") - end - return cached - end - - # Resolve the DADA2 taxonomy database from config when no external path is - # provided. Supports the `databases: / taxonomy.database:` scheme (preferred) - # and the legacy `taxonomy.uri:` key for backwards compatibility. - function _resolve_taxonomy_db(cfg, emit) - tax_cfg = cfg["taxonomy"] - if haskey(tax_cfg, "database") - db_key = string(tax_cfg["database"]) - db_cfg = get(cfg, "databases", Dict()) - haskey(db_cfg, db_key) || - error("taxonomy.database = \"$db_key\" not found in databases: section") - fmt_cfg = get(db_cfg[db_key], "dada2", nothing) - isnothing(fmt_cfg) && - error("databases.$db_key.dada2 is not configured") - db_dir = abspath(get(db_cfg, "dir", "./databases")) - mkpath(db_dir) - return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) - end - # Legacy: taxonomy.uri - tax_uri = tax_cfg["uri"] - if isfile(tax_uri) - emit("Using local taxonomy database: $tax_uri") - return tax_uri - end - db_dir = abspath(get(cfg, "databases_dir", "./databases")) - mkpath(db_dir) - cached = joinpath(db_dir, basename(tax_uri)) - if !isfile(cached) - emit("Downloading taxonomy database: $tax_uri") - Downloads.download(tax_uri, cached) - emit("Written: $cached") - else - emit("Using cached taxonomy database: $cached") - end - return cached - end - - # Config - function validate_config(cfg) - required = ["workspace", "file_patterns", "filter_trim", "dada", - "merge", "asv", "taxonomy", "output"] - missing_secs = filter(k -> !haskey(cfg, k), required) - isempty(missing_secs) || - error("Missing required config sections: $(join(missing_secs, ", "))") - - ws = cfg["workspace"] - haskey(ws, "root") || error("workspace.root is required") - haskey(ws, "input_dir") || error("workspace.input_dir is required") - isdir(ws["input_dir"]) || - error("workspace.input_dir does not exist: $(ws["input_dir"])") - - mode = get(cfg["file_patterns"], "mode", "paired") - mode in ("paired", "forward", "reverse") || - error("file_patterns.mode must be one of: paired, forward, reverse") - - boot_mode = get(cfg["output"], "bootstraps", "combined") - boot_mode in ("none", "combined", "separate") || - error("output.bootstraps must be one of: none, combined, separate") - - if !get(cfg["taxonomy"], "skip", false) - has_uri = haskey(cfg["taxonomy"], "uri") - has_db = haskey(cfg["taxonomy"], "database") - has_uri || has_db || - error("taxonomy: configure `database` (referencing a databases: entry) " * - "or `uri` when skip is not true") - end - - combined_mode = get(cfg["output"], "combined_mode", "regular") - combined_mode in ("regular", "alternative") || - error("output.combined_mode must be one of: regular, alternative") - end - - # File discovery - function find_fastq_files(input_dir, fwd_pattern, rev_pattern, mode) - all_files = sort(readdir(input_dir, join=true)) - fwd = mode in ("paired", "forward") ? - filter(f -> !isnothing(match(Regex(fwd_pattern), basename(f))), all_files) : String[] - rev = mode in ("paired", "reverse") ? - filter(f -> !isnothing(match(Regex(rev_pattern), basename(f))), all_files) : String[] - fwd, rev - end - - function validate_sample_files(fwd, rev, mode) - if mode in ("paired", "forward") - isempty(fwd) && - error("No forward FASTQ files found. " * - "Check workspace.input_dir and file_patterns.forward.") - for f in fwd - isfile(f) || error("Forward file not found: $f") - end - end - if mode in ("paired", "reverse") - isempty(rev) && - error("No reverse FASTQ files found. " * - "Check workspace.input_dir and file_patterns.reverse.") - for f in rev - isfile(f) || error("Reverse file not found: $f") - end - end - if mode == "paired" && length(fwd) != length(rev) - error("Forward file count ($(length(fwd))) does not match " * - "reverse file count ($(length(rev))).") - end - end - - function extract_sample_names(files, split_char, split_index) - [split(basename(f), split_char)[split_index] for f in files] - end - - # Workspace - function setup_workspace(root) - dirs = Dict( - "Tables" => joinpath(root, "Tables"), - "Checkpoints" => joinpath(root, "Checkpoints"), - "Figures" => joinpath(root, "Figures"), - "Filtered" => joinpath(root, "Filtered"), - ) - for d in values(dirs) - mkpath(d) - end - dirs - end - - # Pipeline context: - # Shared setup called at the start of every stage: sources R, loads and - # validates config, discovers files, handles the single-sample fallback, and - # computes all path variables. Returns a NamedTuple so stage functions can - # extract what they need without repeating boilerplate. - # - # Optional overrides (used when main.jl manages directory layout): - # input_dir — overrides cfg["workspace"]["input_dir"] - # workspace_root — overrides cfg["workspace"]["root"] - function _pipeline_context(config_path::String; input_dir=nothing, workspace_root=nothing) - functions_r = joinpath(@__DIR__, "dada2_functions.r") - R"source($functions_r)" - - cfg = YAML.load_file(config_path) - - # Apply caller-supplied overrides before validation so that validate_config - # sees the final paths (including the isdir check on input_dir). - if !isnothing(input_dir) - cfg["workspace"]["input_dir"] = input_dir - end - if !isnothing(workspace_root) - cfg["workspace"]["root"] = workspace_root - end - - validate_config(cfg) - verbose = get(cfg, "verbose", true) - mode = get(cfg["file_patterns"], "mode", "paired") - root = cfg["workspace"]["root"] - dirs = setup_workspace(root) - - fwd_files, rev_files = find_fastq_files( - cfg["workspace"]["input_dir"], - cfg["file_patterns"]["forward"], - cfg["file_patterns"]["reverse"], - mode) - validate_sample_files(fwd_files, rev_files, mode) - - primary_files = isempty(fwd_files) ? rev_files : fwd_files - sample_names = extract_sample_names( - primary_files, - cfg["file_patterns"]["sample_name_split"], - cfg["file_patterns"]["sample_name_index"]) - - # Single-sample fallback: dada() returns a bare object (not a list) for a - # single input file, breaking makeSequenceTable() and sapply() downstream. - # Duplicate the paths so the pipeline sees 2 samples; the extra row is - # dropped in chimera_removal(). ASV calls are unaffected. - single_sample = length(sample_names) == 1 - if single_sample - @warn "Only 1 sample found. Duplicating it to work around dada() " * - "returning a bare object for single-file input. The duplicate " * - "will be dropped from all outputs." - fwd_files = isempty(fwd_files) ? fwd_files : repeat(fwd_files, 2) - rev_files = isempty(rev_files) ? rev_files : repeat(rev_files, 2) - sample_names = [sample_names[1], sample_names[1] * "_dup"] - end - - filtered_dir = dirs["Filtered"] - fwd_out = mode != "reverse" ? - [joinpath(filtered_dir, s * "_R1_filt.fastq.gz") for s in sample_names] : String[] - rev_out = mode != "forward" ? - [joinpath(filtered_dir, s * "_R2_filt.fastq.gz") for s in sample_names] : String[] - - # For single-end modes, route the relevant reads into the forward slots. - in_fwd = mode != "reverse" ? fwd_files : rev_files - out_fwd = mode != "reverse" ? fwd_out : rev_out - in_rev_arg = mode == "paired" ? rev_files : nothing - out_rev_arg = mode == "paired" ? rev_out : nothing - - ckpts = Dict( - "filter" => joinpath(dirs["Checkpoints"], "ckpt_filter.RData"), - "errors" => joinpath(dirs["Checkpoints"], "ckpt_errors.RData"), - "denoise" => joinpath(dirs["Checkpoints"], "ckpt_denoise.RData"), - "chimera" => joinpath(dirs["Checkpoints"], "ckpt_chimera.RData"), - ) - - (; cfg, verbose, mode, dirs, sample_names, single_sample, - fwd_files, rev_files, fwd_out, rev_out, - in_fwd, out_fwd, in_rev_arg, out_rev_arg, ckpts) - end - - # Pre-filter quality assessment - """ - prefilter_qc(config_path; progress) - - **Stage 1** — Plot unfiltered quality profiles. - - Review `Figures/quality_unfiltered.pdf` to choose `truncLen` and `maxEE` - values in config before running `filter_trim()`. - """ - function prefilter_qc(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - - emit("Plotting unfiltered quality profiles") - fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files - rev_for_plot = isempty(ctx.rev_files) ? nothing : ctx.rev_files - unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") - R"plot_quality_profiles($fwd_for_plot, $rev_for_plot, $unfiltered_pdf)" - emit("Written: $unfiltered_pdf") - nothing - end - - # Filter and trim - """ - filter_trim(config_path; progress) - - **Stage 2** — Filter and trim reads; plot filtered quality profiles. - - Review `Figures/quality_filtered.pdf`. If filtering looks appropriate, - proceed to `learn_errors()`; otherwise adjust `truncLen` / `maxEE` in - config and re-run this stage. - - Saves: `Analysis/ckpt_filter.RData` - """ - function filter_trim(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - ft = ctx.cfg["filter_trim"] - trunc_len = ft["trunc_len"] - max_ee = ft["max_ee"] - verbose = ctx.verbose - in_fwd = ctx.in_fwd - out_fwd = ctx.out_fwd - in_rev = ctx.in_rev_arg - out_rev = ctx.out_rev_arg - fwd_out = ctx.fwd_out - rev_out = ctx.rev_out - - emit("Filtering and trimming reads") - if ctx.mode == "paired" - R""" - filter_stats <- filterAndTrim( - $in_fwd, $out_fwd, - $in_rev, $out_rev, - truncQ = $(ft["trunc_q"]), - truncLen = $trunc_len, - maxEE = $max_ee, - minLen = $(ft["min_len"]), - maxN = $(ft["max_n"]), - matchIDs = $(ft["match_ids"]), - rm.phix = $(ft["rm_phix"]), - verbose = $verbose - ) - """ - else - R""" - filter_stats <- filterAndTrim( - $in_fwd, $out_fwd, - truncQ = $(ft["trunc_q"]), - truncLen = $(trunc_len[1]), - maxEE = $(max_ee[1]), - minLen = $(ft["min_len"]), - maxN = $(ft["max_n"]), - rm.phix = $(ft["rm_phix"]), - verbose = $verbose - ) - """ - end - - emit("Plotting filtered quality profiles") - fwd_filt = isempty(fwd_out) ? nothing : fwd_out - rev_filt = isempty(rev_out) ? nothing : rev_out - filtered_pdf = joinpath(ctx.dirs["Figures"], "quality_filtered.pdf") - R"plot_quality_profiles($fwd_filt, $rev_filt, $filtered_pdf)" - - ckpt = ctx.ckpts["filter"] - R"save(filter_stats, file=$ckpt)" - emit("Written: $filtered_pdf") - emit("Checkpoint: $ckpt") - nothing - end - - # Error model - """ - learn_errors(config_path; progress) - - **Stage 3** — Learn substitution error rates and plot diagnostics. - - Review `Figures/error_rates.pdf`: the fitted line should closely follow the - observed points. If not, increase `nbases` or `max_consist` in config and - re-run this stage. - - Saves: `Analysis/ckpt_errors.RData` - """ - function learn_errors(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - seed = get(ctx.cfg["dada"], "seed", 123) - nbases = ctx.cfg["dada"]["nbases"] - max_con = ctx.cfg["dada"]["max_consist"] - verbose = ctx.verbose - fwd_out = ctx.fwd_out - rev_out = ctx.rev_out - - emit("Learning error rates") - R"set.seed($seed)" - - ctx.mode != "reverse" ? - R"fwd_errors <- learnErrors($fwd_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : - R"fwd_errors <- NULL" - - ctx.mode != "forward" ? - R"rev_errors <- learnErrors($rev_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : - R"rev_errors <- NULL" - - error_pdf = joinpath(ctx.dirs["Figures"], "error_rates.pdf") - R"plot_error_rates(fwd_errors, rev_errors, $error_pdf)" - - ckpt = ctx.ckpts["errors"] - R"save(fwd_errors, rev_errors, file=$ckpt)" - emit("Written: $error_pdf") - emit("Checkpoint: $ckpt") - nothing - end - - # Denoise - """ - denoise(config_path; progress) - - **Stage 4** — Denoise reads, merge pairs (paired mode), build the sequence - table, and plot the raw ASV length distribution. - - Review `Figures/length_distribution.pdf` and set `band_size_min` / - `band_size_max` in config to target the expected amplicon peak. Length - filtering is applied in `chimera_removal()`, so this stage does not need - to be re-run when adjusting the length cutoff. - - Requires: `Analysis/ckpt_errors.RData` - Saves: `Analysis/ckpt_denoise.RData` (unfiltered seq_table) - """ - function denoise(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - verbose = ctx.verbose - fwd_out = ctx.fwd_out - rev_out = ctx.rev_out - - isfile(ctx.ckpts["errors"]) || - error("Error model checkpoint not found. Run learn_errors() first.") - errors_ckpt = ctx.ckpts["errors"] - R"load($errors_ckpt)" - - emit("Denoising reads") - pool_method = ctx.cfg["dada"]["pool_method"] - - ctx.mode != "reverse" ? - R"dada_fwd <- dada($fwd_out, err=fwd_errors, pool=$pool_method, verbose=$verbose)" : - R"dada_fwd <- NULL" - - ctx.mode != "forward" ? - R"dada_rev <- dada($rev_out, err=rev_errors, pool=$pool_method, verbose=$verbose)" : - R"dada_rev <- NULL" - - emit("Building sequence table") - if ctx.mode == "paired" - min_overlap = ctx.cfg["merge"]["min_overlap"] - max_mismatch = ctx.cfg["merge"]["max_mismatch"] - trim_overhang = ctx.cfg["merge"]["trim_overhang"] - R""" - merged <- mergePairs( - dada_fwd, $fwd_out, - dada_rev, $rev_out, - minOverlap = $min_overlap, - maxMismatch = $max_mismatch, - trimOverhang = $trim_overhang, - verbose = $verbose - ) - seq_table <- makeSequenceTable(merged) - """ - else - R""" - merged <- NULL - seq_table <- makeSequenceTable(if (!is.null(dada_fwd)) dada_fwd else dada_rev) - """ - end - - len_dist_pdf = joinpath(ctx.dirs["Figures"], "length_distribution.pdf") - R"plot_length_distribution(seq_table, $len_dist_pdf)" - - ckpt = ctx.ckpts["denoise"] - R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" - emit("Written: $len_dist_pdf") - emit("Checkpoint: $ckpt") - nothing - end - - # Chimera removal - """ - chimera_removal(config_path; progress) - - **Stage 5** — Optionally filter ASVs by length, remove chimeras, compute - pipeline statistics, and write core output files. - - Length filtering (`band_size_min` / `band_size_max`) is applied here from - the saved unfiltered seq_table, so this stage can be re-run with different - length cutoffs without re-running `denoise()`. - - Review `Tables/pipeline_stats.csv` for unexpected read loss at any stage - before committing to the (potentially long) `assign_taxonomy()` step. - - Requires: `Checkpoints/ckpt_filter.RData`, `Checkpoints/ckpt_denoise.RData` - Saves: `Checkpoints/ckpt_chimera.RData` - """ - function chimera_removal(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - verbose = ctx.verbose - mode = ctx.mode - - isfile(ctx.ckpts["filter"]) || - error("Filter checkpoint not found. Run filter_trim() first.") - isfile(ctx.ckpts["denoise"]) || - error("Denoise checkpoint not found. Run denoise() first.") - filter_ckpt = ctx.ckpts["filter"] - denoise_ckpt = ctx.ckpts["denoise"] - R"load($filter_ckpt)" - R"load($denoise_ckpt)" - - # Length filtering applied from the saved unfiltered seq_table. - # Re-running this stage with new band sizes does not require re-denoising. - band_min = get(ctx.cfg["asv"], "band_size_min", nothing) - band_max = get(ctx.cfg["asv"], "band_size_max", nothing) - if !isnothing(band_min) && !isnothing(band_max) - emit("Filtering by length: $band_min-$band_max bp") - R"seq_table <- filter_by_length(seq_table, $band_min, $band_max)" - len_filt_pdf = joinpath(ctx.dirs["Figures"], "length_distribution_filtered.pdf") - R"plot_length_distribution(seq_table, $len_filt_pdf)" - emit("Written: $len_filt_pdf") - end - - emit("Removing chimeras") - denovo_method = ctx.cfg["asv"]["denovo_method"] - R""" - seq_table_nochim <- removeBimeraDenovo(seq_table, method=$denovo_method, verbose=$verbose) - nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 - message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), - "% | Retained: ", round(nochim_pct, 2), "%") - """ - - # Drop duplicate sample row (single-sample fallback) - final_names = ctx.sample_names - if ctx.single_sample - R"filter_stats <- filter_stats[1, , drop=FALSE]" - mode != "reverse" && R"dada_fwd <- dada_fwd[1]" - mode != "forward" && R"dada_rev <- dada_rev[1]" - mode == "paired" && R"merged <- merged[1]" - R"seq_table_nochim <- seq_table_nochim[1, , drop=FALSE]" - final_names = [ctx.sample_names[1]] - end - - emit("Computing pipeline stats") - stats_csv = joinpath(ctx.dirs["Tables"], "pipeline_stats.csv") - R""" - stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, - seq_table_nochim, $final_names, $mode) - write.csv(stats, $stats_csv, quote=FALSE) - if ($verbose) print(stats) - """ - - emit("Writing core output tables") - seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") - fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") - tables_dir = ctx.dirs["Tables"] - R"write_seq_table(seq_table_nochim, $tables_dir, $seq_prefix)" - R"index <- write_fasta(seq_table_nochim, $tables_dir, $fasta_prefix)" - - ckpt = ctx.ckpts["chimera"] - R"save(seq_table_nochim, index, file=$ckpt)" - emit("Written: $stats_csv") - emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") - emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") - emit("Checkpoint: $ckpt") - nothing - end - - # Taxonomy - """ - assign_taxonomy(config_path; progress) - - **Stage 6** — Assign taxonomy to ASVs and write the combined output table. - - This is typically the longest step. Set `taxonomy.skip = true` in config to - skip assignment and output sequence/count data only. - - Requires: `Checkpoints/ckpt_chimera.RData` - Saves: `Checkpoints/checkpoint.RData` (full R environment snapshot) - """ - function assign_taxonomy(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) - emit = _emitter(progress) - ctx = _pipeline_context(config_path; input_dir, workspace_root) - verbose = ctx.verbose - - isfile(ctx.ckpts["chimera"]) || - error("Chimera checkpoint not found. Run chimera_removal() first.") - - # Drop all data objects accumulated from prior stages before the - # memory-intensive taxonomy subprocess runs. Named globals in R's - # environment are reachable and gc() won't collect them; rm() them - # explicitly so they don't inflate the subprocess's memory footprint. - # lsf.str() returns function names; setdiff keeps those intact. - R""" - .data_objs <- setdiff(ls(), lsf.str()) - if (length(.data_objs) > 0L) rm(list = .data_objs) - rm(.data_objs) - gc() - """ - - chimera_ckpt = ctx.ckpts["chimera"] - R"load($chimera_ckpt)" - - seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") - fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") - taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") - combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.xlsx") - boot_mode = get(ctx.cfg["output"], "bootstraps", "combined") - comb_mode = get(ctx.cfg["output"], "combined_mode", "regular") - tables_dir = ctx.dirs["Tables"] - - R"combined_input <- index" - - if !get(ctx.cfg["taxonomy"], "skip", false) - emit("Assigning taxonomy") - multithread = get(ctx.cfg["taxonomy"], "multithread", 4) - min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) - tax_levels = ctx.cfg["taxonomy"]["levels"] - - db_path = isnothing(taxonomy_db) ? - _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db - - R""" - taxa_result <- run_assign_taxonomy( - seq_table_nochim, $db_path, - list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), - $verbose) - taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, - $tables_dir, $taxa_prefix, $boot_mode) - """ - R"gc()" - comb_mode == "regular" && R"combined_input <- taxa_df" - else - emit("Skipping taxonomy (taxonomy.skip = true)") - end - - checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") - R"save(seq_table_nochim, index, combined_input, file=$checkpoint)" - emit("Checkpoint: $checkpoint") - - R"write_combined_table(combined_input, seq_table_nochim, $tables_dir, $combined_file)" - - emit("Pipeline complete. Outputs:") - emit(" $(joinpath(tables_dir, seq_prefix * ".csv"))") - emit(" $(joinpath(tables_dir, fasta_prefix * ".fasta"))") - emit(" $(joinpath(tables_dir, fasta_prefix * ".csv"))") - emit(" $(joinpath(tables_dir, taxa_prefix * ".csv"))") - emit(" $(joinpath(tables_dir, combined_file))") - emit(" $(joinpath(tables_dir, "pipeline_stats.csv"))") - emit(" $checkpoint") - nothing - end + include("dada2/context.jl") # shared helpers and _pipeline_context + include("dada2/qc.jl") # prefilter_qc, filter_trim + include("dada2/denoise.jl") # learn_errors, denoise, filter_length + include("dada2/chimera.jl") # chimera_removal + include("dada2/taxonomy.jl") # assign_taxonomy - # Run pipeline """ dada2(config_path; progress) Run the complete DADA2 pipeline from raw reads to a taxonomy-annotated ASV - table, calling all six stages in sequence: + table, calling all stages in sequence: - prefilter_qc → filter_trim → learn_errors → - denoise → chimera_removal → assign_taxonomy + prefilter_qc -> filter_trim -> learn_errors -> + denoise -> filter_length -> chimera_removal -> assign_taxonomy - For interactive use — reviewing intermediate outputs or adjusting parameters - between steps — call the individual stage functions directly instead. + For interactive use - reviewing intermediate outputs or adjusting parameters + between steps - call the individual stage functions directly instead. ## Outputs Written to `workspace.root/Tables/`: - - `seqtab_nochim.csv` — chimera-free ASV count table - - `asvs.fasta` / `asvs.csv` — ASV sequences with short identifiers - - `taxonomy.csv` — taxonomy assignments (with optional bootstraps) - - `tax_counts.xlsx` — combined taxonomy + per-sample counts - - `pipeline_stats.csv` — read counts at each pipeline stage + - `seqtab_nochim.csv` - chimera-free ASV count table + - `asvs.fasta` / `asvs.csv` - ASV sequences with short identifiers + - `taxonomy.csv` - taxonomy assignments (with optional bootstraps) + - `tax_counts.xlsx` - combined taxonomy + per-sample counts + - `pipeline_stats.csv` - read counts at each pipeline stage Written to `workspace.root/Checkpoints/`: - - `ckpt_filter.RData` — filter_stats - - `ckpt_errors.RData` — fwd_errors, rev_errors - - `ckpt_denoise.RData` — dada objects, merged, unfiltered seq_table - - `ckpt_chimera.RData` — seq_table_nochim, index - - `checkpoint.RData` — full R environment snapshot + - `ckpt_filter.RData` - filter_stats + - `ckpt_errors.RData` - fwd_errors, rev_errors + - `ckpt_denoise.RData` - dada objects, merged, unfiltered seq_table + - `ckpt_length.RData` - dada objects, merged, length-filtered seq_table + - `ckpt_chimera.RData` - seq_table_nochim, index + - `checkpoint.RData` - final R environment snapshot """ function dada2(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) - prefilter_qc(config_path; progress, input_dir, workspace_root) - filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" - learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" - denoise(config_path; progress, input_dir, workspace_root); R"gc()" - chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" + prefilter_qc(config_path; progress, input_dir, workspace_root) + filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" + learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" + denoise(config_path; progress, input_dir, workspace_root); R"gc()" + filter_length(config_path; progress, input_dir, workspace_root); R"gc()" + chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end -end \ No newline at end of file +end diff --git a/src/dada2/chimera.jl b/src/dada2/chimera.jl new file mode 100644 index 0000000..db1b6e7 --- /dev/null +++ b/src/dada2/chimera.jl @@ -0,0 +1,76 @@ +# Web UI: Denoising page +# Stages: chimera_removal + + # Chimera removal + """ + chimera_removal(config_path; progress) + + **Stage 6** - Remove chimeric sequences, drop the single-sample duplicate + row if present, and write core output files (seq table, FASTA, pipeline + stats). + + Review `Tables/pipeline_stats.csv` for unexpected read loss at any stage + before committing to the (potentially long) `assign_taxonomy()` step. + + Requires: `Checkpoints/ckpt_filter.RData`, `Checkpoints/ckpt_length.RData` + Saves: `Checkpoints/ckpt_chimera.RData` + """ + function chimera_removal(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + verbose = ctx.verbose + mode = ctx.mode + + isfile(ctx.ckpts["filter"]) || + error("Filter checkpoint not found. Run filter_trim() first.") + isfile(ctx.ckpts["length"]) || + error("Length filter checkpoint not found. Run filter_length() first.") + filter_ckpt = ctx.ckpts["filter"] + length_ckpt = ctx.ckpts["length"] + R"load($filter_ckpt)" + R"load($length_ckpt)" + + emit("Removing chimeras") + denovo_method = ctx.cfg["asv"]["denovo_method"] + R""" + seq_table_nochim <- removeBimeraDenovo(seq_table, method=$denovo_method, verbose=$verbose) + nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 + message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), + "% | Retained: ", round(nochim_pct, 2), "%") + """ + + # Drop duplicate sample row (single-sample fallback) + final_names = ctx.sample_names + if ctx.single_sample + R"filter_stats <- filter_stats[1, , drop=FALSE]" + mode != "reverse" && R"dada_fwd <- dada_fwd[1]" + mode != "forward" && R"dada_rev <- dada_rev[1]" + mode == "paired" && R"merged <- merged[1]" + R"seq_table_nochim <- seq_table_nochim[1, , drop=FALSE]" + final_names = [ctx.sample_names[1]] + end + + emit("Computing pipeline stats") + stats_csv = joinpath(ctx.dirs["Tables"], "pipeline_stats.csv") + R""" + stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, + seq_table_nochim, $final_names, $mode) + write.csv(stats, $stats_csv, quote=FALSE) + if ($verbose) print(stats) + """ + + emit("Writing core output tables") + seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") + fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") + tables_dir = ctx.dirs["Tables"] + R"write_seq_table(seq_table_nochim, $tables_dir, $seq_prefix)" + R"index <- write_fasta(seq_table_nochim, $tables_dir, $fasta_prefix)" + + ckpt = ctx.ckpts["chimera"] + R"save(seq_table_nochim, index, file=$ckpt)" + emit("Written: $stats_csv") + emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") + emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") + emit("Checkpoint: $ckpt") + nothing + end diff --git a/src/dada2/context.jl b/src/dada2/context.jl new file mode 100644 index 0000000..f37b1b7 --- /dev/null +++ b/src/dada2/context.jl @@ -0,0 +1,235 @@ +# Shared pipeline context - helpers and setup called at the start of every stage. +# Included into module DADA2 by src/dada2.jl. + + # Emitter + + # Returns a function that routes progress messages to @info (CLI) or a + # Channel{String} (Genie SSE). Pass the result as `emit` in stage functions. + _emitter(::Nothing) = msg -> @info msg + _emitter(ch::Channel{String}) = msg -> put!(ch, msg) + + # Database helpers + + # Download uri to db_dir/basename(uri) if not already cached. + # Respects the optional local: override in fmt_info. + function _download_db_if_needed(key, fmt_info, db_dir, emit) + local_p = get(fmt_info, "local", nothing) + if !isnothing(local_p) + local_p = string(local_p) + if !isempty(local_p) + isfile(local_p) && (emit("[$key] Using local file: $local_p"); return local_p) + @warn "[$key] Configured local path not found: $local_p - falling back to uri" + end + end + uri = fmt_info["uri"] + cached = joinpath(db_dir, basename(uri)) + if isfile(cached) + emit("[$key] Using cached: $cached") + else + emit("[$key] Downloading: $uri") + Downloads.download(uri, cached) + emit("[$key] Saved to: $cached") + end + return cached + end + + # Resolve the DADA2 taxonomy database from config when no external path is + # provided. Supports the databases:/taxonomy.database: scheme (preferred) + # and the legacy taxonomy.uri: key for backwards compatibility. + function _resolve_taxonomy_db(cfg, emit) + tax_cfg = cfg["taxonomy"] + if haskey(tax_cfg, "database") + db_key = string(tax_cfg["database"]) + db_cfg = get(cfg, "databases", Dict()) + haskey(db_cfg, db_key) || + error("taxonomy.database = \"$db_key\" not found in databases: section") + fmt_cfg = get(db_cfg[db_key], "dada2", nothing) + isnothing(fmt_cfg) && + error("databases.$db_key.dada2 is not configured") + db_dir = abspath(get(db_cfg, "dir", "./databases")) + mkpath(db_dir) + return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) + end + # Legacy: taxonomy.uri + tax_uri = tax_cfg["uri"] + if isfile(tax_uri) + emit("Using local taxonomy database: $tax_uri") + return tax_uri + end + db_dir = abspath(get(cfg, "databases_dir", "./databases")) + mkpath(db_dir) + cached = joinpath(db_dir, basename(tax_uri)) + if !isfile(cached) + emit("Downloading taxonomy database: $tax_uri") + Downloads.download(tax_uri, cached) + emit("Written: $cached") + else + emit("Using cached taxonomy database: $cached") + end + return cached + end + + # Config + function validate_config(cfg) + required = ["workspace", "file_patterns", "filter_trim", "dada", + "merge", "asv", "taxonomy", "output"] + missing_secs = filter(k -> !haskey(cfg, k), required) + isempty(missing_secs) || + error("Missing required config sections: $(join(missing_secs, ", "))") + + ws = cfg["workspace"] + haskey(ws, "root") || error("workspace.root is required") + haskey(ws, "input_dir") || error("workspace.input_dir is required") + isdir(ws["input_dir"]) || + error("workspace.input_dir does not exist: $(ws["input_dir"])") + + mode = get(cfg["file_patterns"], "mode", "paired") + mode in ("paired", "forward", "reverse") || + error("file_patterns.mode must be one of: paired, forward, reverse") + + boot_mode = get(cfg["output"], "bootstraps", "combined") + boot_mode in ("none", "combined", "separate") || + error("output.bootstraps must be one of: none, combined, separate") + + if !get(cfg["taxonomy"], "skip", false) + has_uri = haskey(cfg["taxonomy"], "uri") + has_db = haskey(cfg["taxonomy"], "database") + has_uri || has_db || + error("taxonomy: configure `database` (referencing a databases: entry) " * + "or `uri` when skip is not true") + end + + combined_mode = get(cfg["output"], "combined_mode", "regular") + combined_mode in ("regular", "alternative") || + error("output.combined_mode must be one of: regular, alternative") + end + + # File discovery + function find_fastq_files(input_dir, fwd_pattern, rev_pattern, mode) + all_files = sort(readdir(input_dir, join=true)) + fwd = mode in ("paired", "forward") ? + filter(f -> !isnothing(match(Regex(fwd_pattern), basename(f))), all_files) : String[] + rev = mode in ("paired", "reverse") ? + filter(f -> !isnothing(match(Regex(rev_pattern), basename(f))), all_files) : String[] + fwd, rev + end + + function validate_sample_files(fwd, rev, mode) + if mode in ("paired", "forward") + isempty(fwd) && + error("No forward FASTQ files found. " * + "Check workspace.input_dir and file_patterns.forward.") + for f in fwd + isfile(f) || error("Forward file not found: $f") + end + end + if mode in ("paired", "reverse") + isempty(rev) && + error("No reverse FASTQ files found. " * + "Check workspace.input_dir and file_patterns.reverse.") + for f in rev + isfile(f) || error("Reverse file not found: $f") + end + end + if mode == "paired" && length(fwd) != length(rev) + error("Forward file count ($(length(fwd))) does not match " * + "reverse file count ($(length(rev))).") + end + end + + function extract_sample_names(files, split_char, split_index) + [split(basename(f), split_char)[split_index] for f in files] + end + + # Workspace + function setup_workspace(root) + dirs = Dict( + "Tables" => joinpath(root, "Tables"), + "Checkpoints" => joinpath(root, "Checkpoints"), + "Figures" => joinpath(root, "Figures"), + "Filtered" => joinpath(root, "Filtered"), + ) + for d in values(dirs) + mkpath(d) + end + dirs + end + + # Pipeline context + # Shared setup called at the start of every stage: sources R functions, loads + # and validates config, discovers files, handles the single-sample fallback, and + # computes all path variables. Returns a NamedTuple so stage functions can + # extract what they need without repeating boilerplate. + # + # Optional overrides (used when main.jl manages directory layout): + # input_dir - overrides cfg["workspace"]["input_dir"] + # workspace_root - overrides cfg["workspace"]["root"] + function _pipeline_context(config_path::String; input_dir=nothing, workspace_root=nothing) + functions_r = joinpath(@__DIR__, "dada2_functions.r") + R"source($functions_r)" + + cfg = YAML.load_file(config_path) + + if !isnothing(input_dir) + cfg["workspace"]["input_dir"] = input_dir + end + if !isnothing(workspace_root) + cfg["workspace"]["root"] = workspace_root + end + + validate_config(cfg) + verbose = get(cfg, "verbose", true) + mode = get(cfg["file_patterns"], "mode", "paired") + root = cfg["workspace"]["root"] + dirs = setup_workspace(root) + + fwd_files, rev_files = find_fastq_files( + cfg["workspace"]["input_dir"], + cfg["file_patterns"]["forward"], + cfg["file_patterns"]["reverse"], + mode) + validate_sample_files(fwd_files, rev_files, mode) + + primary_files = isempty(fwd_files) ? rev_files : fwd_files + sample_names = extract_sample_names( + primary_files, + cfg["file_patterns"]["sample_name_split"], + cfg["file_patterns"]["sample_name_index"]) + + # Single-sample fallback: dada() returns a bare object (not a list) for a + # single input file, breaking makeSequenceTable() and sapply() downstream. + # Duplicate the paths so the pipeline sees 2 samples; the extra row is + # dropped in chimera_removal(). ASV calls are unaffected. + single_sample = length(sample_names) == 1 + if single_sample + @warn "Only 1 sample found. Duplicating it to work around dada() " * + "returning a bare object for single-file input. The duplicate " * + "will be dropped from all outputs." + fwd_files = isempty(fwd_files) ? fwd_files : repeat(fwd_files, 2) + rev_files = isempty(rev_files) ? rev_files : repeat(rev_files, 2) + sample_names = [sample_names[1], sample_names[1] * "_dup"] + end + + filtered_dir = dirs["Filtered"] + fwd_out = mode != "reverse" ? + [joinpath(filtered_dir, s * "_R1_filt.fastq.gz") for s in sample_names] : String[] + rev_out = mode != "forward" ? + [joinpath(filtered_dir, s * "_R2_filt.fastq.gz") for s in sample_names] : String[] + + in_fwd = mode != "reverse" ? fwd_files : rev_files + out_fwd = mode != "reverse" ? fwd_out : rev_out + in_rev_arg = mode == "paired" ? rev_files : nothing + out_rev_arg = mode == "paired" ? rev_out : nothing + + ckpts = Dict( + "filter" => joinpath(dirs["Checkpoints"], "ckpt_filter.RData"), + "errors" => joinpath(dirs["Checkpoints"], "ckpt_errors.RData"), + "denoise" => joinpath(dirs["Checkpoints"], "ckpt_denoise.RData"), + "length" => joinpath(dirs["Checkpoints"], "ckpt_length.RData"), + "chimera" => joinpath(dirs["Checkpoints"], "ckpt_chimera.RData"), + ) + + (; cfg, verbose, mode, dirs, sample_names, single_sample, + fwd_files, rev_files, fwd_out, rev_out, + in_fwd, out_fwd, in_rev_arg, out_rev_arg, ckpts) + end diff --git a/src/dada2_functions.r b/src/dada2/dada2_functions.r similarity index 99% rename from src/dada2_functions.r rename to src/dada2/dada2_functions.r index f27a31b..c74795f 100644 --- a/src/dada2_functions.r +++ b/src/dada2/dada2_functions.r @@ -156,7 +156,7 @@ fetch_taxonomy_db <- function(uri, local_dir) { } # Assigns taxonomy to ASVs using a naive Bayesian classifier. -# outputBootstraps is always TRUE so bootstrap confidence values (0–100 per +# outputBootstraps is always TRUE so bootstrap confidence values (0-100 per # rank) are available for downstream filtering regardless of minBoot. # minBoot sets the threshold at which assignments are returned; 0 returns all # assignments. @@ -202,7 +202,7 @@ write_fasta <- function(seq_table, tables_dir, prefix) { index } -# Writes taxonomy assignments to CSV. Bootstrap confidence values (0–100 per +# Writes taxonomy assignments to CSV. Bootstrap confidence values (0-100 per # taxonomic rank) are handled based on bootstrap_mode: # none - taxonomy columns only # combined - taxonomy and bootstrap columns in one file (suffix: _boot) diff --git a/src/dada2/denoise.jl b/src/dada2/denoise.jl new file mode 100644 index 0000000..5ce9063 --- /dev/null +++ b/src/dada2/denoise.jl @@ -0,0 +1,156 @@ +# Web UI: Denoising page +# Stages: learn_errors, denoise, filter_length + + # Error model + """ + learn_errors(config_path; progress) + + **Stage 3** - Learn substitution error rates and plot diagnostics. + + Review `Figures/error_rates.pdf`: the fitted line should closely follow the + observed points. If not, increase `nbases` or `max_consist` in config and + re-run this stage. + + Saves: `Checkpoints/ckpt_errors.RData` + """ + function learn_errors(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + seed = get(ctx.cfg["dada"], "seed", 123) + nbases = ctx.cfg["dada"]["nbases"] + max_con = ctx.cfg["dada"]["max_consist"] + verbose = ctx.verbose + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + emit("Learning error rates") + R"set.seed($seed)" + + ctx.mode != "reverse" ? + R"fwd_errors <- learnErrors($fwd_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"fwd_errors <- NULL" + + ctx.mode != "forward" ? + R"rev_errors <- learnErrors($rev_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"rev_errors <- NULL" + + error_pdf = joinpath(ctx.dirs["Figures"], "error_rates.pdf") + R"plot_error_rates(fwd_errors, rev_errors, $error_pdf)" + + ckpt = ctx.ckpts["errors"] + R"save(fwd_errors, rev_errors, file=$ckpt)" + emit("Written: $error_pdf") + emit("Checkpoint: $ckpt") + nothing + end + + # Denoise + """ + denoise(config_path; progress) + + **Stage 4** - Denoise reads, merge pairs (paired mode), build the sequence + table, and plot the raw ASV length distribution. + + Review `Figures/length_distribution.pdf` and set `band_size_min` / + `band_size_max` in config to target the expected amplicon peak before + running `filter_length()`. + + Requires: `Checkpoints/ckpt_errors.RData` + Saves: `Checkpoints/ckpt_denoise.RData` (unfiltered seq_table) + """ + function denoise(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + verbose = ctx.verbose + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + isfile(ctx.ckpts["errors"]) || + error("Error model checkpoint not found. Run learn_errors() first.") + errors_ckpt = ctx.ckpts["errors"] + R"load($errors_ckpt)" + + emit("Denoising reads") + pool_method = ctx.cfg["dada"]["pool_method"] + + ctx.mode != "reverse" ? + R"dada_fwd <- dada($fwd_out, err=fwd_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_fwd <- NULL" + + ctx.mode != "forward" ? + R"dada_rev <- dada($rev_out, err=rev_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_rev <- NULL" + + emit("Building sequence table") + if ctx.mode == "paired" + min_overlap = ctx.cfg["merge"]["min_overlap"] + max_mismatch = ctx.cfg["merge"]["max_mismatch"] + trim_overhang = ctx.cfg["merge"]["trim_overhang"] + R""" + merged <- mergePairs( + dada_fwd, $fwd_out, + dada_rev, $rev_out, + minOverlap = $min_overlap, + maxMismatch = $max_mismatch, + trimOverhang = $trim_overhang, + verbose = $verbose + ) + seq_table <- makeSequenceTable(merged) + """ + else + R""" + merged <- NULL + seq_table <- makeSequenceTable(if (!is.null(dada_fwd)) dada_fwd else dada_rev) + """ + end + + len_dist_pdf = joinpath(ctx.dirs["Figures"], "length_distribution.pdf") + R"plot_length_distribution(seq_table, $len_dist_pdf)" + + ckpt = ctx.ckpts["denoise"] + R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" + emit("Written: $len_dist_pdf") + emit("Checkpoint: $ckpt") + nothing + end + + # Length filter + """ + filter_length(config_path; progress) + + **Stage 5** - Optionally filter the sequence table by amplicon length and + plot the filtered length distribution. + + Set `asv.band_size_min` and `asv.band_size_max` in config to the expected + amplicon length range. If both are null this stage is a passthrough. Re-run + this stage alone to adjust length cutoffs without re-running `denoise()`. + + Requires: `Checkpoints/ckpt_denoise.RData` + Saves: `Checkpoints/ckpt_length.RData` + """ + function filter_length(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + + isfile(ctx.ckpts["denoise"]) || + error("Denoise checkpoint not found. Run denoise() first.") + denoise_ckpt = ctx.ckpts["denoise"] + R"load($denoise_ckpt)" + + band_min = get(ctx.cfg["asv"], "band_size_min", nothing) + band_max = get(ctx.cfg["asv"], "band_size_max", nothing) + if !isnothing(band_min) && !isnothing(band_max) + emit("Filtering by length: $band_min-$band_max bp") + R"seq_table <- filter_by_length(seq_table, $band_min, $band_max)" + len_filt_pdf = joinpath(ctx.dirs["Figures"], "length_distribution_filtered.pdf") + R"plot_length_distribution(seq_table, $len_filt_pdf)" + emit("Written: $len_filt_pdf") + else + emit("No length filter configured (band_size_min/max not set) - passing through") + end + + ckpt = ctx.ckpts["length"] + R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" + emit("Checkpoint: $ckpt") + nothing + end diff --git a/src/dada2/qc.jl b/src/dada2/qc.jl new file mode 100644 index 0000000..18a12dd --- /dev/null +++ b/src/dada2/qc.jl @@ -0,0 +1,94 @@ +# Web UI: QC page +# Stages: prefilter_qc, filter_trim + + # Pre-filter quality assessment + """ + prefilter_qc(config_path; progress) + + **Stage 1** - Plot unfiltered quality profiles. + + Review `Figures/quality_unfiltered.pdf` to choose `truncLen` and `maxEE` + values in config before running `filter_trim()`. + """ + function prefilter_qc(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + + emit("Plotting unfiltered quality profiles") + fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files + rev_for_plot = isempty(ctx.rev_files) ? nothing : ctx.rev_files + unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") + R"plot_quality_profiles($fwd_for_plot, $rev_for_plot, $unfiltered_pdf)" + emit("Written: $unfiltered_pdf") + nothing + end + + # Filter and trim + """ + filter_trim(config_path; progress) + + **Stage 2** - Filter and trim reads; plot filtered quality profiles. + + Review `Figures/quality_filtered.pdf`. If filtering looks appropriate, + proceed to `learn_errors()`; otherwise adjust `truncLen` / `maxEE` in + config and re-run this stage. + + Saves: `Checkpoints/ckpt_filter.RData` + """ + function filter_trim(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + ft = ctx.cfg["filter_trim"] + trunc_len = ft["trunc_len"] + max_ee = ft["max_ee"] + verbose = ctx.verbose + in_fwd = ctx.in_fwd + out_fwd = ctx.out_fwd + in_rev = ctx.in_rev_arg + out_rev = ctx.out_rev_arg + fwd_out = ctx.fwd_out + rev_out = ctx.rev_out + + emit("Filtering and trimming reads") + if ctx.mode == "paired" + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + $in_rev, $out_rev, + truncQ = $(ft["trunc_q"]), + truncLen = $trunc_len, + maxEE = $max_ee, + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + matchIDs = $(ft["match_ids"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + else + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + truncQ = $(ft["trunc_q"]), + truncLen = $(trunc_len[1]), + maxEE = $(max_ee[1]), + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + end + + emit("Plotting filtered quality profiles") + fwd_filt = isempty(fwd_out) ? nothing : fwd_out + rev_filt = isempty(rev_out) ? nothing : rev_out + filtered_pdf = joinpath(ctx.dirs["Figures"], "quality_filtered.pdf") + R"plot_quality_profiles($fwd_filt, $rev_filt, $filtered_pdf)" + + ckpt = ctx.ckpts["filter"] + R"save(filter_stats, file=$ckpt)" + emit("Written: $filtered_pdf") + emit("Checkpoint: $ckpt") + nothing + end diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl new file mode 100644 index 0000000..f2e3580 --- /dev/null +++ b/src/dada2/taxonomy.jl @@ -0,0 +1,87 @@ +# Web UI: Taxonomy page +# Stages: assign_taxonomy + + # Taxonomy assignment + """ + assign_taxonomy(config_path; progress) + + **Stage 7** - Assign taxonomy to ASVs and write the combined output table. + + This is typically the longest step. Set `taxonomy.skip = true` in config to + skip assignment and output sequence/count data only. + + Requires: `Checkpoints/ckpt_chimera.RData` + Saves: `Checkpoints/checkpoint.RData` + """ + function assign_taxonomy(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) + emit = _emitter(progress) + ctx = _pipeline_context(config_path; input_dir, workspace_root) + verbose = ctx.verbose + + isfile(ctx.ckpts["chimera"]) || + error("Chimera checkpoint not found. Run chimera_removal() first.") + + # Drop all data objects accumulated from prior stages before the + # memory-intensive taxonomy assignment runs. Named globals in R's + # environment are reachable and gc() won't collect them; rm() them + # explicitly so they don't inflate the memory footprint. + # lsf.str() returns function names; setdiff keeps those intact. + R""" + .data_objs <- setdiff(ls(), lsf.str()) + if (length(.data_objs) > 0L) rm(list = .data_objs) + rm(.data_objs) + gc() + """ + + chimera_ckpt = ctx.ckpts["chimera"] + R"load($chimera_ckpt)" + + seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") + fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") + taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") + combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.xlsx") + boot_mode = get(ctx.cfg["output"], "bootstraps", "combined") + comb_mode = get(ctx.cfg["output"], "combined_mode", "regular") + tables_dir = ctx.dirs["Tables"] + + R"combined_input <- index" + + if !get(ctx.cfg["taxonomy"], "skip", false) + emit("Assigning taxonomy") + multithread = get(ctx.cfg["taxonomy"], "multithread", 4) + min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) + tax_levels = ctx.cfg["taxonomy"]["levels"] + + db_path = isnothing(taxonomy_db) ? + _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db + + R""" + taxa_result <- run_assign_taxonomy( + seq_table_nochim, $db_path, + list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), + $verbose) + taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, + $tables_dir, $taxa_prefix, $boot_mode) + """ + R"gc()" + comb_mode == "regular" && R"combined_input <- taxa_df" + else + emit("Skipping taxonomy (taxonomy.skip = true)") + end + + checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") + R"save(seq_table_nochim, index, combined_input, file=$checkpoint)" + emit("Checkpoint: $checkpoint") + + R"write_combined_table(combined_input, seq_table_nochim, $tables_dir, $combined_file)" + + emit("Pipeline complete. Outputs:") + emit(" $(joinpath(tables_dir, seq_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, fasta_prefix * ".fasta"))") + emit(" $(joinpath(tables_dir, fasta_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, taxa_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, combined_file))") + emit(" $(joinpath(tables_dir, "pipeline_stats.csv"))") + emit(" $checkpoint") + nothing + end diff --git a/src/databases.jl b/src/databases.jl index a2443ac..9ccf9db 100644 --- a/src/databases.jl +++ b/src/databases.jl @@ -5,8 +5,8 @@ module Databases # # Usage: # dbs = ensure_databases("config/dada2.yml") -# dbs["pr2_dada2"] # → resolved local path for DADA2 assignTaxonomy -# dbs["pr2_vsearch"] # → resolved local path for VSEARCH --db +# dbs["pr2_dada2"] # -> resolved local path for DADA2 assignTaxonomy +# dbs["pr2_vsearch"] # -> resolved local path for VSEARCH --db # # Keys follow the pattern "_", e.g. "pr2_dada2", # "pr2_vsearch". Format names map to the sub-keys under each database entry @@ -67,7 +67,7 @@ function _resolve_entry(key, fmt_info, db_dir) @info "[$key] Using local file: $local_p" return local_p end - @warn "[$key] Configured local path not found: $local_p — falling back to uri" + @warn "[$key] Configured local path not found: $local_p - falling back to uri" end end diff --git a/src/main.jl b/src/main.jl index f497a69..1f5d7a7 100755 --- a/src/main.jl +++ b/src/main.jl @@ -1,40 +1,13 @@ #!/usr/bin/env julia include("databases.jl") -include("run_cutadapt.jl") +include("call_tools.jl") include("dada2.jl") include("merge_and_filter_taxa.jl") -include("run_vsearch.jl") using CSV using YAML -using .Databases, .Cutadapt, .TaxonomyTableTools, .DADA2, .VSEARCH - -## Tools loading (to move to new module at some point) -""" - load_tools(config_path) -> Dict{String,String} - -Read config/tools.yml and return a Dict of tool name => resolved path. -If the file does not exist, returns an empty Dict so tools fall back to PATH. -Paths with `@` are SSH remote paths (user@host:/path), the calling module is -responsible for routing those calls via SSH. -""" -function load_tools(config_path = joinpath(@__DIR__, "..", "config", "tools.yml")) - isfile(config_path) || return Dict{String,String}() - data = YAML.load_file(config_path) - tools = Dict{String,String}() - for (name, info) in data - path = info isa Dict ? get(info, "path", nothing) : nothing - path !== nothing && (tools[name] = string(path)) - end - tools -end - -# Return the configured binary path for `tool_key`, or `default` if not set. -tool_bin(tools, tool_key, default = tool_key) = get(tools, tool_key, default) - -# Load resolved tool paths from config/tools.yml (empty Dict if not present) -tools = load_tools() +using .Databases, .Tools, .TaxonomyTableTools, .DADA2 ## Instantiate filesystem # Root directories @@ -44,11 +17,14 @@ output_dir = "./output" # Project name - all stage outputs live under output/{project_name}/ # Re-running with the same project_name overwrites previous results. -project_name = "project" +project_name = "Multiplex_pool" project_dir = joinpath(output_dir, project_name) +fastq_input_dir = joinpath(data_dir, project_name) + +# QC paths +fastqc_dir = joinpath(project_dir, "FastQC") # Cutadapt paths -fastq_input_dir = joinpath(data_dir, "fastq") trimmed_dir = joinpath(project_dir, "cutadapt") primers_config = joinpath(config_dir, "primers.yml") @@ -85,11 +61,13 @@ protist_filter = joinpath(config_dir, "protist_filter.yml") dbs = ensure_databases(dada2_config_path) ## Main -#cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args, cutadapt_bin = tool_bin(tools, "cutadapt")) +fastqc_all(fastq_input_dir, fastqc_dir) + +cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) #dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) -vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args, vsearch_bin = tool_bin(tools, "vsearch")) +vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) -CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)) \ No newline at end of file +CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)) diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index e42374e..eaba301 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -1,5 +1,9 @@ module TaxonomyTableTools +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + using CSV, DataFrames, Logging, YAML export merge_taxonomy_counts, filter_table @@ -50,7 +54,7 @@ export merge_taxonomy_counts, filter_table end else # Taxonomy-only format: Domain;Supergroup;...;Species[;] - # No accession/rRNA/Organellum/specimen — those stay as empty strings + # No accession/rRNA/Organellum/specimen - those stay as empty strings parts = filter(!isempty, split(tax_str, ';')) for j in 1:min(length(parts), length(header) - 6) data[j + 6] = parts[j] @@ -148,7 +152,7 @@ export merge_taxonomy_counts, filter_table end # Rename any columns in the DADA2 file that clash with vsearch taxonomy - # columns (e.g. Domain, Supergroup, … when counts_csv_path is taxonomy.csv) + # columns (e.g. Domain, Supergroup, ... when counts_csv_path is taxonomy.csv) overlap = filter(!=(Symbol("SeqName")), intersect(Symbol.(names(df_taxonomy)), Symbol.(names(df_counts_prepared)))) diff --git a/src/run_dada2.jl b/src/run_dada2.jl deleted file mode 100644 index 56352b3..0000000 --- a/src/run_dada2.jl +++ /dev/null @@ -1,21 +0,0 @@ -module OldDADA2 - -# This is now obsolete, I guess...how sad, it was quite cute. - -export dada2 - - using RCall - - """ - dada2(dada2_config_dir) - - Most adorable little wrapper that simply passes the dada2 config to R to run through its pipeline. Requires dada2.r in src. - - ## Arguments: - - `dada2_config_dir`: Path to DADA2 YAML config. - """ - function dada2(dada2_config_dir) - R"system(paste('Rscript', './src/dada2.r', $dada2_config_dir))" - end - -end \ No newline at end of file diff --git a/src/run_vsearch.jl b/src/run_vsearch.jl deleted file mode 100644 index 84399a6..0000000 --- a/src/run_vsearch.jl +++ /dev/null @@ -1,30 +0,0 @@ -module VSEARCH - -export vsearch - - using Logging - - """ - vsearch(fasta_in_dir, reference_database; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") - - Requires `vsearch` installed. Runs `vsearch` command with any optional parameters to perform taxonomy assignment by local alignment against specified database. - - ## Arguments - - `fasta_in_dir`: Specify path of fasta files output by DADA2 pipeline. - - `reference_database` (default: "./databases"): Specify path of reference database. - - ## Keyword Arguments - - `optional_args` (optional, default: "--id 0.75 --query_cov 0.8"): Specify additional arguments passed to `vsearch` command. - - `vsearch_bin` (optional, default: "vsearch"): Path to the vsearch binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. - - """ - function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") - mkpath(vsearch_dir) - outfile = joinpath(vsearch_dir, "taxonomy.tsv") - - cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" - run(`bash -lc $cmd`) - - end - -end \ No newline at end of file From fbba17e42ff85913b1de2db9d7bc799f62493b3c Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 20:45:52 +0100 Subject: [PATCH 021/175] Yep...I knew something had to go wrong... --- .gitignore | 2 +- data/MiSeq_SOP/HMP_MOCK.v35.fasta | 64 ------------------------------- data/MiSeq_SOP/mouse.dpw.metadata | 20 ---------- data/MiSeq_SOP/mouse.time.design | 20 ---------- data/MiSeq_SOP/stability.batch | 26 ------------- data/MiSeq_SOP/stability.files | 20 ---------- 6 files changed, 1 insertion(+), 151 deletions(-) delete mode 100644 data/MiSeq_SOP/HMP_MOCK.v35.fasta delete mode 100644 data/MiSeq_SOP/mouse.dpw.metadata delete mode 100644 data/MiSeq_SOP/mouse.time.design delete mode 100644 data/MiSeq_SOP/stability.batch delete mode 100644 data/MiSeq_SOP/stability.files diff --git a/.gitignore b/.gitignore index e8b929e..42759f2 100644 --- a/.gitignore +++ b/.gitignore @@ -253,7 +253,7 @@ rsconnect/ ### Other # Things with ambiguous intellectual property internal -data/fastq/ +data/ cutadapt/output_* ### Lock files diff --git a/data/MiSeq_SOP/HMP_MOCK.v35.fasta b/data/MiSeq_SOP/HMP_MOCK.v35.fasta deleted file mode 100644 index c1748ed..0000000 --- a/data/MiSeq_SOP/HMP_MOCK.v35.fasta +++ /dev/null @@ -1,64 +0,0 @@ ->A.baumannii.1 -TGGGGAATATTGGACAATGGGGGGAACCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGCCTTATGGTTGTAAAGCACTTTAAGCGAGGAGGAGGCTACTTTAGTTAATACCTAGAGATAGTGGACGTTACTCGCAGAATAAGCACCGGCTAACTCTGTGCCAGCAGCCGCGGTAATACAGAGGGTGCGAGCGTTAATCGGATTTACTGGGCGTAAAGCGTGCGTAGGCGGCTTATTAAGTCGGATGTGAAATCCCCGAGCTTAACTTGGGAATTGCATTCGATACTGGTGAGCTAGAGTATGGGAGAGGATGGTAGAATTCCAGGTGTAGCGGTGAAATGCGTAGAGATCTGGAGGAATACCGATGGCGAAGGCAGCCATCTGGCCTAATACTGACGCTGAGGTACGAAAGCATGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCATGCCGTAAACGATGTCTACTAGCCGTTGGGGCCTTTGAGGCTTTAGTGGCGCAGCTAACGCGATAAGTAGACCGCCTGGGGAGTACGGTC ->A.odontolyticus.1 -TGGGGAATATTGCACAATGGGCGAAAGCCTGATGCAGCGACGCCGCGTGAGGGATGGAGGCCTTCGGGTTGTAAACCTCTTTCGCTCATGGTCAAGCCGCAACTCAAGGTTGTGGTGAGGGTAGTGGGTAAAGAAGCGCCGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGGCGCGAGCGTTGTCCGGAATTATTGGGCGTAAAGGGCTTGTAGGCGGTTGGTCGCGTCTGCCGTGAAATCCTCTGGCTTAACTGGGGGCGTGCGGTGGGTACGGGCTGACTTGAGTGCGGTAGGGGAGACTGGAACTCCTGGTGTAGCGGTGGAATGCGCAGATATCAGGAAGAACACCGGTGGCGAAGGCGGGTCTCTGGGCCGTTACTGACGCTGAGGAGCGAAAGCGTGGGGAGCGAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGTTGGGCACTAGGTGTGGGGGCCACCCGTGGTTTCTGCGCCGTAGCTAACGCTTTAAGTGCCCCGCCTGGGGAGTACGGCC ->B.cereus.1 -TAGGGAATCTTCCGCAATGGACGAAAGTCTGACGGAGCAACGCCGCGTGAGTGATGAAGGCTTTCGGGTCGTAAAACTCTGTTGTTAGGGAAGAACAAGTGCTAGTTGAATAAGCTGGCACCTTGACGGTACCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGCAGGTGGTTTCTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACACTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGAGGGTTTCCGCCCTTTAGTGCTGAAGTTAACGCATTAAGCACTCCGCCTGGGGAGTACGGCC ->B.vulgatus.1 -TGAGGAATATTGGTCAATGGGCGCAGGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG ->B.vulgatus.2 -TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG ->B.vulgatus.4 -TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATATTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACTGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG ->B.vulgatus.5 -TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGGATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAAGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACTGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG ->B.vulgatus.7 -TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTAGCGTGAAGGATGACTGCCCTATGGGTTGTAAACTTCTTTTATAAAGGAATAAAGTCGGGTATGCATACCCGTTTGCATGTACTTTATGAATAAGGATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCTTAGATATCACGAGGAACTCCGATTGCGAAGGCAGCCTGCTAAGCTGCAACTGACATTGAGGCTCGAAAGTGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACACGGTAAACGATGAATACTCGCTGTTTGCGATATACGGCAAGCGGCCAAGCGAAAGCGTTAAGTATTCCACCTGGGGAGTACGCCG ->C.beijerinckii.1 -TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCCTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGCCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC ->C.beijerinckii.3 -TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCCTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGTCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC ->C.beijerinckii.4 -TGGGGAATATTGCACAATGGGGGAAACCCTGATGCAGCAACGCCGCGTGAGTGATGACGGTCTTCGGATTGTAAAGCTCTGTCTTCAGGGACGATAATGACGGTACCTGAGGAGGAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTACTGGGCGTAAAGGGAGCGTAGGTGGATATTTAAGTGGGATGTGAAATACTCGGGCTTAACCTGGGTGCTGCATTCCAAACTGGATATCTAGAGTGCAGGAGAGGAAAGTAGAATTCTTAGTGTAGCGGTGAAATGCGTAGAGATTAGGAAGAATACCAGTGGCGAAGGCGACTTTCTGGACTGTAACTGACACTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATACTAGGTGTAGGGGTTGTCATGACCTCTGTGCCGCCGCTAACGCATTAAGTATTCCGCCTGGGGAGTACGGTC ->D.radiodurans.1 -TTAGGAATCTTCCACAATGGGCGCAAGCCTGATGGAGCGACGCCGCGTGAGGGATGAAGGTTTTCGGATCGTAAACCTCTGAATCTGGGACGAAAGAGCCTTAGGGCAGATGACGGTACCAGAGTAATAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTACCCGGAATCACTGGGCGTAAAGGGCGTGTAGGCGGAAATTTAAGTCTGGTTTTAAAGACCGGGGCTCAACCTCGGGGATGGACTGGATACTGGATTTCTTGACCTCTGGAGAGGTAACTGGAATTCCTGGTGTAGCGGTGGAATGCGTAGATACCAGGAGGAACACCAATGGCGAAGGCAAGTTACTGGACAGAAGGTGACGCTGAGGCGCGAAAGTGTGGGGAGCAAACCGGATTAGATACCCGGGTAGTCCACACCCTAAACGATGTACGTTGGCTAAGCGCAGGATGCTGTGCTTGGCGAAGCTAACGCGATAAACGTACCGCCTGGGAAGTACGGCC ->E.faecalis.1 -TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAACTCTGTTGTTAGAGAAGAACAAGGACGTTAGTAACTGAACGTCCCCTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCCCGGCTCAACCGGGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTGGAGGGTTTCCGCCCTTCAGTGCTGCAGCAAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->E.faecalis.2 -TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAACTCTGTTGTTAGAGAAGAACAAGGACGTTAGTAACTGAACGTCCCCTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCCCGGCTCAACCGGGGAGGGTCATTGGAAACTGGGAGACTTGAGTGCAGAAGAGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTGGAGGGTTTCCGCCCTTCAGTGCTGCAGCAAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->E.coli.1 -TGGGGAATATTGCACAATGGGCGCAAGCCTGATGCAGCCATGCCGCGTGTATGAAGAAGGCCTTCGGGTTGTAAAGTACTTTCAGCGGGGAGGAAGGGAGTAAAGTTAATACCTTTGCTCATTGACGTTACCCGCAGAAGAAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCACGCAGGCGGTTTGTTAAGTCAGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCTGATACTGGCAAGCTTGAGTCTCGTAGAGGGGGGTAGAATTCCAGGTGTAGCGGTGAAATGCGTAGAGATCTGGAGGAATACCGGTGGCGAAGGCGGCCCCCTGGACGAAGACTGACGCTCAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTTGGAGGTTGTGCCCTTGAGGCGTGGCTTCCGGAGCTAACGCGTTAAGTCGACCGCCTGGGGAGTACGGCC ->H.pylori.1 -TAGGGAATATTGCTCAATGGGGGAAACCCTGAAGCAGCAACGCCGCGTGGAGGATGAAGGTTTTAGGATTGTAAACTCCTTTTGTTAGAGAAGATAATGACGGTATCTAACGAATAAGCACCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGTGCAAGCGTTACTCGGAATCACTGGGCGTAAAGAGCGCGTAGGCGGGATAGTCAGTCAGGTGTGAAATCCTATGGCTTAACCATAGAACTGCATTTGAAACTACTATTCTAGAGTGTGGGAGAGGTAGGTGGAATTCTTGGTGTAGGGGTAAAATCCGTAGAGATCAAGAGGAATACTCATTGCGAAGGCGACCTGCTGGAACATTACTGACGCTGATTGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCCTAAACGATGGATGCTAGTTGTTGGAGGGCTTAGTCTCTCCAGTAATGCAGCTAACGCATTAAGCATCCCGCCTGGGGAGTACGGTC ->L.gasseri.1 -TAGGGAATCTTCCACAATGGACGCAAGTCTGATGGAGCAACGCCGCGTGAGTGAAGAAGGGTTTCGGCTCGTAAAGCTCTGTTGGTAGTGAAGAAAGATAGAGGTAGTAACTGGCCTTTATTTGACGGTAATTACTTAGAAAGTCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGTGCAGGCGGTTCAATAAGTCTGATGTGAAAGCCTTCGGCTCAACCGGAGAATTGCATCAGAAACTGTTGAACTTGAGTGCAGAAGAGGAGAGTGGAACTCCATGTGTAGCGGTGGAATGCGTAGATATATGGAAGAACACCAGTGGCGAAGGCGGCTCTCTGGTCTGCAACTGACGCTGAGGCTCGAAAGCATGGGTAGCGAACAGGATTAGATACCCTGGTAGTCCATGCCGTAAACGATGAGTGCTAAGTGTTGGGAGGTTTCCGCCTCTCAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->L.monocytogenes.1 -TAGGGAATCTTCCGCAATGGACGAAAGTCTGACGGAGCAACGCCGCGTGTATGAAGAAGGTTTTCGGATCGTAAAGTACTGTTGTTAGAGAAGAACAAGGATAAGAGTAACTGCTTGTCCCTTGACGGTATCTAACCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGATTTATTGGGCGTAAAGCGCGCGCAGGCGGTCTTTTAAGTCTGATGTGAAAGCCCCCGGCTTAACCGGGGAGGGTCATTGGAAACTGGAAGACTGGAGTGCAGAAGAGGAGAGTGGAATTCCACGTGTAGCGGTGAAATGCGTAGATATGTGGAGGAACACCAGTGGCGAAGGCGACTCTCTGGTCTGTAACTGACGCTGAGGCGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->N.meningitidis.1 -TGGGGAATTTTGGACAATGGGCGCAAGCCTGATCCAGCCATGCCGCGTGTCTGAAGAAGGCCTTCGGGTTGTAAAGGACTTTTGTCAGGGAAGAAAAGGCTGTTGCTAATATCAGCGGCTGATGACGGTACCTGAAGAATAAGCACCGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGGTGCGAGCGTTAATCGGAATTACTGGGCGTAAAGCGGGCGCAGACGGTTACTTAAGCAGGATGTGAAATCCCCGGGCTCAACCCGGGAACTGCGTTCTGAACTGGGTGACTCGAGTGTGTCAGAGGGAGGTAGAATTCCACGTGTAGCAGTGAAATGCGTAGAGATGTGGAGGAATACCGATGGCGAAGGCAGCCTCCTGGGACAACACTGACGTTCATGCCCGAAAGCGTGGGTAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCCTAAACGATGTCAATTAGCTGTTGGGCAACCTGATTGCTTGGTAGCGTAGCTAACGCGTGAAATTGACCGCCTGGGGAGTACGGTC ->P.acnes.1 -TGGGGAATATTGCACAATGGGCGGAAGCCTGATGCAGCAACGCCGCGTGCGGGATGACGGCCTTCGGGTTGTAAACCGCTTTCGCCTGTGACGAAGCGTGAGTGACGGTAATGGGTAAAGAAGCACCGGCTAACTACGTGCCAGCAGCCGCGGTGATACGTAGGGTGCGAGCGTTGTCCGGATTTATTGGGCGTAAAGGGCTCGTAGGTGGTTGATCGCGTCGGAAGTGTAATCTTGGGGCTTAACCCTGAGCGTGCTTTCGATACGGGTTGACTTGAGGAAGGTAGGGGAGAATGGAATTCCTGGTGGAGCGGTGGAATGCGCAGATATCAGGAGGAACACCAGTGGCGAAGGCGGTTCTCTGGGCCTTTCCTGACGCTGAGGAGCGAAAGCGTGGGGAGCGAACAGGCTTAGATACCCTGGTAGTCCACGCTGTAAACGGTGGGTACTAGGTGTGGGGTCCATTCCACGGGTTCCGTGCCGTAGCTAACGCTTTAAGTACCCCGCCTGGGGAGTACGGCC ->P.aeruginosa.1 -TGGGGAATATTGGACAATGGGCGAAAGCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGTCTTCGGATTGTAAAGCACTTTAAGTTGGGAGGAAGGGCAGTAAGTTAATACCTTGCTGTTTTGACGTTACCAACAGAATAAGCACCGGCTAACTTCGTGCCAGCAGCCGCGGTAATACGAAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCGCGTAGGTGGTTCAGCAAGTTGGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCCAAAACTACTGAGCTAGAGTACGGTAGAGGGTGGTGGAATTTCCTGTGTAGCGGTGAAATGCGTAGATATAGGAAGGAACACCAGTGGCGAAGGCGACCACCTGGACTGATACTGACACTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTAGCCGTTGGGATCCTTGAGATCTTAGTGGCGCAGCTAACGCGATAAGTCGACCGCCTGGGGAGTACGGCC ->P.aeruginosa.2 -TGGGGAATATTGGACAATGGGCGAAAGCCTGATCCAGCCATGCCGCGTGTGTGAAGAAGGTCTTCGGATTGTAAAGCACTTTAAGTTGGGAGGAAGGGCAGTAAGTTAATACCTTGCTGTTTTGACGTTACCAACAGAATAAGCACCGGCTAACTTCGTGCCAGCAGCCGCGGTAATACGAAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGCGCGTAGGTGGTTCAGCAAGTTGGATGTGAAATCCCCGGGCTCAACCTGGGAACTGCATCCAAAACTACTGAGCTAGAGTACGGTAGAGGGTGGTGGAATTTCCTGTGTAGCGGTGAAATGCGTAGATATAGGAAGGAACACCAGTGGCGAAGGCGACCACCTGGACTGATACTGACACTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGTCGACTAGCCGTTGGGATCCTTGAGATCATAGTGGCGCAGCTAACGCGATAAGTCGACCGCCTGGGGAGTACGGCC ->P.gingivalis.1 -TGAGGAATATTGGTCAATGGGCGAGAGCCTGAACCAGCCAAGTCGCGTGAAGGAAGACTGTCCTAAGGATTGTAAACTTCTTTTATACGGGAATAACGGGCGATACGAGTATTGCATTGAATGTACCGTAAGAATAAGCATCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGATGCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGTGCGTAGGTTGTTCGGTAAGTCAGCGGTGAAACCTGAGCGCTCAACGTTCAGCCTGCCGTTGAAACTGCCGGGCTTGAGTTCAGCGGCGGCAGGCGGAATTCGTGGTGTAGCGGTGAAATGCATAGATATCACGAGGAACTCCGATTGCGAAGGCAGCTTGCCATACTGCGACTGACACTGAAGCACGAAGGCGTGGGTATCAAACAGGATTAGATACCCTGGTAGTCCACGCAGTAAACGATGATTACTAGGAGTTTGCGATATACCGTCAAGCTTCCACAGCGAAAGCGTTAAGTAATCCACCTGGGGAGTACGCCG ->R.sphaeroides.1 -TGGGGAATCTTAGACAATGGGCGCAAGCCTGATCTAGCCATGCCGCGTGATCGATGAAGGCCTTAGGGTTGTAAAGATCTTTCAGGTGGGAAGATAATGACGGTACCACCAGAAGAAGCCCCGGCTAACTCCGTGCCAGCAGCCGCGGTAATACGGAGGGGGCTAGCGTTATTCGGAATTACTGGGCGTAAAGCGCACGTAGGCGGATCGGAAAGTCAGAGGTGAAATCCCAGGGCTCAACCCTGGAACTGCCTTTGAAACTCCCGATCTTGAGGTCGAGAGAGGTGAGTGGAATTCCGAGTGTAGAGGTGAAATTCGTAGATATTCGGAGGAACACCAGTGGCGAAGGCGGCTCACTGGCTCGATACTGACGCTGAGGTGCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAATGCCAGTCGTCGGGCAGCATGCTGTTCGGTGACACACCTAACGGATTAAGCATTCCGCCTGGGGAGTACGGCC ->S.aureus.1 -TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACATATGTGTAAGTAACTGTGCACATCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.aureus.5 -TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACATATGTGTAAGTAACTGTGCACATCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTTATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.epidermidis.1 -TAGGGAATCTTCCGCAATGGGCGAAAGCCTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTGTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.epidermidis.2 -TAGGGAATCTTCCGCAATGGGCGAAAGCTTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTGTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGCGCGTAGGCGGTTTTTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAAAACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.epidermidis.5 -TAGGGAATCTTCCGCAATGGGCGAAAGCTTGACGGAGCAACGCCGCGTGAGTGATGAAGGTCTTCGGATCGTAAAACTCTGTTATTAGGGAAGAACAAATGTCTAAGTAACTATGCACGTCTTGACGGTACCTAATCAGAAAGCCACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTGGCAAGCGTTGTCCGGAATCATTGGGCGTAAAGCGCGCGTAGGCGGTTTCTTAAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGAGGACTTGAGTGCAGAAGAGGAAAGTGGAATTCCATGTGTAGCGGTGAAATGCGCAGAGATATGGAGGAACACCAGTGGCGAAGGCGACTTTCTGGTCTGTAACTGACGCTGATGTGCGAAAGCGTGGGGATCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAAGTGTTAGGGGGTTTCCGCCCCTTAGTGCTGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.agalactiae.1 -TAGGGAATCTTCGGCAATGGACGGAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTTAGAGAAGAACGTTGGTAGGAGTGGAAAATCTACCAAGTGACGGTAACTAACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTCTTTAAGTCTGAAGTTAAAGGCAGTGGCTTAACCATTGTACGCTTTGGAAACTGGAGGACTTGAGTGCAGAAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCGGTGGCGAAAGCGGCTCTCTGGTCTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAGGTGTTAGGCCCTTTCCGGGGCTTAGTGCCGCAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC ->S.mutans.1 -TAGGGAATCTTCGGCAATGGACGAAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTAAGTCAAGAACGTGTGTGAGAGTGGAAAGTTCACACAGTGACGGTAGCTTACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGGGAGCGCAGGCGGTCAGGAAAGTCTGGAGTAAAAGGCTATGGCTCAACCATAGTGTGCTCTGGAAACTGTCTGACTTGAGTGCAGAAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCAGTGGCGAAAGCGGCTCTCTGGTCTGTCACTGACGCTGAGGCTCGAAAGCGTGGGTAGCGAACAGGATTAGATACCCTGGTAGTCCACGCCGTAAACGATGAGTGCTAGGTGTTAGGCCCTTTCCGGGGCTTAGTGCCGGAGCTAACGCAATAAGCACTCCGCCTGGGGAGTACGACC ->S.pneumoniae.1 -TAGGGAATCTTCGGCAATGGACGGAAGTCTGACCGAGCAACGCCGCGTGAGTGAAGAAGGTTTTCGGATCGTAAAGCTCTGTTGTAAGAGAAGAACGAGTGTGAGAGTGGAAAGTTCACACTGTGACGGTATCTTACCAGAAAGGGACGGCTAACTACGTGCCAGCAGCCGCGGTAATACGTAGGTCCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGCGAGCGCAGGCGGTTAGATAAGTCTGAAGTTAAAGGCTGTGGCTTAACCATAGTAGGCTTTGGAAACTGTTTAACTTGAGTGCAAGAGGGGAGAGTGGAATTCCATGTGTAGCGGTGAAATGCGTAGATATATGGAGGAACACCGGTGGCGAAAGCGGCTCTCTGGCTTGTAACTGACGCTGAGGCTCGAAAGCGTGGGGAGCAAACAGGATTAGATACCCTGGTAGTCCACGCTGTAAACGATGAGTGCTAGGTGTTAGACCCTTTCCGGGGTTTAGTGCCGTAGCTAACGCATTAAGCACTCCGCCTGGGGAGTACGACC diff --git a/data/MiSeq_SOP/mouse.dpw.metadata b/data/MiSeq_SOP/mouse.dpw.metadata deleted file mode 100644 index e754f1c..0000000 --- a/data/MiSeq_SOP/mouse.dpw.metadata +++ /dev/null @@ -1,20 +0,0 @@ -group dpw -F3D0 0 -F3D1 1 -F3D141 141 -F3D142 142 -F3D143 143 -F3D144 144 -F3D145 145 -F3D146 146 -F3D147 147 -F3D148 148 -F3D149 149 -F3D150 150 -F3D2 2 -F3D3 3 -F3D5 5 -F3D6 6 -F3D7 7 -F3D8 8 -F3D9 9 diff --git a/data/MiSeq_SOP/mouse.time.design b/data/MiSeq_SOP/mouse.time.design deleted file mode 100644 index 16a7421..0000000 --- a/data/MiSeq_SOP/mouse.time.design +++ /dev/null @@ -1,20 +0,0 @@ -group time -F3D0 Early -F3D1 Early -F3D141 Late -F3D142 Late -F3D143 Late -F3D144 Late -F3D145 Late -F3D146 Late -F3D147 Late -F3D148 Late -F3D149 Late -F3D150 Late -F3D2 Early -F3D3 Early -F3D5 Early -F3D6 Early -F3D7 Early -F3D8 Early -F3D9 Early diff --git a/data/MiSeq_SOP/stability.batch b/data/MiSeq_SOP/stability.batch deleted file mode 100644 index b818716..0000000 --- a/data/MiSeq_SOP/stability.batch +++ /dev/null @@ -1,26 +0,0 @@ -pcr.seqs(fasta=silva.bacteria.fasta, start=11894, end=25319, keepdots=F) -rename.file(input=silva.bacteria.pcr.fasta, new=silva.v4.fasta) - -#change the name of the file from stability.files to whatever suits your study -make.file(inputdir=., type=fastq, prefix=stability) -make.contigs(file=current, maxambig=0, maxlength=275, maxhomop=8) -unique.seqs(fasta=stability.trim.contigs.fasta, count=stability.contigs.count_table) -align.seqs(fasta=current, reference=silva.v4.fasta) -screen.seqs(fasta=current, count=current, start=1969, end=11551) -filter.seqs(fasta=current, vertical=T, trump=.) -unique.seqs(fasta=current, count=current) -pre.cluster(fasta=current, count=current, diffs=2) -chimera.vsearch(fasta=current, count=current, dereplicate=t) -classify.seqs(fasta=current, count=current, reference=trainset9_032012.pds.fasta, taxonomy=trainset9_032012.pds.tax) -remove.lineage(fasta=current, count=current, taxonomy=current, taxon=Chloroplast-Mitochondria-unknown-Archaea-Eukaryota) -remove.groups(count=current, fasta=current, taxonomy=current, groups=Mock) -cluster.split(fasta=current, count=current, taxonomy=current, taxlevel=4, cutoff=0.03) -make.shared(list=current, count=current, label=0.03) -classify.otu(list=current, count=current, taxonomy=current, label=0.03) -phylotype(taxonomy=current) -make.shared(list=current, count=current, label=1) -classify.otu(list=current, count=current, taxonomy=current, label=1) -make.shared(count=current) -classify.otu(list=current, count=current, taxonomy=current, label=ASV) -dist.seqs(fasta=current, output=lt) -clearcut(phylip=current) diff --git a/data/MiSeq_SOP/stability.files b/data/MiSeq_SOP/stability.files deleted file mode 100644 index 0ff5fec..0000000 --- a/data/MiSeq_SOP/stability.files +++ /dev/null @@ -1,20 +0,0 @@ -F3D0 F3D0_S188_L001_R1_001.fastq F3D0_S188_L001_R2_001.fastq -F3D141 F3D141_S207_L001_R1_001.fastq F3D141_S207_L001_R2_001.fastq -F3D142 F3D142_S208_L001_R1_001.fastq F3D142_S208_L001_R2_001.fastq -F3D143 F3D143_S209_L001_R1_001.fastq F3D143_S209_L001_R2_001.fastq -F3D144 F3D144_S210_L001_R1_001.fastq F3D144_S210_L001_R2_001.fastq -F3D145 F3D145_S211_L001_R1_001.fastq F3D145_S211_L001_R2_001.fastq -F3D146 F3D146_S212_L001_R1_001.fastq F3D146_S212_L001_R2_001.fastq -F3D147 F3D147_S213_L001_R1_001.fastq F3D147_S213_L001_R2_001.fastq -F3D148 F3D148_S214_L001_R1_001.fastq F3D148_S214_L001_R2_001.fastq -F3D149 F3D149_S215_L001_R1_001.fastq F3D149_S215_L001_R2_001.fastq -F3D150 F3D150_S216_L001_R1_001.fastq F3D150_S216_L001_R2_001.fastq -F3D1 F3D1_S189_L001_R1_001.fastq F3D1_S189_L001_R2_001.fastq -F3D2 F3D2_S190_L001_R1_001.fastq F3D2_S190_L001_R2_001.fastq -F3D3 F3D3_S191_L001_R1_001.fastq F3D3_S191_L001_R2_001.fastq -F3D5 F3D5_S193_L001_R1_001.fastq F3D5_S193_L001_R2_001.fastq -F3D6 F3D6_S194_L001_R1_001.fastq F3D6_S194_L001_R2_001.fastq -F3D7 F3D7_S195_L001_R1_001.fastq F3D7_S195_L001_R2_001.fastq -F3D8 F3D8_S196_L001_R1_001.fastq F3D8_S196_L001_R2_001.fastq -F3D9 F3D9_S197_L001_R1_001.fastq F3D9_S197_L001_R2_001.fastq -Mock Mock_S280_L001_R1_001.fastq Mock_S280_L001_R2_001.fastq From bae0473d57ca3c92605bf7247bd8cc0f657ed7dd Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 21:48:20 +0100 Subject: [PATCH 022/175] Added better logging, removed openxlsx R dependency. --- install.jl | 10 ++++++---- src/call_tools.jl | 14 +++++++------- src/dada2/dada2_functions.r | 5 +---- src/dada2/taxonomy.jl | 2 +- src/main.jl | 11 ++++++----- src/merge_and_filter_taxa.jl | 1 + 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/install.jl b/install.jl index 89df766..47e23d8 100644 --- a/install.jl +++ b/install.jl @@ -398,8 +398,7 @@ function resolve_tool( end end -# Main - +## Main function main() println() println("╔═══════════════════════════════════════════╗") @@ -439,14 +438,17 @@ function main() # R packages println() println(" ─── R packages ─────────────────────────────────────────────") - r_packages = ["dada2", "openxlsx", "tidyverse", "yaml"] - if prompt_yn(" Install/check R packages (dada2, openxlsx, tidyverse, yaml)?") + r_packages = ["dada2", "tidyverse"] + if prompt_yn(" Install/check R packages (dada2, tidyverse)?") install_r_packages(r_packages) end # Write config write_tools_config(resolved) + # Create a data directory + mkpath("data") + println() println("Installation complete.") println() diff --git a/src/call_tools.jl b/src/call_tools.jl index 7c9e6b4..5eeed48 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -204,10 +204,10 @@ export cutadapt, vsearch, fastqc_all, fastqc_one """ function fastqc_all(fastq_in_dir, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) mkpath(fastqc_dir) - + @info "FastQC running on $fastq_in_dir" cmd = "$fastqc_bin $fastq_in_dir/*.fastq* -o $fastqc_dir $optional_args" run(`bash -lc $cmd`) - + @info "FastQC complete. Output: $fastqc_dir" end """ @@ -225,10 +225,10 @@ export cutadapt, vsearch, fastqc_all, fastqc_one """ function fastqc_one(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) mkpath(fastqc_dir) - + @info "FastQC running on $fastq_in_file" cmd = "$fastqc_bin $fastq_in_file -o $fastqc_dir $optional_args" run(`bash -lc $cmd`) - + @info "FastQC complete. Output: $fastqc_dir" end ## VSEARCH @@ -249,9 +249,9 @@ export cutadapt, vsearch, fastqc_all, fastqc_one function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = tool_bin("vsearch")) mkpath(vsearch_dir) outfile = joinpath(vsearch_dir, "taxonomy.tsv") - + @info "VSEARCH running: $fasta_in_dir against $(basename(reference_database))" cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" run(`bash -lc $cmd`) - + @info "VSEARCH complete. Output: $outfile" end -end \ No newline at end of file +end diff --git a/src/dada2/dada2_functions.r b/src/dada2/dada2_functions.r index c74795f..76d084a 100644 --- a/src/dada2/dada2_functions.r +++ b/src/dada2/dada2_functions.r @@ -15,7 +15,6 @@ # https://creativecommons.org/licenses/by/4.0/. library(dada2) -library(openxlsx) library(tidyverse) # library(yaml) @@ -244,8 +243,6 @@ write_combined_table <- function(taxa_df, seq_table, tables_dir, filename) { seq_t <- tibble::rownames_to_column(seq_t, "Sequence") combined <- dplyr::left_join(taxa_df, seq_t, by = "Sequence") output_path <- file.path(tables_dir, filename) - openxlsx::write.xlsx(combined, output_path, overwrite = TRUE, - asTable = FALSE, sheetName = "1.sampling", - firstRow = TRUE, zoom = 90, keepNA = TRUE) + write.csv(combined, output_path, quote = FALSE, row.names = FALSE) invisible(NULL) } diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index f2e3580..021b6f5 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -39,7 +39,7 @@ seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") - combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.xlsx") + combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.csv") boot_mode = get(ctx.cfg["output"], "bootstraps", "combined") comb_mode = get(ctx.cfg["output"], "combined_mode", "regular") tables_dir = ctx.dirs["Tables"] diff --git a/src/main.jl b/src/main.jl index 1f5d7a7..c4b27a5 100755 --- a/src/main.jl +++ b/src/main.jl @@ -52,6 +52,7 @@ dada2_config_path = joinpath(config_dir, "dada2.yml") vsearch_optional_args = "--id 0.75 --query_cov 0.8" # Merge and filter (DADA2-VSEARCH) parameters +mkpath(merged_dir) multiv = joinpath(vsearch_dir, "taxonomy.tsv") multid = joinpath(dada2_dir, "Tables/taxonomy.csv") @@ -61,13 +62,13 @@ protist_filter = joinpath(config_dir, "protist_filter.yml") dbs = ensure_databases(dada2_config_path) ## Main -fastqc_all(fastq_input_dir, fastqc_dir) +#fastqc_all(fastq_input_dir, fastqc_dir) -cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) +#cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) #dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) -vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) +#vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) -CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)) -CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)) +CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)); @info "Written: $merged_outfile_multi" +CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)); @info "Written: $filtered_outfile_multi" diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index eaba301..4a915a7 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -164,6 +164,7 @@ export merge_taxonomy_counts, filter_table # Left join instead of outerjoin to keep all taxonomy rows merged_df = leftjoin(df_taxonomy, df_counts_prepared, on="SeqName") sort!(merged_df, "SeqName", by=seqnum) + @info "Merge complete. $(nrow(merged_df)) rows." return merged_df end From 41cfde02d6e035f5b4cf3289c721a340951467b8 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 23:17:51 +0100 Subject: [PATCH 023/175] Allow offload assignTaxonomy() to server. --- .gitignore | 1 + README.md | 137 +++++++++++++++-------- config/{dada2.yml => dada2.example.yml} | 18 ++- src/dada2.jl | 27 +++-- src/dada2/context.jl | 57 ++-------- src/dada2/dada2_functions.r | 52 ++++----- src/dada2/taxonomy.jl | 141 ++++++++++++++++++++---- src/dada2/taxonomy_remote.r | 50 +++++++++ src/main.jl | 2 +- 9 files changed, 321 insertions(+), 164 deletions(-) rename config/{dada2.yml => dada2.example.yml} (64%) create mode 100644 src/dada2/taxonomy_remote.r diff --git a/.gitignore b/.gitignore index 42759f2..34d42c9 100644 --- a/.gitignore +++ b/.gitignore @@ -264,6 +264,7 @@ cutadapt/output_* bin/ databases/ config/tools.yml +config/dada2.yml ### Bits I don't want to share yet hide.* diff --git a/README.md b/README.md index 734c291..65c8fe4 100644 --- a/README.md +++ b/README.md @@ -87,12 +87,14 @@ This allows you to referenece one or multiple primer pairs within the pipeline b ### Configuring DADA2 (`dada2.yml`) -> **Performance tip:** For large datasets and reference databases, consider installing the [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides acceleration for CPU and Nvidia CUDA GPUs for taxonomy assignment, with no changes to the API or configuration required. This fork is experimental, so if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. +> **Performance tip:** For large datasets and reference databases, consider installing the [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides acceleration for CPU and Nvidia CUDA GPUs for taxonomy assignment, with no changes to the API or configuration required. This fork is experimental, so if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. It was too experimental for me, so opted to offload assignTaxonomy() to server in dada2.yml... + +Copy `config/dada2.example.yml` to `config/dada2.yml` and edit it. It contains your local paths and optionally remote server credentials and should never be shared. ```yaml workspace: root: "./output/dada2/" # output directories are created here - input_dir: "/path/to/fastqs" # trimmed FASTQ input (cutadapt output) + input_dir: "./output/cutadapt/" # trimmed FASTQ input (cutadapt output) file_patterns: forward: "_R1_trimmed.fastq.gz" @@ -101,49 +103,52 @@ file_patterns: sample_name_index: 1 # which element is the sample name (1-based) mode: "paired" # paired | forward | reverse -##Filter and trim - corresponds to DADA2's `filterAndTrim()`: - +# Filter and trim - DADA2's filterAndTrim(): filter_trim: - trunc_q: 2 # truncate reads at first base with quality less than this - trunc_len: [220, 220] # truncate F and R reads to this length; single value for single-end - max_ee: [3, 3] # maximum errors permitted in F and R reads - min_len: 175 # discard reads shorter than this after truncation - max_n: 0 # discard reads containing any ambiguous bases - match_ids: true # require F/R read ID's to match - rm_phix: true # remove PhiX spike-in reads - -##Denoising - corresponds to `learnErrors()` and `dada()`: - + trunc_q: 2 + trunc_len: [220, 220] # [forward, reverse]; first value used for single-end + max_ee: [3, 3] # maximum expected errors in F and R reads + min_len: 175 + max_n: 0 + match_ids: true + rm_phix: true + +# Denoising - learnErrors() and dada(): dada: - seed: 123 # random seed for reproducibility - nbases: 200000000 # bases used to learn the error model - max_consist: 15 # error model convergence iterations - pool_method: "pseudo" # none | pseudo | true - # pseudo improves sensitivity for rare variants across samples - -## Merging - `mergePairs()`, paired mode only: + seed: 123 + nbases: 200000000 + max_consist: 15 + pool_method: "pseudo" # none | pseudo | true +# Merging - mergePairs(), paired mode only: merge: - min_overlap: 20 # minimum overlap between F and R reads - max_mismatch: 0 # mismatches permitted in the overlap region + min_overlap: 20 + max_mismatch: 0 trim_overhang: true -## ASV length filtering and chimera removal: - +# ASV length filtering and chimera removal: asv: - band_size_min: 250 # retain only ASVs within this length range (null to skip) - band_size_max: 256 - denovo_method: "consensus" # chimera removal method: consensus | pooled | per-sample - -## Taxonomy - `assignTaxonomy()` against a reference database: - + band_size_min: 200 # null to skip length filtering + band_size_max: 430 + denovo_method: "consensus" # consensus | pooled | per-sample + +# Reference databases - downloaded and cached on first use: +databases: + dir: "./databases" + pr2: + dada2: + uri: "https://..." # DADA2-format FASTA + local: ~ # set to a local path to skip download + vsearch: + uri: "https://..." # vsearch-format FASTA + local: ~ + +# Taxonomy - assignTaxonomy() against the configured database: taxonomy: - skip: false # set true to skip taxonomy and output SeqName + Sequence + counts only - uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" - # URL or local path to a DADA2-formatted reference FASTA - multithread: true - min_boot: 0 # minimum bootstrap confidence to retain an assignment (0–100) - levels: # taxonomic ranks in the reference database (must match its headers) + database: pr2 # key into databases: section above + multithread: 4 # threads for assignTaxonomy(); higher = more RAM + min_boot: 0 # minimum bootstrap confidence to retain (0-100) + levels: - "Domain" - "Supergroup" - "Division" @@ -154,17 +159,38 @@ taxonomy: - "Genus" - "Species" -## Output - + # Optional: offload the memory-intensive assignTaxonomy() step to a remote + # server via SSH. Omit or set host to null to run locally. + # DISCLAIMER: You are solely responsible for ensuring you have authorisation + # to use the configured host. See dada2.example.yml for the full disclaimer. + remote: + host: ~ # user@hostname + rscript: "Rscript" # path to Rscript on the server + staging_dir: "/absolute/path/on/server" + db_path: ~ # absolute path to database on server (null = transfer local copy) + +# Output filenames (all written to workspace.root/Tables/): output: - bootstraps: "combined" # none | combined (appended columns) | separate (second sheet) - combined_mode: "regular" # regular: taxonomy + counts | alternative: SeqName + Sequence + counts seq_table_prefix: "seqtab_nochim" fasta_prefix: "asvs" taxa_prefix: "taxonomy" - combined_filename: "tax_counts.xlsx" + combined_filename: "tax_counts.csv" + asv_filename: "asv_counts.csv" ``` +**Outputs written to `workspace.root/Tables/`:** + +| File | Contents | +|------|----------| +| `seqtab_nochim.csv` | Chimera-free ASV count table (samples × ASVs) | +| `asvs.fasta` / `asvs.csv` | ASV sequences with short identifiers (seq1, seq2, …) | +| `taxonomy.csv` | Taxonomy assignments per ASV | +| `taxonomy_bootstraps.csv` | Bootstrap confidence values per rank | +| `taxonomy_combined.csv` | Taxonomy + bootstrap columns combined | +| `tax_counts.csv` | Taxonomy + per-sample counts | +| `asv_counts.csv` | ASV sequences + per-sample counts (no taxonomy) | +| `pipeline_stats.csv` | Read counts retained at each pipeline stage | + ### Configuring taxonomic filtering (`protist_filter.yml`) The filter file controls the `filter_table()` step, which removes non-target taxa @@ -244,12 +270,27 @@ SampleName_*_L001_R2_001.fastq.gz ### Output structure ``` -output/ -├── cutadapt/ # Trimmed FASTQ pairs and per-run logs -├── dada2/ # ASV count table, FASTA, taxonomy CSV, pipeline stats -└── vsearch/ # Taxonomy TSV files -merged_multi.csv # Merged DADA2 + vsearch results -protist_filtered.csv # After taxonomic filtering +output/{project_name}/ +├── cutadapt/ # Trimmed FASTQ pairs and logs +├── FastQC/ # FastQC HTML reports +├── dada2/ +│ ├── Tables/ +│ │ ├── seqtab_nochim.csv # ASV count table +│ │ ├── asvs.fasta # ASV sequences +│ │ ├── asvs.csv # ASV sequence index +│ │ ├── taxonomy.csv # Taxonomy assignments +│ │ ├── taxonomy_bootstraps.csv +│ │ ├── taxonomy_combined.csv +│ │ ├── tax_counts.csv # Taxonomy + per-sample counts +│ │ ├── asv_counts.csv # Sequences + per-sample counts +│ │ └── pipeline_stats.csv +│ ├── Figures/ # Quality profile and error rate PDFs +│ └── Checkpoints/ # RData checkpoints for stage resumption +├── vsearch/ +│ └── taxonomy.tsv +└── merged/ + ├── merged_multi.csv # Merged DADA2 + vsearch taxonomy + counts + └── protist_filtered_multi.csv ``` ## Third-party tools diff --git a/config/dada2.yml b/config/dada2.example.yml similarity index 64% rename from config/dada2.yml rename to config/dada2.example.yml index 4d14965..18a32da 100644 --- a/config/dada2.yml +++ b/config/dada2.example.yml @@ -1,6 +1,6 @@ workspace: root: "./output/dada2/" - input_dir: "./output/cutadapt/output_2026-01-31T19:51" + input_dir: "./output/cutadapt/" # path to trimmed FASTQ directory file_patterns: forward: "_R1_trimmed.fastq.gz" @@ -46,8 +46,17 @@ databases: taxonomy: database: pr2 # key into databases: section above - skip: false # true to skip taxonomy and use alternative output mode multithread: 4 # higher values increase memory use significantly (mclapply forks one process per thread) + # DISCLAIMER: Configuring a remote host causes this pipeline to connect via SSH, + # transfer files, execute Rscript, and delete the staging directory on the remote + # server. You are solely responsible for ensuring you have authorisation to use + # the configured host and that staging_dir is a safe path to write to and delete. + # The authors accept no liability for data loss or misuse arising from this feature. + remote: # omit or set host to null to run locally + host: ~ # user@hostname + rscript: "Rscript" # path to Rscript on the server + staging_dir: "/absolute/path/on/server" # must be an explicit absolute path on the server + db_path: ~ # path to pre-installed database on server; null to transfer local copy min_boot: 0 levels: - "Domain" @@ -61,11 +70,10 @@ taxonomy: - "Species" output: - bootstraps: "combined" # none | combined | separate - combined_mode: "regular" # regular (taxonomy + counts) | alternative (SeqName + Sequence + counts, no taxonomy) seq_table_prefix: "seqtab_nochim" fasta_prefix: "asvs" taxa_prefix: "taxonomy" - combined_filename: "tax_counts.xlsx" + combined_filename: "tax_counts.csv" + asv_filename: "asv_counts.csv" verbose: true diff --git a/src/dada2.jl b/src/dada2.jl index 80d430b..28365c3 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -15,7 +15,7 @@ module DADA2 # # Notice: # -# (c) 2026 Joshua Benjamin Jewell. All rights reserved. +# © 2026 Joshua Benjamin Jewell. All rights reserved. # # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). # @@ -51,11 +51,14 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, ## Outputs Written to `workspace.root/Tables/`: - - `seqtab_nochim.csv` - chimera-free ASV count table - - `asvs.fasta` / `asvs.csv` - ASV sequences with short identifiers - - `taxonomy.csv` - taxonomy assignments (with optional bootstraps) - - `tax_counts.xlsx` - combined taxonomy + per-sample counts - - `pipeline_stats.csv` - read counts at each pipeline stage + - `seqtab_nochim.csv` - chimera-free ASV count table + - `asvs.fasta` / `asvs.csv` - ASV sequences with short identifiers + - `taxonomy.csv` - taxonomy assignments + - `taxonomy_bootstraps.csv` - bootstrap confidence values + - `taxonomy_combined.csv` - taxonomy + bootstraps combined + - `tax_counts.csv` - taxonomy + per-sample counts + - `asv_counts.csv` - ASV sequences + per-sample counts (no taxonomy) + - `pipeline_stats.csv` - read counts at each pipeline stage Written to `workspace.root/Checkpoints/`: - `ckpt_filter.RData` - filter_stats @@ -66,12 +69,12 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `checkpoint.RData` - final R environment snapshot """ function dada2(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) - prefilter_qc(config_path; progress, input_dir, workspace_root) - filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" - learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" - denoise(config_path; progress, input_dir, workspace_root); R"gc()" - filter_length(config_path; progress, input_dir, workspace_root); R"gc()" - chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" + #prefilter_qc(config_path; progress, input_dir, workspace_root) + #filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" + #learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" + #denoise(config_path; progress, input_dir, workspace_root); R"gc()" + #filter_length(config_path; progress, input_dir, workspace_root); R"gc()" + #chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end diff --git a/src/dada2/context.jl b/src/dada2/context.jl index f37b1b7..f3eee23 100644 --- a/src/dada2/context.jl +++ b/src/dada2/context.jl @@ -34,39 +34,19 @@ end # Resolve the DADA2 taxonomy database from config when no external path is - # provided. Supports the databases:/taxonomy.database: scheme (preferred) - # and the legacy taxonomy.uri: key for backwards compatibility. + # provided. Reads taxonomy.database key and looks it up in the databases: section. function _resolve_taxonomy_db(cfg, emit) tax_cfg = cfg["taxonomy"] - if haskey(tax_cfg, "database") - db_key = string(tax_cfg["database"]) - db_cfg = get(cfg, "databases", Dict()) - haskey(db_cfg, db_key) || - error("taxonomy.database = \"$db_key\" not found in databases: section") - fmt_cfg = get(db_cfg[db_key], "dada2", nothing) - isnothing(fmt_cfg) && - error("databases.$db_key.dada2 is not configured") - db_dir = abspath(get(db_cfg, "dir", "./databases")) - mkpath(db_dir) - return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) - end - # Legacy: taxonomy.uri - tax_uri = tax_cfg["uri"] - if isfile(tax_uri) - emit("Using local taxonomy database: $tax_uri") - return tax_uri - end - db_dir = abspath(get(cfg, "databases_dir", "./databases")) + db_key = string(tax_cfg["database"]) + db_cfg = get(cfg, "databases", Dict()) + haskey(db_cfg, db_key) || + error("taxonomy.database = \"$db_key\" not found in databases: section") + fmt_cfg = get(db_cfg[db_key], "dada2", nothing) + isnothing(fmt_cfg) && + error("databases.$db_key.dada2 is not configured") + db_dir = abspath(get(db_cfg, "dir", "./databases")) mkpath(db_dir) - cached = joinpath(db_dir, basename(tax_uri)) - if !isfile(cached) - emit("Downloading taxonomy database: $tax_uri") - Downloads.download(tax_uri, cached) - emit("Written: $cached") - else - emit("Using cached taxonomy database: $cached") - end - return cached + return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) end # Config @@ -87,21 +67,8 @@ mode in ("paired", "forward", "reverse") || error("file_patterns.mode must be one of: paired, forward, reverse") - boot_mode = get(cfg["output"], "bootstraps", "combined") - boot_mode in ("none", "combined", "separate") || - error("output.bootstraps must be one of: none, combined, separate") - - if !get(cfg["taxonomy"], "skip", false) - has_uri = haskey(cfg["taxonomy"], "uri") - has_db = haskey(cfg["taxonomy"], "database") - has_uri || has_db || - error("taxonomy: configure `database` (referencing a databases: entry) " * - "or `uri` when skip is not true") - end - - combined_mode = get(cfg["output"], "combined_mode", "regular") - combined_mode in ("regular", "alternative") || - error("output.combined_mode must be one of: regular, alternative") + haskey(cfg["taxonomy"], "database") || + error("taxonomy.database is required") end # File discovery diff --git a/src/dada2/dada2_functions.r b/src/dada2/dada2_functions.r index 76d084a..e246efc 100644 --- a/src/dada2/dada2_functions.r +++ b/src/dada2/dada2_functions.r @@ -202,47 +202,37 @@ write_fasta <- function(seq_table, tables_dir, prefix) { } # Writes taxonomy assignments to CSV. Bootstrap confidence values (0-100 per -# taxonomic rank) are handled based on bootstrap_mode: -# none - taxonomy columns only -# combined - taxonomy and bootstrap columns in one file (suffix: _boot) -# separate - taxonomy and bootstraps written to two separate files +# taxonomic rank) # Returns taxa_df (SeqName + Sequence + taxonomy), used by write_combined_table(). -write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, - prefix, bootstrap_mode) { +write_taxa_table <- function(tax_matrix, boot_matrix, index, tables_dir, prefix) { taxa_df <- as.data.frame(tax_matrix, stringsAsFactors = FALSE) taxa_df$Sequence <- rownames(taxa_df) taxa_df <- dplyr::left_join(index, taxa_df, by = "Sequence") - if (bootstrap_mode == "combined") { - boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) - colnames(boot_df) <- paste0(colnames(boot_df), "_boot") - boot_df$Sequence <- rownames(boot_matrix) - combined_df <- dplyr::left_join(taxa_df, boot_df, by = "Sequence") - write.csv(combined_df, file.path(tables_dir, paste0(prefix, ".csv")), - quote = FALSE, row.names = FALSE) - } else { - write.csv(taxa_df, file.path(tables_dir, paste0(prefix, ".csv")), - quote = FALSE, row.names = FALSE) - if (bootstrap_mode == "separate") { - boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) - boot_df$Sequence <- rownames(boot_matrix) - boot_df <- dplyr::left_join(index, boot_df, by = "Sequence") - write.csv(boot_df, - file.path(tables_dir, paste0(prefix, "_bootstraps.csv")), - quote = FALSE, row.names = FALSE) - } - } + write.csv(taxa_df, file.path(tables_dir, paste0(prefix, ".csv")), + quote = FALSE, row.names = FALSE) + + boot_df <- as.data.frame(boot_matrix, stringsAsFactors = FALSE) + colnames(boot_df) <- paste0(colnames(boot_df), "_boot") + boot_df$Sequence <- rownames(boot_matrix) + boot_df <- dplyr::left_join(index, boot_df, by = "Sequence") + write.csv(boot_df, file.path(tables_dir, paste0(prefix, "_bootstraps.csv")), + quote = FALSE, row.names = FALSE) + + combined_df <- dplyr::left_join(taxa_df, boot_df, by = c("SeqName", "Sequence")) + write.csv(combined_df, file.path(tables_dir, paste0(prefix, "_combined.csv")), + quote = FALSE, row.names = FALSE) taxa_df } -# Joins taxonomy (or index for alternative mode) with per-sample counts and -# writes an Excel workbook. -write_combined_table <- function(taxa_df, seq_table, tables_dir, filename) { +# Joins taxonomy with per-sample counts +write_combined_table <- function(taxa_df, index, seq_table, tables_dir, tax_filename, asv_filename) { seq_t <- as.data.frame(t(seq_table), stringsAsFactors = FALSE) seq_t <- tibble::rownames_to_column(seq_t, "Sequence") - combined <- dplyr::left_join(taxa_df, seq_t, by = "Sequence") - output_path <- file.path(tables_dir, filename) - write.csv(combined, output_path, quote = FALSE, row.names = FALSE) + write.csv(dplyr::left_join(taxa_df, seq_t, by = "Sequence"), + file.path(tables_dir, tax_filename), quote = FALSE, row.names = FALSE) + write.csv(dplyr::left_join(index, seq_t, by = "Sequence"), + file.path(tables_dir, asv_filename), quote = FALSE, row.names = FALSE) invisible(NULL) } diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index 021b6f5..dfaa4fd 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -13,6 +13,97 @@ Requires: `Checkpoints/ckpt_chimera.RData` Saves: `Checkpoints/checkpoint.RData` """ + # DISCLAIMER: Remote taxonomy execution connects to a user-configured server via SSH, + # transfers files via SCP, executes Rscript, and deletes the staging directory on + # completion. The user is solely responsible for ensuring they have authorisation to + # use the configured remote host, that the staging_dir is a safe path to write to + # and delete from, and that the remote server has sufficient resources. The authors + # of this software accept no liability for unintended data loss, unauthorised access, + # or any other consequences arising from misconfiguration or misuse of this feature. + function _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, + checkpoint, taxa_prefix, multithread, + min_boot, tax_levels, verbose, remote_cfg) + host = remote_cfg["host"] + rscript = get(remote_cfg, "rscript", "Rscript") + base_dir = get(remote_cfg, "staging_dir", nothing) + remote_db = get(remote_cfg, "db_path", nothing) + + isnothing(base_dir) && + error("taxonomy.remote.staging_dir must be set explicitly in config") + !isnothing(remote_db) && !startswith(string(remote_db), "/") && + error("taxonomy.remote.db_path must be an absolute path on the server " * + "(got: '$remote_db'). Do not include the hostname.") + + # Append a unique run ID so cleanup only ever touches this specific run's dir. + run_id = string(floor(Int, time())) + staging_dir = "$base_dir/run_$run_id" + + scripts_dir = @__DIR__ + functions_r = joinpath(scripts_dir, "dada2_functions.r") + remote_r = joinpath(scripts_dir, "taxonomy_remote.r") + remote_tables = "$staging_dir/Tables" + remote_ckpt = "$staging_dir/checkpoint.RData" + + # Use a ControlMaster socket so the password is entered once and all + # subsequent ssh/scp calls reuse the existing connection silently. + ctl = "/tmp/ssh_mux_$run_id" + ssh = (args...) -> `ssh -o ControlMaster=auto -o ControlPath=$ctl -o ControlPersist=yes $args` + scp = (args...) -> `scp -o ControlMaster=auto -o ControlPath=$ctl -q $args` + + emit(" Setting up staging directory on $host") + run(ssh(host, "mkdir -p $remote_tables")) + + emit(" Transferring files to $host") + run(scp(chimera_ckpt, "$host:$staging_dir/ckpt_chimera.RData")) + run(scp(functions_r, "$host:$staging_dir/dada2_functions.r")) + run(scp(remote_r, "$host:$staging_dir/taxonomy_remote.r")) + + remote_db_path = if !isnothing(remote_db) + emit(" Using remote database: $remote_db") + string(remote_db) + else + db_basename = basename(db_path) + emit(" Transferring database ($db_basename) to $host") + run(scp(db_path, "$host:$staging_dir/$db_basename")) + "$staging_dir/$db_basename" + end + + levels_str = join(tax_levels, ",") + verbose_str = verbose ? "true" : "false" + remote_cmd = "$rscript $staging_dir/taxonomy_remote.r " * + "functions=$staging_dir/dada2_functions.r " * + "ckpt=$staging_dir/ckpt_chimera.RData " * + "db=$remote_db_path " * + "tables=$remote_tables " * + "save=$remote_ckpt " * + "prefix=$taxa_prefix " * + "multithread=$multithread " * + "min_boot=$min_boot " * + "levels=$levels_str " * + "verbose=$verbose_str" + + emit(" Running Rscript on $host:$staging_dir") + run(ssh(host, remote_cmd)) + + emit(" Retrieving results from $host") + run(scp("$host:$remote_tables/$taxa_prefix.csv", "$tables_dir/")) + run(scp("$host:$remote_tables/$(taxa_prefix)_bootstraps.csv", "$tables_dir/")) + run(scp("$host:$remote_tables/$(taxa_prefix)_combined.csv", "$tables_dir/")) + run(scp("$host:$remote_ckpt", checkpoint)) + + # Safety check before cleanup: staging_dir must be at least 3 components + # deep to guard against dangerous paths resolving to / or home root. + parts = filter(!isempty, split(staging_dir, '/')) + if length(parts) >= 3 + emit(" Cleaning up $host:$staging_dir") + run(ssh(host, "rm -rf $staging_dir")) + else + @warn "Skipping remote cleanup: staging path '$staging_dir' looks too shallow to delete safely" + end + + run(`ssh -o ControlPath=$ctl -O exit $host`) + end + function assign_taxonomy(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) emit = _emitter(progress) ctx = _pipeline_context(config_path; input_dir, workspace_root) @@ -33,54 +124,60 @@ gc() """ - chimera_ckpt = ctx.ckpts["chimera"] - R"load($chimera_ckpt)" - + chimera_ckpt = ctx.ckpts["chimera"] seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.csv") - boot_mode = get(ctx.cfg["output"], "bootstraps", "combined") - comb_mode = get(ctx.cfg["output"], "combined_mode", "regular") + asv_file = get(ctx.cfg["output"], "asv_filename", "asv_counts.csv") tables_dir = ctx.dirs["Tables"] + checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") + multithread = get(ctx.cfg["taxonomy"], "multithread", 4) + min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) + tax_levels = ctx.cfg["taxonomy"]["levels"] - R"combined_input <- index" + remote_cfg = get(get(ctx.cfg, "taxonomy", Dict()), "remote", nothing) + use_remote = !isnothing(remote_cfg) && + !isnothing(get(remote_cfg, "host", nothing)) - if !get(ctx.cfg["taxonomy"], "skip", false) - emit("Assigning taxonomy") - multithread = get(ctx.cfg["taxonomy"], "multithread", 4) - min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) - tax_levels = ctx.cfg["taxonomy"]["levels"] - - db_path = isnothing(taxonomy_db) ? - _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db + # Skip local DB resolution when remote has its own db_path configured. + skip_local_db = use_remote && !isnothing(get(remote_cfg, "db_path", nothing)) + db_path = skip_local_db ? nothing : + isnothing(taxonomy_db) ? _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db + if use_remote + emit("Assigning taxonomy (remote: $(remote_cfg["host"]))") + _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, + checkpoint, taxa_prefix, multithread, + min_boot, tax_levels, verbose, remote_cfg) + R"load($checkpoint)" + else + emit("Assigning taxonomy") + R"load($chimera_ckpt)" R""" taxa_result <- run_assign_taxonomy( seq_table_nochim, $db_path, list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), $verbose) taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, - $tables_dir, $taxa_prefix, $boot_mode) + $tables_dir, $taxa_prefix) """ R"gc()" - comb_mode == "regular" && R"combined_input <- taxa_df" - else - emit("Skipping taxonomy (taxonomy.skip = true)") + R"save(seq_table_nochim, index, taxa_df, file=$checkpoint)" end - checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") - R"save(seq_table_nochim, index, combined_input, file=$checkpoint)" emit("Checkpoint: $checkpoint") - - R"write_combined_table(combined_input, seq_table_nochim, $tables_dir, $combined_file)" + R"write_combined_table(taxa_df, index, seq_table_nochim, $tables_dir, $combined_file, $asv_file)" emit("Pipeline complete. Outputs:") emit(" $(joinpath(tables_dir, seq_prefix * ".csv"))") emit(" $(joinpath(tables_dir, fasta_prefix * ".fasta"))") emit(" $(joinpath(tables_dir, fasta_prefix * ".csv"))") emit(" $(joinpath(tables_dir, taxa_prefix * ".csv"))") + emit(" $(joinpath(tables_dir, taxa_prefix * "_bootstraps.csv"))") + emit(" $(joinpath(tables_dir, taxa_prefix * "_combined.csv"))") emit(" $(joinpath(tables_dir, combined_file))") + emit(" $(joinpath(tables_dir, asv_file))") emit(" $(joinpath(tables_dir, "pipeline_stats.csv"))") emit(" $checkpoint") nothing diff --git a/src/dada2/taxonomy_remote.r b/src/dada2/taxonomy_remote.r new file mode 100644 index 0000000..7b79820 --- /dev/null +++ b/src/dada2/taxonomy_remote.r @@ -0,0 +1,50 @@ +#!/usr/bin/env Rscript +# Standalone taxonomy assignment for remote execution. +# Called by _assign_taxonomy_remote() in taxonomy.jl via SSH. +# All paths refer to the remote filesystem. +# +# Arguments (key=value): +# functions path to dada2_functions.r +# ckpt path to ckpt_chimera.RData +# db path to taxonomy database +# tables output directory for taxonomy CSV files +# save path to write checkpoint.RData +# prefix taxonomy output prefix (e.g. "taxonomy") +# multithread number of threads +# min_boot minimum bootstrap threshold +# levels comma-separated taxonomy level names +# verbose true|false + +args <- commandArgs(trailingOnly = TRUE) +p <- list() +for (a in args) { + kv <- strsplit(a, "=", fixed = TRUE)[[1]] + if (length(kv) >= 2) p[[kv[1]]] <- paste(kv[-1], collapse = "=") +} + +required <- c("functions", "ckpt", "db", "tables", "save", + "prefix", "multithread", "min_boot", "levels") +missing_args <- setdiff(required, names(p)) +if (length(missing_args) > 0) + stop("Missing required arguments: ", paste(missing_args, collapse = ", ")) + +source(p[["functions"]]) + +load(p[["ckpt"]]) +dir.create(p[["tables"]], recursive = TRUE, showWarnings = FALSE) + +verbose <- tolower(p[["verbose"]]) == "true" +multithread <- as.integer(p[["multithread"]]) +min_boot <- as.integer(p[["min_boot"]]) +levels <- strsplit(p[["levels"]], ",", fixed = TRUE)[[1]] + +taxa_result <- run_assign_taxonomy( + seq_table_nochim, p[["db"]], + list(multithread = multithread, min_boot = min_boot, levels = levels), + verbose +) +taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, + p[["tables"]], p[["prefix"]]) + +save(seq_table_nochim, index, taxa_df, file = p[["save"]]) +message("Remote taxonomy assignment complete.") diff --git a/src/main.jl b/src/main.jl index c4b27a5..846f65d 100755 --- a/src/main.jl +++ b/src/main.jl @@ -66,7 +66,7 @@ dbs = ensure_databases(dada2_config_path) #cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) -#dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) +dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) #vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) From 883eb3d6e55c29c2e2b5f404dc4883d1c9c8434b Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 23:18:26 +0100 Subject: [PATCH 024/175] I meant to stage all changes, oops. --- src/dada2.jl | 12 ++++++------ src/dada2/taxonomy.jl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/dada2.jl b/src/dada2.jl index 28365c3..f279c98 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -69,12 +69,12 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `checkpoint.RData` - final R environment snapshot """ function dada2(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) - #prefilter_qc(config_path; progress, input_dir, workspace_root) - #filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" - #learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" - #denoise(config_path; progress, input_dir, workspace_root); R"gc()" - #filter_length(config_path; progress, input_dir, workspace_root); R"gc()" - #chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" + prefilter_qc(config_path; progress, input_dir, workspace_root) + filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" + learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" + denoise(config_path; progress, input_dir, workspace_root); R"gc()" + filter_length(config_path; progress, input_dir, workspace_root); R"gc()" + chimera_removal(config_path; progress, input_dir, workspace_root); R"gc()" assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index dfaa4fd..5bd6fef 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -92,7 +92,7 @@ run(scp("$host:$remote_ckpt", checkpoint)) # Safety check before cleanup: staging_dir must be at least 3 components - # deep to guard against dangerous paths resolving to / or home root. + # deep to guard against dangerous paths. parts = filter(!isempty, split(staging_dir, '/')) if length(parts) >= 3 emit(" Cleaning up $host:$staging_dir") From 51174770f57d8cb0c0aac5aca704b938ec21c060 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 20 Feb 2026 23:44:57 +0100 Subject: [PATCH 025/175] Added MultiQC report from FastQC. --- README.md | 4 +++- src/call_tools.jl | 49 ++++++++++++++++++++++++++++++----------------- src/main.jl | 10 +++++----- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 65c8fe4..5159996 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,9 @@ SampleName_*_L001_R2_001.fastq.gz ``` output/{project_name}/ ├── cutadapt/ # Trimmed FASTQ pairs and logs -├── FastQC/ # FastQC HTML reports +├── QC/ +│ ├── fastqc/ # Per-file FastQC HTML reports +│ └── multiqc_report.html # MultiQC summary across all samples ├── dada2/ │ ├── Tables/ │ │ ├── seqtab_nochim.csv # ASV count table diff --git a/src/call_tools.jl b/src/call_tools.jl index 5eeed48..b1d07aa 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -4,7 +4,7 @@ module Tools # # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). -export cutadapt, vsearch, fastqc_all, fastqc_one +export cutadapt, vsearch, multiqc, fastqc using YAML using Logging @@ -188,42 +188,55 @@ export cutadapt, vsearch, fastqc_all, fastqc_one ) end - ## FastQC + ## FastQC / MultiQC """ - function fastqc_all(fastq_in_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = "fastqc") + multiqc(fastq_in_dir, qc_dir; fastqc_args, multiqc_args, fastqc_bin, multiqc_bin) - Requires `fastqc` installed. Runs `fastqc` command with any optional parameters on a whole set. + Run FastQC on all FASTQ files in `fastq_in_dir`, saving individual reports to + `qc_dir/fastqc/`, then aggregate them into a MultiQC summary report at `qc_dir/`. ## Arguments - - `fastq_in_dir`: Specify path of fastq files to qc. + - `fastq_in_dir`: Directory containing `.fastq` or `.fastq.gz` files. + - `qc_dir`: Parent output directory. FastQC reports go to `qc_dir/fastqc/`; + the MultiQC summary report goes to `qc_dir/`. ## Keyword Arguments - - `optional_args` (optional, default: "-t 20 --extract --delete"): Specify additional arguments passed to `fastqc` command. - - `fastqc_bin` (optional, default: "fastqc"): Path to the fastqc binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. - + - `fastqc_args` (optional, default: `"-t 20 --extract --delete"`): Additional arguments passed to `fastqc`. + - `multiqc_args` (optional, default: `""`): Additional arguments passed to `multiqc`. + - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. + - `multiqc_bin` (optional): Path to the multiqc binary. Set via `config/tools.yml`. """ - function fastqc_all(fastq_in_dir, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) + function multiqc(fastq_in_dir, qc_dir; + fastqc_args = "-t 20 --extract --delete", + multiqc_args = "", + fastqc_bin = tool_bin("fastqc"), + multiqc_bin = tool_bin("multiqc")) + fastqc_dir = joinpath(qc_dir, "fastqc") mkpath(fastqc_dir) @info "FastQC running on $fastq_in_dir" - cmd = "$fastqc_bin $fastq_in_dir/*.fastq* -o $fastqc_dir $optional_args" - run(`bash -lc $cmd`) + fqc_cmd = "$fastqc_bin $fastq_in_dir/*.fastq* -o $fastqc_dir $fastqc_args" + run(`bash -lc $fqc_cmd`) @info "FastQC complete. Output: $fastqc_dir" + @info "MultiQC running on $fastqc_dir" + mqc_cmd = "$multiqc_bin $fastqc_dir -o $qc_dir $multiqc_args" + run(`bash -lc $mqc_cmd`) + @info "MultiQC complete. Output: $qc_dir" end """ - function fastqc_one(fastq_in_file; optional_args = "-t 20 --extract --delete", fastqc_bin = "fastqc") + fastqc(fastq_in_file, fastqc_dir; optional_args, fastqc_bin) - Requires `fastqc` installed. Runs `fastqc` command with any optional parameters on a whole set. + Run FastQC on a single FASTQ file. No MultiQC report is generated. ## Arguments - - `fastq_in_file`: Specify path of fastq file to qc. + - `fastq_in_file`: Path to a single `.fastq` or `.fastq.gz` file. + - `fastqc_dir`: Output directory for the FastQC report. ## Keyword Arguments - - `optional_args` (optional, default: "-t 20 --extract --delete"): Specify additional arguments passed to `fastqc` command. - - `fastqc_bin` (optional, default: "fastqc"): Path to the fastqc binary. Defaults to PATH lookup. Set via `config/tools.yml` and `load_tools()` in main.jl. - + - `optional_args` (optional, default: `"-t 20 --extract --delete"`): Additional arguments passed to `fastqc`. + - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. """ - function fastqc_one(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) + function fastqc(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) mkpath(fastqc_dir) @info "FastQC running on $fastq_in_file" cmd = "$fastqc_bin $fastq_in_file -o $fastqc_dir $optional_args" diff --git a/src/main.jl b/src/main.jl index 846f65d..bf28cd8 100755 --- a/src/main.jl +++ b/src/main.jl @@ -22,7 +22,7 @@ project_dir = joinpath(output_dir, project_name) fastq_input_dir = joinpath(data_dir, project_name) # QC paths -fastqc_dir = joinpath(project_dir, "FastQC") +qc_dir = joinpath(project_dir, "QC") # Cutadapt paths trimmed_dir = joinpath(project_dir, "cutadapt") @@ -62,13 +62,13 @@ protist_filter = joinpath(config_dir, "protist_filter.yml") dbs = ensure_databases(dada2_config_path) ## Main -#fastqc_all(fastq_input_dir, fastqc_dir) +multiqc(fastq_input_dir, qc_dir) #cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) -dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) +#dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) #vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) -CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)); @info "Written: $merged_outfile_multi" -CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)); @info "Written: $filtered_outfile_multi" +#CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)); @info "Written: $merged_outfile_multi" +#CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)); @info "Written: $filtered_outfile_multi" From d8657cb8ee7932b0f35d1956be9640db222e8a83 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 21 Feb 2026 18:18:26 +0100 Subject: [PATCH 026/175] Overhauled execution pipeline with wiretypes and mtime based skipping of previously run stages. Created a versatile project structure. --- .gitignore | 7 +- README.md | 252 +++++++++++++----- config/cdhit.yml | 1 + config/cutadapt.yml | 6 + config/defaults/cdhit.yml | 1 + config/defaults/cutadapt.yml | 5 + .../{dada2.example.yml => defaults/dada2.yml} | 16 +- config/defaults/databases.yml | 12 + config/defaults/merge_taxa.yml | 6 + .../{tools.example.yml => defaults/tools.yml} | 0 config/defaults/vsearch.yml | 1 + config/{ => filters}/protist_filter.yml | 0 config/merge_taxa.yml | 3 + config/vsearch.yml | 1 + install.jl | 11 +- src/call_tools.jl | 188 +++++++++++-- src/dada2.jl | 31 +++ src/dada2/chimera.jl | 104 +++++--- src/dada2/context.jl | 49 ++-- src/dada2/denoise.jl | 201 +++++++++----- src/dada2/qc.jl | 128 ++++++--- src/dada2/taxonomy.jl | 74 +++-- src/databases.jl | 108 ++++---- src/main.jl | 74 ++--- src/merge_and_filter_taxa.jl | 60 ++++- src/project.jl | 125 +++++++++ src/types.jl | 31 +++ 27 files changed, 1079 insertions(+), 416 deletions(-) create mode 100644 config/cdhit.yml create mode 100644 config/cutadapt.yml create mode 100644 config/defaults/cdhit.yml create mode 100644 config/defaults/cutadapt.yml rename config/{dada2.example.yml => defaults/dada2.yml} (73%) create mode 100644 config/defaults/databases.yml create mode 100644 config/defaults/merge_taxa.yml rename config/{tools.example.yml => defaults/tools.yml} (100%) create mode 100644 config/defaults/vsearch.yml rename config/{ => filters}/protist_filter.yml (100%) create mode 100644 config/merge_taxa.yml create mode 100644 config/vsearch.yml create mode 100644 src/project.jl create mode 100644 src/types.jl diff --git a/.gitignore b/.gitignore index 34d42c9..509eb21 100644 --- a/.gitignore +++ b/.gitignore @@ -254,7 +254,6 @@ rsconnect/ # Things with ambiguous intellectual property internal data/ -cutadapt/output_* ### Lock files .~lock.* @@ -265,8 +264,10 @@ bin/ databases/ config/tools.yml config/dada2.yml +config/databases.yml ### Bits I don't want to share yet hide.* -output/ -pipelinesteps.txt \ No newline at end of file +projects/ +pipelinesteps.txt +archive/ \ No newline at end of file diff --git a/README.md b/README.md index 5159996..08a9236 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ # MetaManifold +[![License: AGPL-3.0](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](LICENSE) +[![Julia ≥ 1.0](https://img.shields.io/badge/Julia-%E2%89%A51.0-9558B2?logo=julia)](https://julialang.org) +[![R ≥ 4.0](https://img.shields.io/badge/R-%E2%89%A54.0-276DC3?logo=r)](https://www.r-project.org) + A Julia pipeline for amplicon metabarcoding from raw paired-end Illumina reads to filtered, taxonomy-annotated ASV tables. ## Overview -This project aims to wrap a standard amplicon sequencing workflow into a single, configurable project. It handles multiplex primer trimming amplicon denoising, taxonomy assignment, and taxonomic filtering. +This project aims to wrap a standard amplicon sequencing workflow into a single, configurable project. It handles multiplex primer trimming, amplicon denoising, taxonomy assignment, and taxonomic filtering. **Pipeline stages** @@ -15,12 +19,12 @@ This project aims to wrap a standard amplicon sequencing workflow into a single, | Amplicon denoising | DADA2 (R) | ASV count table, FASTA, taxonomy | | Taxonomy assignment | vsearch | Per-ASV taxonomy TSV | | Clustering | cd-hit-est | Clustered ASV representatives | -| Merge + filter | Julia | Combined taxonomy/count table, protist-filtered output | +| Merge + filter | Julia | Combined taxonomy/count table, filtered output | ## Prerequisites - **Julia** ≥ 1.0 - installed automatically by `install.sh` if missing -- **R** ≥ 4.0 - must be installed before running `install.sh` +- **R** ≥ 4.0 - required for the DADA2 stage - Ubuntu/Debian: `sudo apt install r-base` - macOS: `brew install r` or [CRAN package](https://cran.r-project.org/bin/macosx/) @@ -49,22 +53,62 @@ vsearch: ``` When a remote SSH path is set, the pipeline routes that tool's invocations through -`ssh`. `config/tools.example.yml` contains full config format. +`ssh`. `config/defaults/tools.yml` contains the full config format. + +### Creating a new project + +```julia +projects = new_project("MyProject") +# Bootstraps projects/MyProject/ mirroring data/MyProject/ subdirectory structure - ensure "MyProject" name matches folder in data/. +# Creates dada2.yml, cutadapt.yml, vsearch.yml, cdhit.yml, merge_taxa.yml at each level. +``` + +Re-running `new_project` never overwrites existing configs. To reset a level to its parent's +settings, delete that config file and re-run. + +Or manually: +```bash +mkdir -p projects/MyProject +cp config/defaults/dada2.yml projects/MyProject/dada2.yml +cp config/defaults/cutadapt.yml projects/MyProject/cutadapt.yml +# etc. +``` ## Configuration -All configuration lives in `config/`: +Global configuration lives in `config/`; per-project pipeline configs live in `projects/{name}/`. When per-run configs are missing, they are automatically populated with per-project configs, which are populated by global config if missing: | File | Purpose | |------|---------| -| `primers.yml` | Primer sequences and pair definitions | -| `dada2.yml` | DADA2 pipeline parameters (truncation, error model, taxonomy DB, etc.) | -| `protist_filter.yml` | Taxonomic filtering rules for protist output | -| `tools.yml` | Tool paths (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est) | +| `config/databases.yml` | Database URIs and optional local paths | +| `config/primers.yml` | Primer sequences and pair definitions | +| `config/filters/` | Directory of taxonomic filter configs | +| `config/tools.yml` | Tool binary paths (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est) | +| `projects/{name}/cutadapt.yml` | Primer pair selection and cutadapt optional args | +| `projects/{name}/dada2.yml` | DADA2 pipeline parameters | +| `projects/{name}/vsearch.yml` | vsearch alignment thresholds | +| `projects/{name}/cdhit.yml` | cd-hit-est clustering threshold (optional stage) | +| `projects/{name}/merge_taxa.yml` | Which filter configs to apply in the merge step | + +### Configuring databases (`config/databases.yml`) + +This is the single place to manage DB URI's shared across all projects. + +```yaml +databases: + dir: "./databases" + pr2: + dada2: + uri: "https://..." # DADA2-format FASTA (downloaded on first use) + local: ~ # set to a local path to skip download + vsearch: + uri: "https://..." # vsearch-format FASTA + local: ~ +``` ### Defining primer pairs `primers.yml` - Maps primer names to sequences and defines which forward/reverse sequences constitute a pair: +Maps primer names to sequences and defines which forward/reverse sequences constitute a pair: ```yaml Forward: @@ -83,19 +127,28 @@ Pairs: - Primer2R ``` -This allows you to referenece one or multiple primer pairs within the pipeline by name. This allows you to store all primer pairs you often work on in one place and then refer to them in whichever combinations you need as and when. Shared primers across pairs (same forward in two pairs, as the above example) are automatically deduplicated in the cutadapt invocation since otherwise it complains a bit. - -### Configuring DADA2 (`dada2.yml`) +Store all primer pairs in here and reference whichever combinations you need per project. Shared primers across pairs (same forward primer in two pairs) are automatically deduplicated in the `cutadapt` invocation since otherwise it complains a bit. If you need duplicates, you must create the same sequence under a different name. -> **Performance tip:** For large datasets and reference databases, consider installing the [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides acceleration for CPU and Nvidia CUDA GPUs for taxonomy assignment, with no changes to the API or configuration required. This fork is experimental, so if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. It was too experimental for me, so opted to offload assignTaxonomy() to server in dada2.yml... +### Configuring cutadapt (`projects/{name}/cutadapt.yml`) -Copy `config/dada2.example.yml` to `config/dada2.yml` and edit it. It contains your local paths and optionally remote server credentials and should never be shared. +Selects which primer pairs to apply and passes additional arguments to cutadapt. Inherits from `config/cutadapt.yml` if no per-project file exists. ```yaml -workspace: - root: "./output/dada2/" # output directories are created here - input_dir: "./output/cutadapt/" # trimmed FASTQ input (cutadapt output) +# Names must match keys in the Pairs section of config/primers.yml. +primer_pairs: + - PrimerPair1 + - PrimerPair2 + +optional_args: "-m 200 --discard-untrimmed" +``` + +`optional_args` is passed verbatim to cutadapt after the primer arguments. Here, the `-m` flag sets the minimum read length after trimming; `--discard-untrimmed` drops reads with no primer match. + +### Configuring DADA2 (`projects/{name}/dada2.yml`) +> **Performance tip:** For large datasets and reference databases, consider installing my [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides CPU and Nvidia CUDA GPU acceleration for taxonomy assignment with no API or config changes required. This fork is experimental - if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. Ultimately, I had to offload `assignTaxonomy()` to a remote server (`taxonomy.remote` setting). + +```yaml file_patterns: forward: "_R1_trimmed.fastq.gz" reverse: "_R2_trimmed.fastq.gz" @@ -106,7 +159,7 @@ file_patterns: # Filter and trim - DADA2's filterAndTrim(): filter_trim: trunc_q: 2 - trunc_len: [220, 220] # [forward, reverse]; first value used for single-end + trunc_len: [220, 220] # [forward, reverse]; first value used for single-end mode max_ee: [3, 3] # maximum expected errors in F and R reads min_len: 175 max_n: 0 @@ -132,21 +185,10 @@ asv: band_size_max: 430 denovo_method: "consensus" # consensus | pooled | per-sample -# Reference databases - downloaded and cached on first use: -databases: - dir: "./databases" - pr2: - dada2: - uri: "https://..." # DADA2-format FASTA - local: ~ # set to a local path to skip download - vsearch: - uri: "https://..." # vsearch-format FASTA - local: ~ - # Taxonomy - assignTaxonomy() against the configured database: taxonomy: - database: pr2 # key into databases: section above - multithread: 4 # threads for assignTaxonomy(); higher = more RAM + database: pr2 # key into config/databases.yml + multithread: true # threads for assignTaxonomy() min_boot: 0 # minimum bootstrap confidence to retain (0-100) levels: - "Domain" @@ -162,14 +204,14 @@ taxonomy: # Optional: offload the memory-intensive assignTaxonomy() step to a remote # server via SSH. Omit or set host to null to run locally. # DISCLAIMER: You are solely responsible for ensuring you have authorisation - # to use the configured host. See dada2.example.yml for the full disclaimer. + # to use the configured host. See config/defaults/dada2.yml for the full disclaimer. remote: host: ~ # user@hostname rscript: "Rscript" # path to Rscript on the server staging_dir: "/absolute/path/on/server" db_path: ~ # absolute path to database on server (null = transfer local copy) -# Output filenames (all written to workspace.root/Tables/): +# Output filenames (all written to workspace_root/Tables/): output: seq_table_prefix: "seqtab_nochim" fasta_prefix: "asvs" @@ -178,12 +220,12 @@ output: asv_filename: "asv_counts.csv" ``` -**Outputs written to `workspace.root/Tables/`:** +**Outputs written to `projects/{name}/{run}/dada2/Tables/`:** | File | Contents | |------|----------| -| `seqtab_nochim.csv` | Chimera-free ASV count table (samples × ASVs) | -| `asvs.fasta` / `asvs.csv` | ASV sequences with short identifiers (seq1, seq2, …) | +| `seqtab_nochim.csv` | Chimera-free ASV count table (samples x ASVs) | +| `asvs.fasta` / `asvs.csv` | ASV sequences with short identifiers (seq1, seq2, ...) | | `taxonomy.csv` | Taxonomy assignments per ASV | | `taxonomy_bootstraps.csv` | Bootstrap confidence values per rank | | `taxonomy_combined.csv` | Taxonomy + bootstrap columns combined | @@ -191,90 +233,149 @@ output: | `asv_counts.csv` | ASV sequences + per-sample counts (no taxonomy) | | `pipeline_stats.csv` | Read counts retained at each pipeline stage | -### Configuring taxonomic filtering (`protist_filter.yml`) +### Configuring vsearch (`projects/{name}/vsearch.yml`) -The filter file controls the `filter_table()` step, which removes non-target taxa -and remaps Supergroup labels for consistency with PR2 division names. +Controls the alignment thresholds used when assigning taxonomy against the reference database. -> **Database compatibility:** The default config is tuned specifically for [PR2](https://pr2-database.org/). Column names (`Supergroup`, `Division`, `Subdivision`) and pattern strings (`Eukaryota:plas`, `TSAR:chro`, etc.) reflect PR2's rank structure and nomenclature. If you use a different reference database, you will need to update both the column names here and the `levels` list in `dada2.yml` to match that database's ranks. +```yaml +optional_args: "--id 0.75 --query_cov 0.8" +``` -**Division → Supergroup remapping** +`optional_args` is passed verbatim to `vsearch --usearch_global`. Key thresholds: +- `--id` - minimum sequence identity (0-1); lower values recover more hits at the cost of specificity +- `--query_cov` - minimum fraction of the query that must be aligned; filters partial matches -PR2 assigns a `Supergroup` label that can be coarser than `Division` for some lineages. The `mappings` block overrides `Supergroup` with the `Division` value where they should be treated as equivalent: +### Configuring cd-hit-est (`projects/{name}/cdhit.yml`) + +Clustering step that collapses near-identical ASVs before taxonomy assignment. ```yaml -mappings: - Rhizaria: Rhizaria - Alveolata: Alveolata - Stramenopiles: Stramenopiles - # add further Division: Supergroup pairs as needed +optional_args: "-c 0.9" ``` -**Exclusion filters** +`optional_args` is passed verbatim to `cd-hit-est`. `-c` sets the sequence identity threshold for clustering (default 0.9 = 90%). + +### Configuring merge_taxa (`projects/{name}/merge_taxa.yml`) -Each entry specifies a column and a substring. Rows where that column contains the substring (partial match) are removed: +Controls which filter configs are applied when merging taxonomy and count tables. `merged.csv` (unfiltered) is always written; each entry in `filters` produces an additional filtered CSV. ```yaml +filters: + - "protist_filter.yml" # -> merged/protist_filter.csv +``` + +Each entry is a filename relative to `config/filters/`. Remove all entries (or set `filters: []`) to produce only the unfiltered `merged.csv`. + +### Configuring taxonomic filtering (`config/filters/protist_filter.yml`) + +The filter file controls the `filter_table()` step, which removes non-target taxa and remaps Supergroup labels for consistency with PR2 division names. + +Place filter configs in `config/filters/` and reference them by filename in `merge_taxa.yml`. The default `protist_filter.yml` targets eukaryotic protists from PR2-annotated data: + +```yaml +# Division -> Supergroup remapping. +# This block overrides Supergroup with the Division value where they should be equivalent. +mappings: + Rhizaria: Rhizaria + Alveolata: Alveolata + Stramenopiles: Stramenopiles + Hemimastigophora: Hemimastigophora + Discoba: Discoba + Metamonada: Metamonada + Telonemia: Telonemia + Ancyromonadida: Ancyromonadida + +# Exclusion filters: rows where the named column contains the pattern are removed. filters: - column: Domain pattern: Bacteria + - column: Domain + pattern: Archaea - column: Domain pattern: Eukaryota:plas - column: Domain pattern: Eukaryota:mito + - column: Supergroup + pattern: TSAR:chro - column: Subdivision pattern: Metazoa - column: Subdivision pattern: Fungi - # add further column/pattern pairs to exclude additional lineages + - column: Division + pattern: Rhodophyta + - column: Class + pattern: Embryophyceae + +# Remove rows with an empty or unassigned Domain field. +remove_empty_domain: true ``` +> **Database compatibility:** Column names and patterns above are tuned for [PR2](https://pr2-database.org/). If you use a different reference database, update the column names to match that database's rank structure and adjust the `levels` list in `dada2.yml` accordingly. + ## Usage -Edit `src/main.jl` to set your parameters, then uncomment or add pipeline steps you want to run, e.g.: +Place FASTQ files in `data/MyProject`. If you want to run multiple sets of FASTQ files, you can use subdirectories like `data/MyProject/Primer1`, `data/MyProject/Primer2`. `new_project()` will automatically generate a matching project directory structure, nothing in `data/` is ever overwritten. Edit `src/main.jl` to set your project name and run: ```julia -# Remove primers from Illumina reads -cutadapt(primer_pairs, primers_config, fastq_input_dir, - cutadapt_dir, optional_args = optional_args) +dbs = ensure_databases(databases_config) +projects = new_project("MyProject") -# Run dada2 denoising, chimera removal, etc. in accordance with config -dada2(dada2_config) +const r_lock = ReentrantLock() -# Use vsearch local alignment to create a taxonomy table -vsearch(fasta) +Threads.@threads for project in projects + multiqc(project.data_dir, joinpath(project.dir, "QC")) -# Merge vsearch and idtax tables by ASV -merged = merge_taxonomy_counts(vsearch_tsv, dada2_csv) + trimmed = cutadapt(project) -# Output table of all ID'd reads and table of reads for protists -CSV.write(merged_outfile_name, merged) -CSV.write(filtered_outfile_name, filter_table(merged, protist_filter_path)) + asvs = lock(r_lock) do + dada2(project, trimmed, taxonomy_db = dbs["pr2_dada2"]) + end + # asvs = lock(r_lock) do; cdhit(project, asvs); end # optional clustering + + tax = vsearch(project, asvs, dbs["pr2_vsearch"]) + merged = merge_taxa(project, asvs, tax) +end ``` +Each stage returns a wire type (`TrimmedReads`, `ASVResult`, `TaxonomyHits`, `MergedTables`) and skips automatically if outputs are already up to date relative to their inputs (mtime-based). This means that rerunning pipeline after config change will only perform the minimum necessary steps to output. + Run from the project root: ```bash julia --project=. src/main.jl ``` +Or with threads: + +```bash +julia -t 2 --project=. src/main.jl +``` + ### Input data -Place paired-end FASTQ files in `data/fastq/` following Illumina naming: +Place paired-end FASTQ files under `data/{project_name}/` following Illumina naming: ``` -SampleName_*_L001_R1_001.fastq.gz -SampleName_*_L001_R2_001.fastq.gz +data/MyProject/SampleName_*_L001_R1_001.fastq.gz +data/MyProject/SampleName_*_L001_R2_001.fastq.gz ``` +For multi-run projects, nest runs in subdirectories - `new_project` will detect any directory containing `.fastq.gz` files as a leaf run and create a matching project directory under `projects/{project_name}/`. + ### Output structure +All outputs for a given run live under `projects/{project_name}/{run}/`: + ``` -output/{project_name}/ +projects/{project_name}/{run}/ ├── cutadapt/ # Trimmed FASTQ pairs and logs +│ └── logs/ +│ ├── cutadapt_primer_trimming_stats.txt +│ └── cutadapt_trimmed_percentage.txt ├── QC/ │ ├── fastqc/ # Per-file FastQC HTML reports -│ └── multiqc_report.html # MultiQC summary across all samples +│ ├── multiqc_report.html # MultiQC summary across all samples +│ └── logs/ ├── dada2/ │ ├── Tables/ │ │ ├── seqtab_nochim.csv # ASV count table @@ -287,12 +388,17 @@ output/{project_name}/ │ │ ├── asv_counts.csv # Sequences + per-sample counts │ │ └── pipeline_stats.csv │ ├── Figures/ # Quality profile and error rate PDFs -│ └── Checkpoints/ # RData checkpoints for stage resumption +│ ├── Checkpoints/ # RData checkpoints for stage resumption +│ └── Logs/ # Per-stage R logs +├── cdhit/ +│ ├── asvs.fasta # Clustered ASV sequences +│ └── asvs.fasta.clstr # Cluster membership file ├── vsearch/ -│ └── taxonomy.tsv +│ ├── taxonomy.tsv +│ └── logs/ └── merged/ - ├── merged_multi.csv # Merged DADA2 + vsearch taxonomy + counts - └── protist_filtered_multi.csv + ├── merged.csv # Merged vsearch taxonomy + ASV counts (all taxa) + └── protist_filter.csv # Filtered subset (one file per entry in merge_taxa.yml) ``` ## Third-party tools diff --git a/config/cdhit.yml b/config/cdhit.yml new file mode 100644 index 0000000..157b0b3 --- /dev/null +++ b/config/cdhit.yml @@ -0,0 +1 @@ +optional_args: "-c 0.9" diff --git a/config/cutadapt.yml b/config/cutadapt.yml new file mode 100644 index 0000000..b3a1342 --- /dev/null +++ b/config/cutadapt.yml @@ -0,0 +1,6 @@ +# Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. +primer_pairs: + - TarEuk + - Meta2 + +optional_args: "-m 200 --discard-untrimmed" diff --git a/config/defaults/cdhit.yml b/config/defaults/cdhit.yml new file mode 100644 index 0000000..157b0b3 --- /dev/null +++ b/config/defaults/cdhit.yml @@ -0,0 +1 @@ +optional_args: "-c 0.9" diff --git a/config/defaults/cutadapt.yml b/config/defaults/cutadapt.yml new file mode 100644 index 0000000..993e831 --- /dev/null +++ b/config/defaults/cutadapt.yml @@ -0,0 +1,5 @@ +# Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. +primer_pairs: + - PrimerPair1 + +optional_args: "-m 200 --discard-untrimmed" diff --git a/config/dada2.example.yml b/config/defaults/dada2.yml similarity index 73% rename from config/dada2.example.yml rename to config/defaults/dada2.yml index 18a32da..8042504 100644 --- a/config/dada2.example.yml +++ b/config/defaults/dada2.yml @@ -1,7 +1,3 @@ -workspace: - root: "./output/dada2/" - input_dir: "./output/cutadapt/" # path to trimmed FASTQ directory - file_patterns: forward: "_R1_trimmed.fastq.gz" reverse: "_R2_trimmed.fastq.gz" @@ -34,18 +30,8 @@ asv: band_size_max: 430 denovo_method: "consensus" -databases: - dir: "./databases" # shared cache directory; absolute or relative to working directory - pr2: - dada2: - uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" - local: ~ # set to a local path to skip download - vsearch: - uri: "https://github.com/pr2database/pr2database/releases/download/v5.1.0.0/pr2_version_5.1.0_SSU_taxo_long.fasta.gz" - local: ~ # set to a local path to skip download - taxonomy: - database: pr2 # key into databases: section above + database: pr2 # key into config/databases.yml multithread: 4 # higher values increase memory use significantly (mclapply forks one process per thread) # DISCLAIMER: Configuring a remote host causes this pipeline to connect via SSH, # transfer files, execute Rscript, and delete the staging directory on the remote diff --git a/config/defaults/databases.yml b/config/defaults/databases.yml new file mode 100644 index 0000000..50f9470 --- /dev/null +++ b/config/defaults/databases.yml @@ -0,0 +1,12 @@ +# Shared database configuration for all projects. +# Copy to config/databases.yml and set local: paths for pre-downloaded files. + +databases: + dir: "./databases" # shared cache directory; absolute or relative to working directory + pr2: + dada2: + uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" + local: ~ # set to a local path to skip download + vsearch: + uri: "https://github.com/pr2database/pr2database/releases/download/v5.1.0.0/pr2_version_5.1.0_SSU_taxo_long.fasta.gz" + local: ~ # set to a local path to skip download diff --git a/config/defaults/merge_taxa.yml b/config/defaults/merge_taxa.yml new file mode 100644 index 0000000..7d4cd72 --- /dev/null +++ b/config/defaults/merge_taxa.yml @@ -0,0 +1,6 @@ +# List of filter configs to apply, each relative to config/filters/. +# Each entry produces a separate filtered CSV in the project's merged/ directory. +# The unfiltered merged.csv is always written regardless of this list. +# Set to [] or omit entries to produce merged.csv only. +filters: + - "protist_filter.yml" diff --git a/config/tools.example.yml b/config/defaults/tools.yml similarity index 100% rename from config/tools.example.yml rename to config/defaults/tools.yml diff --git a/config/defaults/vsearch.yml b/config/defaults/vsearch.yml new file mode 100644 index 0000000..e31b1d7 --- /dev/null +++ b/config/defaults/vsearch.yml @@ -0,0 +1 @@ +optional_args: "--id 0.75 --query_cov 0.8" diff --git a/config/protist_filter.yml b/config/filters/protist_filter.yml similarity index 100% rename from config/protist_filter.yml rename to config/filters/protist_filter.yml diff --git a/config/merge_taxa.yml b/config/merge_taxa.yml new file mode 100644 index 0000000..72821b7 --- /dev/null +++ b/config/merge_taxa.yml @@ -0,0 +1,3 @@ +# Filename of filter config relative to config/filters/. +# Leave empty or omit to produce merged.csv only (no filtering). +filter: "protist_filter.yml" diff --git a/config/vsearch.yml b/config/vsearch.yml new file mode 100644 index 0000000..e31b1d7 --- /dev/null +++ b/config/vsearch.yml @@ -0,0 +1 @@ +optional_args: "--id 0.75 --query_cov 0.8" diff --git a/install.jl b/install.jl index 47e23d8..d98830e 100644 --- a/install.jl +++ b/install.jl @@ -184,6 +184,7 @@ function download_vsearch()::String dest end + function download_fastqc()::String # Pin to a known-good version; update periodically. version = "0.12.1" @@ -335,7 +336,7 @@ function resolve_tool( install_fn::Union{Function,Nothing} = nothing )::Union{String,Nothing} println() - println(" ─── $display_name ─────────────────────────────────────────") + println(" --- $display_name -------------------------------------------------") bin = bin_name(key) @@ -401,9 +402,9 @@ end ## Main function main() println() - println("╔═══════════════════════════════════════════╗") - println("║ MetabarcodingPipeline — Install Script ║") - println("╚═══════════════════════════════════════════╝") + println("+-------------------------------------------+") + println("| MetabarcodingPipeline - Install Script |") + println("+-------------------------------------------+") UPDATE_MODE && println(" Mode: UPDATE") println() @@ -437,7 +438,7 @@ function main() # R packages println() - println(" ─── R packages ─────────────────────────────────────────────") + println(" --- R packages -----------------------------------------------------") r_packages = ["dada2", "tidyverse"] if prompt_yn(" Install/check R packages (dada2, tidyverse)?") install_r_packages(r_packages) diff --git a/src/call_tools.jl b/src/call_tools.jl index b1d07aa..1c662e3 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -4,10 +4,24 @@ module Tools # # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). -export cutadapt, vsearch, multiqc, fastqc +export cutadapt, vsearch, multiqc, fastqc, cdhit using YAML using Logging + using ..PipelineTypes + + # Run cmd_str via bash, capturing stdout+stderr to log_path. + # On failure, prints the log to stderr before rethrowing so the error is visible. + function _run_logged(cmd_str::String, log_path::String; mode::String="w") + try + open(log_path, mode) do io + run(pipeline(`bash -lc $cmd_str`; stdout=io, stderr=io)) + end + catch e + isfile(log_path) && print(stderr, read(log_path, String)) + rethrow() + end + end ## Tools loading from config """ @@ -135,9 +149,10 @@ export cutadapt, vsearch, multiqc, fastqc end """ - function cutadapt(primer_pairs, primers_path, fastq_in_dir, fastq_out_dir, optional_args) - - Requires `cutadapt` installed. Runs `cutadapt` command with as many primer pairs as necessary (useful for multiplex) and any optional parameters. Primer aguments are determined from YAML file in format: + cutadapt(primer_pairs, primers_path, fastq_in_dir, cutadapt_dir; optional_args, cutadapt_bin) + + Requires `cutadapt` installed. Runs `cutadapt` with as many primer pairs as necessary + (useful for multiplex) and any optional parameters. Primer arguments are determined from YAML file in format: ``` YAML Forward: PrimerF: "CCAGCASCYGCGGTAATTCC" @@ -154,7 +169,7 @@ export cutadapt, vsearch, multiqc, fastqc - PrimerF - Primer2R ``` - Where a primer pair contains the same forward or reverse as another specified pair, these are deduplicated by name rather than sequence. For example: + Where a primer pair contains the same forward or reverse as another specified pair, these are deduplicated by name rather than sequence. For example: ```julia primer_pairs = ["PrimerPair1", "PrimerPair2"] ``` @@ -179,13 +194,22 @@ export cutadapt, vsearch, multiqc, fastqc optional_args = "-m 200 --discard-untrimmed", cutadapt_bin = tool_bin("cutadapt") ) - return run_cutadapt( + if isdir(cutadapt_dir) + trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) + if !isempty(trimmed) && + all(f -> mtime(joinpath(cutadapt_dir, f)) > mtime(primers_path), trimmed) + @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" + return TrimmedReads(cutadapt_dir) + end + end + run_cutadapt( get_primer_args(primer_pairs, primers_path), optional_args, fastq_in_dir, cutadapt_dir, cutadapt_bin ) + return TrimmedReads(cutadapt_dir) end ## FastQC / MultiQC @@ -201,26 +225,42 @@ export cutadapt, vsearch, multiqc, fastqc the MultiQC summary report goes to `qc_dir/`. ## Keyword Arguments - - `fastqc_args` (optional, default: `"-t 20 --extract --delete"`): Additional arguments passed to `fastqc`. + - `fastqc_args` (optional, default: `"-t 20 --extract --delete --force"`): Additional arguments passed to `fastqc`. - `multiqc_args` (optional, default: `""`): Additional arguments passed to `multiqc`. - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. - `multiqc_bin` (optional): Path to the multiqc binary. Set via `config/tools.yml`. """ function multiqc(fastq_in_dir, qc_dir; - fastqc_args = "-t 20 --extract --delete", + fastqc_args = "--force --verbose", multiqc_args = "", fastqc_bin = tool_bin("fastqc"), multiqc_bin = tool_bin("multiqc")) + report = joinpath(qc_dir, "multiqc_report.html") + if isfile(report) + fastqs = filter(f -> occursin(r"\.fastq(\.gz)?$", f), readdir(fastq_in_dir)) + if !isempty(fastqs) && all(f -> mtime(report) > mtime(joinpath(fastq_in_dir, f)), fastqs) + @info "Skipping multiqc: $report up to date" + return + end + end + fastqc_dir = joinpath(qc_dir, "fastqc") + log_dir = joinpath(qc_dir, "logs") mkpath(fastqc_dir) + mkpath(log_dir) + + fastqc_log = joinpath(log_dir, "fastqc.log") + multiqc_log = joinpath(log_dir, "multiqc.log") + @info "FastQC running on $fastq_in_dir" fqc_cmd = "$fastqc_bin $fastq_in_dir/*.fastq* -o $fastqc_dir $fastqc_args" - run(`bash -lc $fqc_cmd`) - @info "FastQC complete. Output: $fastqc_dir" + _run_logged(fqc_cmd, fastqc_log) + @info "FastQC complete. Output: $fastqc_dir Log: $fastqc_log" + @info "MultiQC running on $fastqc_dir" mqc_cmd = "$multiqc_bin $fastqc_dir -o $qc_dir $multiqc_args" - run(`bash -lc $mqc_cmd`) - @info "MultiQC complete. Output: $qc_dir" + _run_logged(mqc_cmd, multiqc_log) + @info "MultiQC complete. Output: $qc_dir Log: $multiqc_log" end """ @@ -237,11 +277,14 @@ export cutadapt, vsearch, multiqc, fastqc - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. """ function fastqc(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) + log_dir = joinpath(fastqc_dir, "logs") mkpath(fastqc_dir) + mkpath(log_dir) + log_path = joinpath(log_dir, "fastqc.log") @info "FastQC running on $fastq_in_file" cmd = "$fastqc_bin $fastq_in_file -o $fastqc_dir $optional_args" - run(`bash -lc $cmd`) - @info "FastQC complete. Output: $fastqc_dir" + _run_logged(cmd, log_path) + @info "FastQC complete. Output: $fastqc_dir Log: $log_path" end ## VSEARCH @@ -260,11 +303,122 @@ export cutadapt, vsearch, multiqc, fastqc """ function vsearch(fasta_in_dir, reference_database, vsearch_dir; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = tool_bin("vsearch")) + log_dir = joinpath(vsearch_dir, "logs") mkpath(vsearch_dir) - outfile = joinpath(vsearch_dir, "taxonomy.tsv") + mkpath(log_dir) + outfile = joinpath(vsearch_dir, "taxonomy.tsv") + log_path = joinpath(log_dir, "vsearch.log") @info "VSEARCH running: $fasta_in_dir against $(basename(reference_database))" cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" - run(`bash -lc $cmd`) - @info "VSEARCH complete. Output: $outfile" + _run_logged(cmd, log_path) + @info "VSEARCH complete. Output: $outfile Log: $log_path" + end + + ## cd-hit-est + """ + cdhit(fasta_in, cdhit_dir; optional_args, cdhit_bin) + + Run cd-hit-est on a FASTA file, writing the clustered output to `cdhit_dir`. + Returns the path to the output FASTA. + + ## Arguments + - `fasta_in`: Path to the input FASTA file. + - `cdhit_dir`: Output directory for the clustered FASTA and `.clstr` file. + + ## Keyword Arguments + - `optional_args` (optional, default: `"-c 0.9"`): Additional arguments passed to `cd-hit-est`. + - `cdhit_bin` (optional): Path to the cd-hit-est binary. Set via `config/tools.yml`. + """ + function cdhit(fasta_in, cdhit_dir; optional_args = "-c 0.9", cdhit_bin = tool_bin("cd_hit_est")) + log_dir = joinpath(cdhit_dir, "logs") + mkpath(cdhit_dir) + mkpath(log_dir) + fasta_out = joinpath(cdhit_dir, basename(fasta_in)) + log_path = joinpath(log_dir, "cdhit.log") + @info "cd-hit-est running on $fasta_in" + cmd = "$cdhit_bin -i $fasta_in -o $fasta_out $optional_args" + _run_logged(cmd, log_path) + @info "cd-hit-est complete. Output: $fasta_out Log: $log_path" + return fasta_out + end + + function vsearch(input::HasFasta, reference_database::String, vsearch_dir::String; + optional_args = "--id 0.75 --query_cov 0.8", + vsearch_bin = tool_bin("vsearch")) + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + if isfile(tsv) && mtime(tsv) > mtime(input.fasta) + @info "Skipping vsearch: $tsv up to date" + return TaxonomyHits(tsv) + end + vsearch(input.fasta, reference_database, vsearch_dir; + optional_args, vsearch_bin) + return TaxonomyHits(tsv) + end + + function cdhit(input::ASVResult, cdhit_dir::String; + optional_args = "-c 0.9", + cdhit_bin = tool_bin("cd_hit_est")) + new_fasta = joinpath(cdhit_dir, basename(input.fasta)) + if isfile(new_fasta) && mtime(new_fasta) > mtime(input.fasta) + @info "Skipping cdhit: $new_fasta up to date" + return ASVResult(new_fasta, input.count_table, input.taxonomy) + end + new_fasta = cdhit(input.fasta, cdhit_dir; optional_args, cdhit_bin) + return ASVResult(new_fasta, input.count_table, input.taxonomy) + end + + # ProjectCtx overloads - read parameters from per-project configs and fix output dirs. + + function cutadapt(project::ProjectCtx; + cutadapt_bin = tool_bin("cutadapt")) + config_path = joinpath(project.dir, "cutadapt.yml") + primers_path = joinpath(project.config_dir, "primers.yml") + cfg = YAML.load_file(config_path) + primer_pairs = cfg["primer_pairs"] + optional_args = get(cfg, "optional_args", "-m 200 --discard-untrimmed") + cutadapt_dir = joinpath(project.dir, "cutadapt") + if isdir(cutadapt_dir) + trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) + cfg_mtime = max(mtime(config_path), mtime(primers_path)) + if !isempty(trimmed) && all(f -> mtime(joinpath(cutadapt_dir, f)) > cfg_mtime, trimmed) + @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" + return TrimmedReads(cutadapt_dir) + end + end + run_cutadapt(get_primer_args(primer_pairs, primers_path), optional_args, + project.data_dir, cutadapt_dir, cutadapt_bin) + return TrimmedReads(cutadapt_dir) + end + + function vsearch(project::ProjectCtx, input::HasFasta, reference_database::String; + vsearch_bin = tool_bin("vsearch")) + config_path = joinpath(project.dir, "vsearch.yml") + cfg = YAML.load_file(config_path) + optional_args = get(cfg, "optional_args", "--id 0.75 --query_cov 0.8") + vsearch_dir = joinpath(project.dir, "vsearch") + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + cfg_mtime = max(mtime(config_path), mtime(input.fasta)) + if isfile(tsv) && mtime(tsv) > cfg_mtime + @info "Skipping vsearch: $tsv up to date" + return TaxonomyHits(tsv) + end + vsearch(input.fasta, reference_database, vsearch_dir; optional_args, vsearch_bin) + return TaxonomyHits(tsv) + end + + function cdhit(project::ProjectCtx, input::ASVResult; + cdhit_bin = tool_bin("cd_hit_est")) + config_path = joinpath(project.dir, "cdhit.yml") + cfg = YAML.load_file(config_path) + optional_args = get(cfg, "optional_args", "-c 0.9") + cdhit_dir = joinpath(project.dir, "cdhit") + new_fasta = joinpath(cdhit_dir, basename(input.fasta)) + cfg_mtime = max(mtime(config_path), mtime(input.fasta)) + if isfile(new_fasta) && mtime(new_fasta) > cfg_mtime + @info "Skipping cdhit: $new_fasta up to date" + return ASVResult(new_fasta, input.count_table, input.taxonomy) + end + new_fasta = cdhit(input.fasta, cdhit_dir; optional_args, cdhit_bin) + return ASVResult(new_fasta, input.count_table, input.taxonomy) end end diff --git a/src/dada2.jl b/src/dada2.jl index f279c98..bf5e51d 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -30,6 +30,7 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, import Downloads using Logging, RCall, YAML + using ..PipelineTypes include("dada2/context.jl") # shared helpers and _pipeline_context include("dada2/qc.jl") # prefilter_qc, filter_trim @@ -69,6 +70,7 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, - `checkpoint.RData` - final R environment snapshot """ function dada2(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing, taxonomy_db=nothing) + R"rm(list=ls())" prefilter_qc(config_path; progress, input_dir, workspace_root) filter_trim(config_path; progress, input_dir, workspace_root); R"gc()" learn_errors(config_path; progress, input_dir, workspace_root); R"gc()" @@ -78,4 +80,33 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end + function dada2(project::ProjectCtx, trimmed::TrimmedReads; + taxonomy_db=nothing, progress=nothing) + config_path = joinpath(project.dir, "dada2.yml") + workspace_root = joinpath(project.dir, "dada2") + cfg = YAML.load_file(config_path) + out_cfg = get(cfg, "output", Dict()) + tables_dir = joinpath(workspace_root, "Tables") + result = ASVResult( + joinpath(tables_dir, get(out_cfg, "fasta_prefix", "asvs") * ".fasta"), + joinpath(tables_dir, get(out_cfg, "seq_table_prefix", "seqtab_nochim") * ".csv"), + joinpath(tables_dir, get(out_cfg, "taxa_prefix", "taxonomy") * ".csv") + ) + trimmed_files = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(trimmed.dir)) + trimmed_mtime = isempty(trimmed_files) ? 0.0 : + maximum(mtime(joinpath(trimmed.dir, f)) for f in trimmed_files) + input_mtime = max(mtime(config_path), trimmed_mtime) + if all(isfile, (result.fasta, result.count_table, result.taxonomy)) && + all(f -> mtime(f) > input_mtime, (result.fasta, result.count_table, result.taxonomy)) + @info "Skipping dada2: outputs up to date in $tables_dir" + return result + end + dada2(config_path; + input_dir = trimmed.dir, + workspace_root, + taxonomy_db, + progress) + return result + end + end diff --git a/src/dada2/chimera.jl b/src/dada2/chimera.jl index db1b6e7..1175b98 100644 --- a/src/dada2/chimera.jl +++ b/src/dada2/chimera.jl @@ -27,50 +27,82 @@ error("Length filter checkpoint not found. Run filter_length() first.") filter_ckpt = ctx.ckpts["filter"] length_ckpt = ctx.ckpts["length"] - R"load($filter_ckpt)" - R"load($length_ckpt)" - emit("Removing chimeras") - denovo_method = ctx.cfg["asv"]["denovo_method"] - R""" - seq_table_nochim <- removeBimeraDenovo(seq_table, method=$denovo_method, verbose=$verbose) - nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 - message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), - "% | Retained: ", round(nochim_pct, 2), "%") - """ - - # Drop duplicate sample row (single-sample fallback) - final_names = ctx.sample_names - if ctx.single_sample - R"filter_stats <- filter_stats[1, , drop=FALSE]" - mode != "reverse" && R"dada_fwd <- dada_fwd[1]" - mode != "forward" && R"dada_rev <- dada_rev[1]" - mode == "paired" && R"merged <- merged[1]" - R"seq_table_nochim <- seq_table_nochim[1, , drop=FALSE]" - final_names = [ctx.sample_names[1]] + chimera_ckpt = ctx.ckpts["chimera"] + if isfile(chimera_ckpt) + input_mtime = max(mtime(config_path), mtime(filter_ckpt), mtime(length_ckpt)) + if mtime(chimera_ckpt) > input_mtime + @info "Skipping chimera_removal: checkpoint up to date" + return nothing + end end - emit("Computing pipeline stats") - stats_csv = joinpath(ctx.dirs["Tables"], "pipeline_stats.csv") - R""" - stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, - seq_table_nochim, $final_names, $mode) - write.csv(stats, $stats_csv, quote=FALSE) - if ($verbose) print(stats) - """ - - emit("Writing core output tables") seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") tables_dir = ctx.dirs["Tables"] - R"write_seq_table(seq_table_nochim, $tables_dir, $seq_prefix)" - R"index <- write_fasta(seq_table_nochim, $tables_dir, $fasta_prefix)" - ckpt = ctx.ckpts["chimera"] - R"save(seq_table_nochim, index, file=$ckpt)" - emit("Written: $stats_csv") + log_path = joinpath(ctx.dirs["Logs"], "chimera_removal.log") + open(log_path, "w") do io; println(io, "=== chimera_removal ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + R"load($filter_ckpt)" + R"load($length_ckpt)" + + has_data = rcopy(R"isTRUE(sum(seq_table, na.rm=TRUE) > 0)") + emit("Removing chimeras") + if has_data + denovo_method = ctx.cfg["asv"]["denovo_method"] + R""" + seq_table_nochim <- removeBimeraDenovo(seq_table, method=$denovo_method, verbose=$verbose) + nochim_pct <- sum(seq_table_nochim) / sum(seq_table) * 100 + message(" Chimeric reads removed: ", round(100 - nochim_pct, 2), + "% | Retained: ", round(nochim_pct, 2), "%") + """ + else + R"seq_table_nochim <- seq_table" + @info "Chimera removal skipped: seq_table is empty" + end + + # Drop duplicate sample row (single-sample fallback) + final_names = ctx.sample_names + if ctx.single_sample + R"filter_stats <- filter_stats[1, , drop=FALSE]" + mode != "reverse" && R"dada_fwd <- dada_fwd[1]" + mode != "forward" && R"dada_rev <- dada_rev[1]" + mode == "paired" && R"merged <- merged[1]" + R"seq_table_nochim <- seq_table_nochim[1, , drop=FALSE]" + final_names = [ctx.sample_names[1]] + end + + emit("Computing pipeline stats") + stats_csv = joinpath(ctx.dirs["Tables"], "pipeline_stats.csv") + R""" + stats <- compute_pipeline_stats(filter_stats, dada_fwd, dada_rev, merged, + seq_table_nochim, $final_names, $mode) + write.csv(stats, $stats_csv, quote=FALSE) + if ($verbose) print(stats) + """ + + emit("Writing core output tables") + if has_data + R"write_seq_table(seq_table_nochim, $tables_dir, $seq_prefix)" + R"index <- write_fasta(seq_table_nochim, $tables_dir, $fasta_prefix)" + else + R"index <- data.frame(SeqName=character(0), sequence=character(0))" + touch(joinpath(tables_dir, seq_prefix * ".csv")) + touch(joinpath(tables_dir, fasta_prefix * ".fasta")) + touch(joinpath(tables_dir, fasta_prefix * ".csv")) + end + + ckpt = ctx.ckpts["chimera"] + R"save(seq_table_nochim, index, file=$ckpt)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Written: $(joinpath(tables_dir, "pipeline_stats.csv"))") emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") - emit("Checkpoint: $ckpt") + emit("Checkpoint: $(ctx.ckpts["chimera"])") + emit("Log: $log_path") nothing end diff --git a/src/dada2/context.jl b/src/dada2/context.jl index f3eee23..d98cc01 100644 --- a/src/dada2/context.jl +++ b/src/dada2/context.jl @@ -33,36 +33,37 @@ return cached end - # Resolve the DADA2 taxonomy database from config when no external path is - # provided. Reads taxonomy.database key and looks it up in the databases: section. + # Resolve the DADA2 taxonomy database. + # Reads from global config/databases.yml; falls back to pipeline config's + # databases: section for backward compatibility with old per-project configs. function _resolve_taxonomy_db(cfg, emit) - tax_cfg = cfg["taxonomy"] - db_key = string(tax_cfg["database"]) - db_cfg = get(cfg, "databases", Dict()) - haskey(db_cfg, db_key) || - error("taxonomy.database = \"$db_key\" not found in databases: section") - fmt_cfg = get(db_cfg[db_key], "dada2", nothing) + db_key = string(cfg["taxonomy"]["database"]) + global_dbs = joinpath(@__DIR__, "..", "..", "config", "databases.yml") + + db_source = if isfile(global_dbs) + get(YAML.load_file(global_dbs), "databases", Dict()) + else + get(cfg, "databases", Dict()) # backward compat + end + + haskey(db_source, db_key) || + error("taxonomy.database = \"$db_key\" not found in databases config") + fmt_cfg = get(db_source[db_key], "dada2", nothing) isnothing(fmt_cfg) && - error("databases.$db_key.dada2 is not configured") - db_dir = abspath(get(db_cfg, "dir", "./databases")) + error("databases.$db_key.dada2 is not configured in config/databases.yml") + db_dir = abspath(get(db_source, "dir", "./databases")) mkpath(db_dir) return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) end # Config function validate_config(cfg) - required = ["workspace", "file_patterns", "filter_trim", "dada", + required = ["file_patterns", "filter_trim", "dada", "merge", "asv", "taxonomy", "output"] missing_secs = filter(k -> !haskey(cfg, k), required) isempty(missing_secs) || error("Missing required config sections: $(join(missing_secs, ", "))") - ws = cfg["workspace"] - haskey(ws, "root") || error("workspace.root is required") - haskey(ws, "input_dir") || error("workspace.input_dir is required") - isdir(ws["input_dir"]) || - error("workspace.input_dir does not exist: $(ws["input_dir"])") - mode = get(cfg["file_patterns"], "mode", "paired") mode in ("paired", "forward", "reverse") || error("file_patterns.mode must be one of: paired, forward, reverse") @@ -115,6 +116,7 @@ "Checkpoints" => joinpath(root, "Checkpoints"), "Figures" => joinpath(root, "Figures"), "Filtered" => joinpath(root, "Filtered"), + "Logs" => joinpath(root, "Logs"), ) for d in values(dirs) mkpath(d) @@ -137,21 +139,18 @@ cfg = YAML.load_file(config_path) - if !isnothing(input_dir) - cfg["workspace"]["input_dir"] = input_dir - end - if !isnothing(workspace_root) - cfg["workspace"]["root"] = workspace_root - end + isnothing(input_dir) && error("input_dir must be provided") + isnothing(workspace_root) && error("workspace_root must be provided") + isdir(input_dir) || error("input_dir does not exist: $input_dir") validate_config(cfg) verbose = get(cfg, "verbose", true) mode = get(cfg["file_patterns"], "mode", "paired") - root = cfg["workspace"]["root"] + root = workspace_root dirs = setup_workspace(root) fwd_files, rev_files = find_fastq_files( - cfg["workspace"]["input_dir"], + input_dir, cfg["file_patterns"]["forward"], cfg["file_patterns"]["reverse"], mode) diff --git a/src/dada2/denoise.jl b/src/dada2/denoise.jl index 5ce9063..162808d 100644 --- a/src/dada2/denoise.jl +++ b/src/dada2/denoise.jl @@ -16,6 +16,17 @@ function learn_errors(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) ctx = _pipeline_context(config_path; input_dir, workspace_root) + + errors_ckpt = ctx.ckpts["errors"] + filter_ckpt = ctx.ckpts["filter"] + if isfile(errors_ckpt) && isfile(filter_ckpt) + input_mtime = max(mtime(config_path), mtime(filter_ckpt)) + if mtime(errors_ckpt) > input_mtime + @info "Skipping learn_errors: checkpoint up to date" + return nothing + end + end + seed = get(ctx.cfg["dada"], "seed", 123) nbases = ctx.cfg["dada"]["nbases"] max_con = ctx.cfg["dada"]["max_consist"] @@ -23,24 +34,32 @@ fwd_out = ctx.fwd_out rev_out = ctx.rev_out - emit("Learning error rates") - R"set.seed($seed)" + log_path = joinpath(ctx.dirs["Logs"], "learn_errors.log") + open(log_path, "w") do io; println(io, "=== learn_errors ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + emit("Learning error rates") + R"set.seed($seed)" - ctx.mode != "reverse" ? - R"fwd_errors <- learnErrors($fwd_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : - R"fwd_errors <- NULL" + ctx.mode != "reverse" ? + R"fwd_errors <- learnErrors($fwd_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"fwd_errors <- NULL" - ctx.mode != "forward" ? - R"rev_errors <- learnErrors($rev_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : - R"rev_errors <- NULL" + ctx.mode != "forward" ? + R"rev_errors <- learnErrors($rev_out, nbases=$nbases, MAX_CONSIST=$max_con, verbose=$verbose)" : + R"rev_errors <- NULL" - error_pdf = joinpath(ctx.dirs["Figures"], "error_rates.pdf") - R"plot_error_rates(fwd_errors, rev_errors, $error_pdf)" + error_pdf = joinpath(ctx.dirs["Figures"], "error_rates.pdf") + R"plot_error_rates(fwd_errors, rev_errors, $error_pdf)" - ckpt = ctx.ckpts["errors"] - R"save(fwd_errors, rev_errors, file=$ckpt)" - emit("Written: $error_pdf") - emit("Checkpoint: $ckpt") + ckpt = ctx.ckpts["errors"] + R"save(fwd_errors, rev_errors, file=$ckpt)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Written: $(joinpath(ctx.dirs["Figures"], "error_rates.pdf"))") + emit("Checkpoint: $(ctx.ckpts["errors"])") + emit("Log: $log_path") nothing end @@ -68,49 +87,73 @@ isfile(ctx.ckpts["errors"]) || error("Error model checkpoint not found. Run learn_errors() first.") errors_ckpt = ctx.ckpts["errors"] - R"load($errors_ckpt)" - - emit("Denoising reads") - pool_method = ctx.cfg["dada"]["pool_method"] - ctx.mode != "reverse" ? - R"dada_fwd <- dada($fwd_out, err=fwd_errors, pool=$pool_method, verbose=$verbose)" : - R"dada_fwd <- NULL" - - ctx.mode != "forward" ? - R"dada_rev <- dada($rev_out, err=rev_errors, pool=$pool_method, verbose=$verbose)" : - R"dada_rev <- NULL" + denoise_ckpt = ctx.ckpts["denoise"] + if isfile(denoise_ckpt) + input_mtime = max(mtime(config_path), mtime(errors_ckpt)) + if mtime(denoise_ckpt) > input_mtime + @info "Skipping denoise: checkpoint up to date" + return nothing + end + end - emit("Building sequence table") - if ctx.mode == "paired" - min_overlap = ctx.cfg["merge"]["min_overlap"] - max_mismatch = ctx.cfg["merge"]["max_mismatch"] - trim_overhang = ctx.cfg["merge"]["trim_overhang"] - R""" - merged <- mergePairs( - dada_fwd, $fwd_out, - dada_rev, $rev_out, - minOverlap = $min_overlap, - maxMismatch = $max_mismatch, - trimOverhang = $trim_overhang, - verbose = $verbose - ) - seq_table <- makeSequenceTable(merged) - """ - else + log_path = joinpath(ctx.dirs["Logs"], "denoise.log") + open(log_path, "w") do io; println(io, "=== denoise ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + R"load($errors_ckpt)" + + emit("Denoising reads") + pool_method = ctx.cfg["dada"]["pool_method"] + + ctx.mode != "reverse" ? + R"dada_fwd <- dada($fwd_out, err=fwd_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_fwd <- NULL" + + ctx.mode != "forward" ? + R"dada_rev <- dada($rev_out, err=rev_errors, pool=$pool_method, verbose=$verbose)" : + R"dada_rev <- NULL" + + emit("Building sequence table") + if ctx.mode == "paired" + min_overlap = ctx.cfg["merge"]["min_overlap"] + max_mismatch = ctx.cfg["merge"]["max_mismatch"] + trim_overhang = ctx.cfg["merge"]["trim_overhang"] + R""" + merged <- mergePairs( + dada_fwd, $fwd_out, + dada_rev, $rev_out, + minOverlap = $min_overlap, + maxMismatch = $max_mismatch, + trimOverhang = $trim_overhang, + verbose = $verbose + ) + seq_table <- makeSequenceTable(merged) + """ + else + R""" + merged <- NULL + seq_table <- makeSequenceTable(if (!is.null(dada_fwd)) dada_fwd else dada_rev) + """ + end + + len_dist_pdf = joinpath(ctx.dirs["Figures"], "length_distribution.pdf") R""" - merged <- NULL - seq_table <- makeSequenceTable(if (!is.null(dada_fwd)) dada_fwd else dada_rev) + if (sum(seq_table) > 0) { + plot_length_distribution(seq_table, $len_dist_pdf) + } else { + message("Skipping length distribution plot: seq_table is empty after merging") + } """ - end - len_dist_pdf = joinpath(ctx.dirs["Figures"], "length_distribution.pdf") - R"plot_length_distribution(seq_table, $len_dist_pdf)" - - ckpt = ctx.ckpts["denoise"] - R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" - emit("Written: $len_dist_pdf") - emit("Checkpoint: $ckpt") + ckpt = ctx.ckpts["denoise"] + R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Written: $(joinpath(ctx.dirs["Figures"], "length_distribution.pdf"))") + emit("Checkpoint: $(ctx.ckpts["denoise"])") + emit("Log: $log_path") nothing end @@ -135,22 +178,46 @@ isfile(ctx.ckpts["denoise"]) || error("Denoise checkpoint not found. Run denoise() first.") denoise_ckpt = ctx.ckpts["denoise"] - R"load($denoise_ckpt)" - - band_min = get(ctx.cfg["asv"], "band_size_min", nothing) - band_max = get(ctx.cfg["asv"], "band_size_max", nothing) - if !isnothing(band_min) && !isnothing(band_max) - emit("Filtering by length: $band_min-$band_max bp") - R"seq_table <- filter_by_length(seq_table, $band_min, $band_max)" - len_filt_pdf = joinpath(ctx.dirs["Figures"], "length_distribution_filtered.pdf") - R"plot_length_distribution(seq_table, $len_filt_pdf)" - emit("Written: $len_filt_pdf") - else - emit("No length filter configured (band_size_min/max not set) - passing through") + + length_ckpt = ctx.ckpts["length"] + if isfile(length_ckpt) + input_mtime = max(mtime(config_path), mtime(denoise_ckpt)) + if mtime(length_ckpt) > input_mtime + @info "Skipping filter_length: checkpoint up to date" + return nothing + end end - ckpt = ctx.ckpts["length"] - R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" - emit("Checkpoint: $ckpt") + log_path = joinpath(ctx.dirs["Logs"], "filter_length.log") + open(log_path, "w") do io; println(io, "=== filter_length ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + R"load($denoise_ckpt)" + + band_min = get(ctx.cfg["asv"], "band_size_min", nothing) + band_max = get(ctx.cfg["asv"], "band_size_max", nothing) + if !isnothing(band_min) && !isnothing(band_max) + emit("Filtering by length: $band_min-$band_max bp") + R"seq_table <- filter_by_length(seq_table, $band_min, $band_max)" + len_filt_pdf = joinpath(ctx.dirs["Figures"], "length_distribution_filtered.pdf") + R""" + if (sum(seq_table) > 0) { + plot_length_distribution(seq_table, $len_filt_pdf) + } else { + message("Skipping filtered length distribution plot: seq_table is empty after length filter") + } + """ + emit("Written: $len_filt_pdf") + else + emit("No length filter configured (band_size_min/max not set) - passing through") + end + + ckpt = ctx.ckpts["length"] + R"save(dada_fwd, dada_rev, merged, seq_table, file=$ckpt)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Checkpoint: $(ctx.ckpts["length"])") + emit("Log: $log_path") nothing end diff --git a/src/dada2/qc.jl b/src/dada2/qc.jl index 18a12dd..17326d2 100644 --- a/src/dada2/qc.jl +++ b/src/dada2/qc.jl @@ -14,12 +14,32 @@ emit = _emitter(progress) ctx = _pipeline_context(config_path; input_dir, workspace_root) - emit("Plotting unfiltered quality profiles") - fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files - rev_for_plot = isempty(ctx.rev_files) ? nothing : ctx.rev_files unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") - R"plot_quality_profiles($fwd_for_plot, $rev_for_plot, $unfiltered_pdf)" - emit("Written: $unfiltered_pdf") + if isfile(unfiltered_pdf) + all_inputs = vcat(ctx.fwd_files, ctx.rev_files) + input_mtime = isempty(all_inputs) ? mtime(config_path) : + max(mtime(config_path), maximum(mtime(f) for f in all_inputs)) + if mtime(unfiltered_pdf) > input_mtime + @info "Skipping prefilter_qc: quality_unfiltered.pdf up to date" + return nothing + end + end + + log_path = joinpath(ctx.dirs["Logs"], "prefilter_qc.log") + open(log_path, "w") do io; println(io, "=== prefilter_qc ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + emit("Plotting unfiltered quality profiles") + fwd_for_plot = isempty(ctx.fwd_files) ? nothing : ctx.fwd_files + rev_for_plot = isempty(ctx.rev_files) ? nothing : ctx.rev_files + unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") + R"plot_quality_profiles($fwd_for_plot, $rev_for_plot, $unfiltered_pdf)" + + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf"))") + emit("Log: $log_path") nothing end @@ -38,6 +58,18 @@ function filter_trim(config_path::String; progress=nothing, input_dir=nothing, workspace_root=nothing) emit = _emitter(progress) ctx = _pipeline_context(config_path; input_dir, workspace_root) + + filter_ckpt = ctx.ckpts["filter"] + if isfile(filter_ckpt) + all_inputs = vcat(ctx.fwd_files, ctx.rev_files) + input_mtime = isempty(all_inputs) ? mtime(config_path) : + max(mtime(config_path), maximum(mtime(f) for f in all_inputs)) + if mtime(filter_ckpt) > input_mtime + @info "Skipping filter_trim: checkpoint up to date" + return nothing + end + end + ft = ctx.cfg["filter_trim"] trunc_len = ft["trunc_len"] max_ee = ft["max_ee"] @@ -49,46 +81,54 @@ fwd_out = ctx.fwd_out rev_out = ctx.rev_out - emit("Filtering and trimming reads") - if ctx.mode == "paired" - R""" - filter_stats <- filterAndTrim( - $in_fwd, $out_fwd, - $in_rev, $out_rev, - truncQ = $(ft["trunc_q"]), - truncLen = $trunc_len, - maxEE = $max_ee, - minLen = $(ft["min_len"]), - maxN = $(ft["max_n"]), - matchIDs = $(ft["match_ids"]), - rm.phix = $(ft["rm_phix"]), - verbose = $verbose - ) - """ - else - R""" - filter_stats <- filterAndTrim( - $in_fwd, $out_fwd, - truncQ = $(ft["trunc_q"]), - truncLen = $(trunc_len[1]), - maxEE = $(max_ee[1]), - minLen = $(ft["min_len"]), - maxN = $(ft["max_n"]), - rm.phix = $(ft["rm_phix"]), - verbose = $verbose - ) - """ - end + log_path = joinpath(ctx.dirs["Logs"], "filter_trim.log") + open(log_path, "w") do io; println(io, "=== filter_trim ===\nconfig: $config_path") end + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + emit("Filtering and trimming reads") + if ctx.mode == "paired" + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + $in_rev, $out_rev, + truncQ = $(ft["trunc_q"]), + truncLen = $trunc_len, + maxEE = $max_ee, + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + matchIDs = $(ft["match_ids"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + else + R""" + filter_stats <- filterAndTrim( + $in_fwd, $out_fwd, + truncQ = $(ft["trunc_q"]), + truncLen = $(trunc_len[1]), + maxEE = $(max_ee[1]), + minLen = $(ft["min_len"]), + maxN = $(ft["max_n"]), + rm.phix = $(ft["rm_phix"]), + verbose = $verbose + ) + """ + end - emit("Plotting filtered quality profiles") - fwd_filt = isempty(fwd_out) ? nothing : fwd_out - rev_filt = isempty(rev_out) ? nothing : rev_out - filtered_pdf = joinpath(ctx.dirs["Figures"], "quality_filtered.pdf") - R"plot_quality_profiles($fwd_filt, $rev_filt, $filtered_pdf)" + emit("Plotting filtered quality profiles") + fwd_filt = isempty(fwd_out) ? nothing : fwd_out + rev_filt = isempty(rev_out) ? nothing : rev_out + filtered_pdf = joinpath(ctx.dirs["Figures"], "quality_filtered.pdf") + R"plot_quality_profiles($fwd_filt, $rev_filt, $filtered_pdf)" - ckpt = ctx.ckpts["filter"] - R"save(filter_stats, file=$ckpt)" - emit("Written: $filtered_pdf") - emit("Checkpoint: $ckpt") + ckpt = ctx.ckpts["filter"] + R"save(filter_stats, file=$ckpt)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_filtered.pdf"))") + emit("Checkpoint: $(ctx.ckpts["filter"])") + emit("Log: $log_path") nothing end diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index 5bd6fef..72b0ff9 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -7,9 +7,6 @@ **Stage 7** - Assign taxonomy to ASVs and write the combined output table. - This is typically the longest step. Set `taxonomy.skip = true` in config to - skip assignment and output sequence/count data only. - Requires: `Checkpoints/ckpt_chimera.RData` Saves: `Checkpoints/checkpoint.RData` """ @@ -22,7 +19,8 @@ # or any other consequences arising from misconfiguration or misuse of this feature. function _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, checkpoint, taxa_prefix, multithread, - min_boot, tax_levels, verbose, remote_cfg) + min_boot, tax_levels, verbose, remote_cfg, + log_path) host = remote_cfg["host"] rscript = get(remote_cfg, "rscript", "Rscript") base_dir = get(remote_cfg, "staging_dir", nothing) @@ -83,7 +81,9 @@ "verbose=$verbose_str" emit(" Running Rscript on $host:$staging_dir") - run(ssh(host, remote_cmd)) + open(log_path, "a") do io + run(pipeline(ssh(host, remote_cmd); stdout=io, stderr=io)) + end emit(" Retrieving results from $host") run(scp("$host:$remote_tables/$taxa_prefix.csv", "$tables_dir/")) @@ -112,6 +112,16 @@ isfile(ctx.ckpts["chimera"]) || error("Chimera checkpoint not found. Run chimera_removal() first.") + chimera_ckpt = ctx.ckpts["chimera"] + checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") + if isfile(checkpoint) + input_mtime = max(mtime(config_path), mtime(chimera_ckpt)) + if mtime(checkpoint) > input_mtime + @info "Skipping assign_taxonomy: checkpoint up to date" + return nothing + end + end + # Drop all data objects accumulated from prior stages before the # memory-intensive taxonomy assignment runs. Named globals in R's # environment are reachable and gc() won't collect them; rm() them @@ -124,14 +134,12 @@ gc() """ - chimera_ckpt = ctx.ckpts["chimera"] seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.csv") asv_file = get(ctx.cfg["output"], "asv_filename", "asv_counts.csv") tables_dir = ctx.dirs["Tables"] - checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") multithread = get(ctx.cfg["taxonomy"], "multithread", 4) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) tax_levels = ctx.cfg["taxonomy"]["levels"] @@ -145,29 +153,51 @@ db_path = skip_local_db ? nothing : isnothing(taxonomy_db) ? _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db - if use_remote + R"load($chimera_ckpt)" + has_data = rcopy(R"isTRUE(sum(seq_table_nochim, na.rm=TRUE) > 0)") + + log_path = joinpath(ctx.dirs["Logs"], "assign_taxonomy.log") + open(log_path, "w") do io; println(io, "=== assign_taxonomy ===\nconfig: $config_path") end + + if !has_data + @info "Taxonomy assignment skipped: no ASVs in seq_table_nochim" + R"taxa_df <- data.frame()" + R"save(seq_table_nochim, index, taxa_df, file=$checkpoint)" + for suffix in (taxa_prefix * ".csv", taxa_prefix * "_bootstraps.csv", + taxa_prefix * "_combined.csv", combined_file, asv_file) + touch(joinpath(tables_dir, suffix)) + end + elseif use_remote emit("Assigning taxonomy (remote: $(remote_cfg["host"]))") _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, checkpoint, taxa_prefix, multithread, - min_boot, tax_levels, verbose, remote_cfg) + min_boot, tax_levels, verbose, remote_cfg, + log_path) R"load($checkpoint)" + R"write_combined_table(taxa_df, index, seq_table_nochim, $tables_dir, $combined_file, $asv_file)" + emit("Log: $log_path") else - emit("Assigning taxonomy") - R"load($chimera_ckpt)" - R""" - taxa_result <- run_assign_taxonomy( - seq_table_nochim, $db_path, - list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), - $verbose) - taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, - $tables_dir, $taxa_prefix) - """ - R"gc()" - R"save(seq_table_nochim, index, taxa_df, file=$checkpoint)" + R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" + try + emit("Assigning taxonomy") + R""" + taxa_result <- run_assign_taxonomy( + seq_table_nochim, $db_path, + list(multithread=$multithread, min_boot=$min_boot, levels=$tax_levels), + $verbose) + taxa_df <- write_taxa_table(taxa_result$tax, taxa_result$boot, index, + $tables_dir, $taxa_prefix) + """ + R"gc()" + R"save(seq_table_nochim, index, taxa_df, file=$checkpoint)" + finally + R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" + end + R"write_combined_table(taxa_df, index, seq_table_nochim, $tables_dir, $combined_file, $asv_file)" + emit("Log: $log_path") end emit("Checkpoint: $checkpoint") - R"write_combined_table(taxa_df, index, seq_table_nochim, $tables_dir, $combined_file, $asv_file)" emit("Pipeline complete. Outputs:") emit(" $(joinpath(tables_dir, seq_prefix * ".csv"))") diff --git a/src/databases.jl b/src/databases.jl index 9ccf9db..d3733c1 100644 --- a/src/databases.jl +++ b/src/databases.jl @@ -21,69 +21,75 @@ using YAML, Logging export ensure_databases -""" - ensure_databases(config_path) -> Dict{String,String} + """ + ensure_databases(config_path) -> Dict{String,String} -Reads the `databases:` section of `config_path`, ensures every declared -database file is present in `databases.dir` (downloading from `uri` if the -file is absent), and returns a Dict mapping `"_"` keys to -resolved absolute local paths. + Reads the `databases:` section of `config_path`, ensures every declared + database file is present in `databases.dir` (downloading from `uri` if the + file is absent), and returns a Dict mapping `"_"` keys to + resolved absolute local paths. -Set a `local:` path under any entry to use a pre-existing file directly. -If the `local:` path does not exist, the function warns and falls back to -downloading from `uri`. -""" -function ensure_databases(config_path::String) - cfg = YAML.load_file(config_path) - db_cfg = get(cfg, "databases", nothing) + Set a `local:` path under any entry to use a pre-existing file directly. + If the `local:` path does not exist, the function warns and falls back to + downloading from `uri`. + """ + function ensure_databases(config_path::String) + if !isfile(config_path) + @warn "ensure_databases: $config_path not found. " * + "Copy config/databases.example.yml to config/databases.yml and set local: paths." + return Dict{String,String}() + end - if isnothing(db_cfg) || isempty(db_cfg) - @warn "ensure_databases: no `databases:` section found in $config_path" - return Dict{String,String}() - end + cfg = YAML.load_file(config_path) + db_cfg = get(cfg, "databases", nothing) + + if isnothing(db_cfg) || isempty(db_cfg) + @warn "ensure_databases: no `databases:` section found in $config_path" + return Dict{String,String}() + end - db_dir = abspath(get(db_cfg, "dir", "./databases")) - mkpath(db_dir) + db_dir = abspath(get(db_cfg, "dir", "./databases")) + mkpath(db_dir) - resolved = Dict{String,String}() - for (db_name, db_info) in db_cfg - db_name == "dir" && continue - !(db_info isa AbstractDict) && continue - for (fmt, fmt_info) in db_info - !(fmt_info isa AbstractDict) && continue - key = "$(db_name)_$(fmt)" - resolved[key] = _resolve_entry(key, fmt_info, db_dir) + resolved = Dict{String,String}() + for (db_name, db_info) in db_cfg + db_name == "dir" && continue + !(db_info isa AbstractDict) && continue + for (fmt, fmt_info) in db_info + !(fmt_info isa AbstractDict) && continue + key = "$(db_name)_$(fmt)" + resolved[key] = _resolve_entry(key, fmt_info, db_dir) + end end + return resolved end - return resolved -end -function _resolve_entry(key, fmt_info, db_dir) - local_p = get(fmt_info, "local", nothing) - if !isnothing(local_p) - local_p = string(local_p) - if !isempty(local_p) - if isfile(local_p) - @info "[$key] Using local file: $local_p" - return local_p + function _resolve_entry(key, fmt_info, db_dir) + local_p = get(fmt_info, "local", nothing) + if !isnothing(local_p) + local_p = string(local_p) + if !isempty(local_p) + if isfile(local_p) + @info "[$key] Using local file: $local_p" + return local_p + end + @warn "[$key] Configured local path not found: $local_p - falling back to uri" end - @warn "[$key] Configured local path not found: $local_p - falling back to uri" end - end - uri = get(fmt_info, "uri", nothing) - isnothing(uri) && - error("databases entry '$key': no valid local path and no uri is configured") + uri = get(fmt_info, "uri", nothing) + isnothing(uri) && + error("databases entry '$key': no valid local path and no uri is configured") - cached = joinpath(db_dir, basename(uri)) - if isfile(cached) - @info "[$key] Using cached: $cached" - else - @info "[$key] Downloading: $uri" - Downloads.download(uri, cached) - @info "[$key] Saved to: $cached" + cached = joinpath(db_dir, basename(uri)) + if isfile(cached) + @info "[$key] Using cached: $cached" + else + @info "[$key] Downloading: $uri" + Downloads.download(uri, cached) + @info "[$key] Saved to: $cached" + end + return cached end - return cached -end end \ No newline at end of file diff --git a/src/main.jl b/src/main.jl index bf28cd8..3dd6c49 100755 --- a/src/main.jl +++ b/src/main.jl @@ -1,74 +1,36 @@ #!/usr/bin/env julia +include("types.jl") include("databases.jl") include("call_tools.jl") include("dada2.jl") include("merge_and_filter_taxa.jl") +include("project.jl") using CSV using YAML -using .Databases, .Tools, .TaxonomyTableTools, .DADA2 - -## Instantiate filesystem -# Root directories -data_dir = "./data" -config_dir = "./config" -output_dir = "./output" - -# Project name - all stage outputs live under output/{project_name}/ -# Re-running with the same project_name overwrites previous results. -project_name = "Multiplex_pool" -project_dir = joinpath(output_dir, project_name) -fastq_input_dir = joinpath(data_dir, project_name) - -# QC paths -qc_dir = joinpath(project_dir, "QC") - -# Cutadapt paths -trimmed_dir = joinpath(project_dir, "cutadapt") -primers_config = joinpath(config_dir, "primers.yml") - -# DADA2 paths -dada2_dir = joinpath(project_dir, "dada2") - -# VSEARCH paths -vsearch_dir = joinpath(project_dir, "vsearch") -fasta_outfile = joinpath(dada2_dir, "Tables/asvs.fasta") - -# Merge/filter paths (derived from project_dir) -merged_dir = joinpath(project_dir, "merged") -merged_outfile_multi = joinpath(merged_dir, "merged_multi.csv") -filtered_outfile_multi = joinpath(merged_dir, "protist_filtered_multi.csv") +using .PipelineTypes, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup ## Instantiate parameters -# Cutadapt parameters -primer_pairs = ["TarEuk", "Meta2"] -cutadapt_optional_args = "-m 200 --discard-untrimmed" - -# DADA2 parameters -dada2_config_path = joinpath(config_dir, "dada2.yml") - -# VSEARCH parameters -vsearch_optional_args = "--id 0.75 --query_cov 0.8" - -# Merge and filter (DADA2-VSEARCH) parameters -mkpath(merged_dir) -multiv = joinpath(vsearch_dir, "taxonomy.tsv") -multid = joinpath(dada2_dir, "Tables/taxonomy.csv") - -protist_filter = joinpath(config_dir, "protist_filter.yml") +config_dir = "./config" +databases_config = joinpath(config_dir, "databases.yml") -# Download/use database -dbs = ensure_databases(dada2_config_path) +dbs = ensure_databases(databases_config) ## Main -multiqc(fastq_input_dir, qc_dir) +const r_lock = ReentrantLock() -#cutadapt(primer_pairs, primers_config, fastq_input_dir, trimmed_dir, optional_args = cutadapt_optional_args) +projects = new_project("Multi_v_Vespa") -#dada2(dada2_config_path, input_dir = trimmed_dir, workspace_root = dada2_dir, taxonomy_db = dbs["pr2_dada2"]) +Threads.@threads for project in projects + multiqc(project.data_dir, joinpath(project.dir, "QC")) -#vsearch(fasta_outfile, dbs["pr2_vsearch"], vsearch_dir, optional_args = vsearch_optional_args) + trimmed = cutadapt(project) -#CSV.write(merged_outfile_multi, merge_taxonomy_counts(multiv, multid)); @info "Written: $merged_outfile_multi" -#CSV.write(filtered_outfile_multi, filter_table(merge_taxonomy_counts(multiv, multid), protist_filter)); @info "Written: $filtered_outfile_multi" + asvs = lock(r_lock) do + dada2(project, trimmed, taxonomy_db = dbs["pr2_dada2"]) + end + #asvs = lock(r_lock) do; cdhit(project, asvs); end # optional, uncomment to enable + tax = vsearch(project, asvs, dbs["pr2_vsearch"]) + merged = merge_taxa(project, asvs, tax) +end diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index 4a915a7..d6711e7 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -5,8 +5,9 @@ module TaxonomyTableTools # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). using CSV, DataFrames, Logging, YAML +using ..PipelineTypes -export merge_taxonomy_counts, filter_table +export merge_taxonomy_counts, filter_table, merge_taxa # Import vsearch taxonomy function import_vsearch(file::AbstractString) @@ -66,7 +67,7 @@ export merge_taxonomy_counts, filter_table return header, out_rows end - # Sorting helper + # Sorting helper for DADA2 seq IDs (seq1, seq2, ...). function seqnum(x) if ismissing(x) return typemax(Int) @@ -124,6 +125,10 @@ export merge_taxonomy_counts, filter_table end end + # Fallback: if no data rows, detect seq-ID column by name + if isnothing(seq_id_col) + "SeqName" in names(df_counts) && (seq_id_col = "SeqName") + end if isnothing(seq_id_col) error("Cannot find a column with seq IDs (seq1, seq2, ...) in counts file!") end @@ -268,4 +273,55 @@ export merge_taxonomy_counts, filter_table return df end + function merge_taxa(project::ProjectCtx, source::ASVResult, tax::TaxonomyHits) + config_path = joinpath(project.dir, "merge_taxa.yml") + cfg = YAML.load_file(config_path) + filter_list = get(cfg, "filters", [nothing]) + merge_dir = joinpath(project.dir, "merged") + + data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy), mtime(config_path)) + merged_csv = joinpath(merge_dir, "merged.csv") + + # Determine what needs to be (re-)computed. + need_base = !isfile(merged_csv) || mtime(merged_csv) <= data_mtime + stale_filters = Pair{String,String}[] # stem => filter_path + tables = Dict{String,String}("merged" => merged_csv) + + for entry in filter_list + isnothing(entry) && continue + filter_name = string(entry) + filter_path = joinpath(project.config_dir, "filters", filter_name) + stem = splitext(filter_name)[1] + output_csv = joinpath(merge_dir, stem * ".csv") + tables[stem] = output_csv + if !isfile(output_csv) || mtime(output_csv) <= max(data_mtime, mtime(filter_path)) + push!(stale_filters, stem => filter_path) + else + @info "Skipping merge_taxa filter '$stem': $output_csv up to date" + end + end + + if !need_base && isempty(stale_filters) + @info "Skipping merge_taxa: all outputs up to date in $merge_dir" + return MergedTables(tables) + end + + mkpath(merge_dir) + df = need_base ? merge_taxonomy_counts(tax.tsv, source.taxonomy) : + CSV.read(merged_csv, DataFrame) + + if need_base + CSV.write(merged_csv, df) + @info "Written: $merged_csv" + end + + for (stem, filter_path) in stale_filters + output_csv = tables[stem] + CSV.write(output_csv, filter_table(df, filter_path)) + @info "Written: $output_csv" + end + + return MergedTables(tables) + end + end \ No newline at end of file diff --git a/src/project.jl b/src/project.jl new file mode 100644 index 0000000..c98b958 --- /dev/null +++ b/src/project.jl @@ -0,0 +1,125 @@ +module ProjectSetup + +export new_project + + using ..PipelineTypes + + # Walk upward from dirname(dir) to find the nearest ancestor within + # root_project that already has fname, then check config_dir, then + # fall back to defaults_dir. + function _find_config_source(dir::String, fname::String, + root_project::String, + config_dir::String, + defaults_dir::String) + root_norm = normpath(root_project) + candidate = normpath(dirname(dir)) + while candidate == root_norm || + startswith(candidate * "/", root_norm * "/") + path = joinpath(candidate, fname) + isfile(path) && return path + candidate == root_norm && break + candidate = normpath(dirname(candidate)) + end + global_path = joinpath(config_dir, fname) + isfile(global_path) && return global_path + return joinpath(defaults_dir, fname) + end + + """ + new_project(name; data_dir, projects_dir, config_dir) + + Bootstrap a project tree under `projects_dir/{name}/` mirroring the + directory structure found under `data_dir/{name}/`. + + Any directory within `data_dir/{name}/` that contains `.fastq.gz` files is + treated as a leaf run. Intermediate directories and leaf runs each receive + copies of the stage config templates, cascading downward: each level + inherits from the nearest ancestor that already has a given config, falling + back to `config_dir/defaults/` when no ancestor has it. + + Config cascade (nearest-first): run -> study root -> intermediate levels + -> `config_dir/` (global) -> `config_dir/defaults/` (ultimate fallback). + All levels are bootstrapped at call time; `config_dir/` stage configs are + created alongside `databases.yml` and `tools.yml`. + + Re-running `new_project` never overwrites existing configs. To reset a + level to its parent's settings, delete that config file and re-run. + + Returns a `Vector{ProjectCtx}`, one per leaf run. + """ + function new_project(name::String; + data_dir::String = "./data", + projects_dir::String = "./projects", + config_dir::String = "./config") + + defaults_dir = joinpath(config_dir, "defaults") + root_data = joinpath(data_dir, name) + root_project = joinpath(projects_dir, name) + + isdir(root_data) || error("Data directory not found: $root_data") + + stage_configs = ("dada2.yml", "cutadapt.yml", "vsearch.yml", + "cdhit.yml", "merge_taxa.yml") + + # Bootstrap global configs from defaults if missing. + for fname in ("databases.yml", "tools.yml", stage_configs...) + dst = joinpath(config_dir, fname) + if !isfile(dst) + src = joinpath(defaults_dir, fname) + isfile(src) || error("Default template not found: $src") + cp(src, dst) + @info "Created: $dst" + end + end + + # Find leaf directories (those containing .fastq.gz), relative to root_data. + leaf_relpaths = String[] + for (dirpath, _, files) in walkdir(root_data) + if any(f -> endswith(f, ".fastq.gz"), files) + push!(leaf_relpaths, relpath(dirpath, root_data)) + end + end + isempty(leaf_relpaths) && error("No .fastq.gz files found under $root_data") + + # Collect all directories to bootstrap (study root, intermediates, leaves). + # root_project is always included so it sits in the cascade between + # config/ and any deeper levels. + all_dirs = Set{String}([root_project]) + for rp in leaf_relpaths + rp == "." && continue + parts = splitpath(rp) + for i in 1:length(parts) + push!(all_dirs, joinpath(root_project, parts[1:i]...)) + end + end + + # Process top-down so each level can serve as source for its children. + sorted_dirs = sort(collect(all_dirs), by = d -> length(splitpath(d))) + + for dir in sorted_dirs + mkpath(dir) + for fname in stage_configs + dst = joinpath(dir, fname) + if isfile(dst) + @info "Exists (skipping): $dst" + continue + end + src = _find_config_source(dir, fname, root_project, config_dir, defaults_dir) + isfile(src) || error("Default template not found: $src") + cp(src, dst) + @info "Created: $dst" + end + end + + # Return one ProjectCtx per leaf. + projects = ProjectCtx[] + for rp in leaf_relpaths + proj_dir = rp == "." ? root_project : joinpath(root_project, rp) + fastq_dir = rp == "." ? root_data : joinpath(root_data, rp) + push!(projects, ProjectCtx(proj_dir, config_dir, fastq_dir)) + end + @info "$(length(projects)) project(s) ready under $root_project" + return projects + end + +end diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 0000000..185fdb7 --- /dev/null +++ b/src/types.jl @@ -0,0 +1,31 @@ +module PipelineTypes + + export HasFasta, ProjectCtx, TrimmedReads, ASVResult, TaxonomyHits, MergedTables + + abstract type HasFasta end + + struct ProjectCtx + dir::String # projects/{.../run}/ - configs and stage output dirs live here + config_dir::String # config/ + data_dir::String # data/{.../run}/ - FASTQ input files + end + + struct TrimmedReads + dir::String # .../cutadapt/ + end + + struct ASVResult <: HasFasta + fasta::String # .../dada2/Tables/asvs.fasta (or cdhit/asvs.fasta after cdhit) + count_table::String # .../dada2/Tables/seqtab_nochim.csv + taxonomy::String # .../dada2/Tables/taxonomy.csv + end + + struct TaxonomyHits + tsv::String # .../vsearch/taxonomy.tsv + end + + struct MergedTables + tables::Dict{String,String} # name => CSV path; always includes "merged" (unfiltered) + end + +end From 2b283f62db22b3371d359397f1dca61daccdcf8c Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 21 Feb 2026 21:39:43 +0100 Subject: [PATCH 027/175] Overhauled config setup. --- .gitignore | 3 +- Project.toml | 1 + README.md | 202 ++++++++++++++--------------- config/cdhit.yml | 1 - config/cutadapt.yml | 6 - config/defaults/cdhit.yml | 1 - config/defaults/cutadapt.yml | 5 - config/defaults/dada2.yml | 65 ---------- config/defaults/default_configs.md | 13 ++ config/defaults/merge_taxa.yml | 6 - config/defaults/pipeline.yml | 80 ++++++++++++ config/defaults/primers.yml | 30 +++++ config/defaults/vsearch.yml | 1 - config/global_configs.md | 17 +++ config/merge_taxa.yml | 3 - config/primers.yml | 27 ---- config/vsearch.yml | 1 - src/call_tools.jl | 45 ++++--- src/config.jl | 165 +++++++++++++++++++++++ src/dada2.jl | 84 ++++++++++-- src/dada2/chimera.jl | 16 ++- src/dada2/context.jl | 75 +++-------- src/dada2/denoise.jl | 45 ++++--- src/dada2/qc.jl | 24 ++-- src/dada2/taxonomy.jl | 35 +++-- src/dada2/taxonomy_remote.r | 6 +- src/databases.jl | 40 +++++- src/main.jl | 3 +- src/merge_and_filter_taxa.jl | 17 ++- src/project.jl | 186 +++++++++++++++++++------- src/types.jl | 10 +- 31 files changed, 799 insertions(+), 414 deletions(-) delete mode 100644 config/cdhit.yml delete mode 100644 config/cutadapt.yml delete mode 100644 config/defaults/cdhit.yml delete mode 100644 config/defaults/cutadapt.yml delete mode 100644 config/defaults/dada2.yml create mode 100644 config/defaults/default_configs.md delete mode 100644 config/defaults/merge_taxa.yml create mode 100644 config/defaults/pipeline.yml create mode 100644 config/defaults/primers.yml delete mode 100644 config/defaults/vsearch.yml create mode 100644 config/global_configs.md delete mode 100644 config/merge_taxa.yml delete mode 100644 config/primers.yml delete mode 100644 config/vsearch.yml create mode 100644 src/config.jl diff --git a/.gitignore b/.gitignore index 509eb21..e9612e8 100644 --- a/.gitignore +++ b/.gitignore @@ -263,8 +263,9 @@ data/ bin/ databases/ config/tools.yml -config/dada2.yml +config/pipeline.yml config/databases.yml +config/primers.yml ### Bits I don't want to share yet hide.* diff --git a/Project.toml b/Project.toml index 88ccc29..2ffa552 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +1,6 @@ [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" RCall = "6f49c342-dc21-5d91-9882-a32aef131414" diff --git a/README.md b/README.md index 08a9236..6898f9d 100644 --- a/README.md +++ b/README.md @@ -59,36 +59,31 @@ When a remote SSH path is set, the pipeline routes that tool's invocations throu ```julia projects = new_project("MyProject") -# Bootstraps projects/MyProject/ mirroring data/MyProject/ subdirectory structure - ensure "MyProject" name matches folder in data/. -# Creates dada2.yml, cutadapt.yml, vsearch.yml, cdhit.yml, merge_taxa.yml at each level. +# Bootstraps projects/MyProject/ mirroring data/MyProject/ subdirectory structure. +# Ensure "MyProject" matches the folder name under data/. +# Creates an empty pipeline.yml override stub at each level (study, group, run). ``` -Re-running `new_project` never overwrites existing configs. To reset a level to its parent's -settings, delete that config file and re-run. - -Or manually: -```bash -mkdir -p projects/MyProject -cp config/defaults/dada2.yml projects/MyProject/dada2.yml -cp config/defaults/cutadapt.yml projects/MyProject/cutadapt.yml -# etc. -``` +Re-running `new_project` never overwrites existing files. To reset a level to its parent's settings, delete that level's `pipeline.yml` and re-run. ## Configuration -Global configuration lives in `config/`; per-project pipeline configs live in `projects/{name}/`. When per-run configs are missing, they are automatically populated with per-project configs, which are populated by global config if missing: +Pipeline settings use a cascade: each level overrides the one above it, and any key you omit is inherited from the nearest ancestor. The fully merged result is written to `run_config.yml` at runtime - that is the single place to see exactly what was used for a run. | File | Purpose | |------|---------| -| `config/databases.yml` | Database URIs and optional local paths | -| `config/primers.yml` | Primer sequences and pair definitions | +| `config/defaults/` | Canonical defaults for every setting - do not edit | | `config/filters/` | Directory of taxonomic filter configs | +| `config/databases.yml` | Database URI's and optional local paths | +| `config/primers.yml` | Primer sequences and pair definitions | | `config/tools.yml` | Tool binary paths (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est) | -| `projects/{name}/cutadapt.yml` | Primer pair selection and cutadapt optional args | -| `projects/{name}/dada2.yml` | DADA2 pipeline parameters | -| `projects/{name}/vsearch.yml` | vsearch alignment thresholds | -| `projects/{name}/cdhit.yml` | cd-hit-est clustering threshold (optional stage) | -| `projects/{name}/merge_taxa.yml` | Which filter configs to apply in the merge step | +| `config/pipeline.yml` | Machine-level overrides (lowest user-editable precedence) | +| `projects/{name}/pipeline.yml` | Study-level overrides | +| `projects/{name}/{group}/pipeline.yml` | Group-level overrides (intermediate directories) | +| `projects/{name}/{run}/pipeline.yml` | Run-level overrides (highest precedence) | +| `projects/{name}/{run}/run_config.yml` | Generated merged config (provenance) - do not edit | + +Each `pipeline.yml` stub is created with a comment block explaining that level's role. Write only the keys you want to change; omit the rest. ### Configuring databases (`config/databases.yml`) @@ -129,95 +124,91 @@ Pairs: Store all primer pairs in here and reference whichever combinations you need per project. Shared primers across pairs (same forward primer in two pairs) are automatically deduplicated in the `cutadapt` invocation since otherwise it complains a bit. If you need duplicates, you must create the same sequence under a different name. -### Configuring cutadapt (`projects/{name}/cutadapt.yml`) +### Configuring cutadapt (`cutadapt:` in `pipeline.yml`) -Selects which primer pairs to apply and passes additional arguments to cutadapt. Inherits from `config/cutadapt.yml` if no per-project file exists. +Selects which primer pairs to apply and passes additional arguments to cutadapt. ```yaml -# Names must match keys in the Pairs section of config/primers.yml. -primer_pairs: - - PrimerPair1 - - PrimerPair2 +cutadapt: + # Names must match keys in the Pairs section of config/primers.yml. + primer_pairs: + - PrimerPair1 + - PrimerPair2 -optional_args: "-m 200 --discard-untrimmed" + optional_args: "-m 200 --discard-untrimmed" ``` `optional_args` is passed verbatim to cutadapt after the primer arguments. Here, the `-m` flag sets the minimum read length after trimming; `--discard-untrimmed` drops reads with no primer match. -### Configuring DADA2 (`projects/{name}/dada2.yml`) +### Configuring DADA2 (`dada2:` in `pipeline.yml`) > **Performance tip:** For large datasets and reference databases, consider installing my [optimised DADA2 fork](https://github.com/JoshuaJewell/dada2) in place of the standard Bioconductor package. It provides CPU and Nvidia CUDA GPU acceleration for taxonomy assignment with no API or config changes required. This fork is experimental - if you encounter unexpected results, the standard Bioconductor release should be considered the reference implementation. Ultimately, I had to offload `assignTaxonomy()` to a remote server (`taxonomy.remote` setting). ```yaml -file_patterns: - forward: "_R1_trimmed.fastq.gz" - reverse: "_R2_trimmed.fastq.gz" - sample_name_split: "_" # character to split filenames on - sample_name_index: 1 # which element is the sample name (1-based) - mode: "paired" # paired | forward | reverse - -# Filter and trim - DADA2's filterAndTrim(): -filter_trim: - trunc_q: 2 - trunc_len: [220, 220] # [forward, reverse]; first value used for single-end mode - max_ee: [3, 3] # maximum expected errors in F and R reads - min_len: 175 - max_n: 0 - match_ids: true - rm_phix: true - -# Denoising - learnErrors() and dada(): -dada: - seed: 123 - nbases: 200000000 - max_consist: 15 - pool_method: "pseudo" # none | pseudo | true - -# Merging - mergePairs(), paired mode only: -merge: - min_overlap: 20 - max_mismatch: 0 - trim_overhang: true - -# ASV length filtering and chimera removal: -asv: - band_size_min: 200 # null to skip length filtering - band_size_max: 430 - denovo_method: "consensus" # consensus | pooled | per-sample - -# Taxonomy - assignTaxonomy() against the configured database: -taxonomy: - database: pr2 # key into config/databases.yml - multithread: true # threads for assignTaxonomy() - min_boot: 0 # minimum bootstrap confidence to retain (0-100) - levels: - - "Domain" - - "Supergroup" - - "Division" - - "Subdivision" - - "Class" - - "Order" - - "Family" - - "Genus" - - "Species" - - # Optional: offload the memory-intensive assignTaxonomy() step to a remote - # server via SSH. Omit or set host to null to run locally. - # DISCLAIMER: You are solely responsible for ensuring you have authorisation - # to use the configured host. See config/defaults/dada2.yml for the full disclaimer. - remote: - host: ~ # user@hostname - rscript: "Rscript" # path to Rscript on the server - staging_dir: "/absolute/path/on/server" - db_path: ~ # absolute path to database on server (null = transfer local copy) - -# Output filenames (all written to workspace_root/Tables/): -output: - seq_table_prefix: "seqtab_nochim" - fasta_prefix: "asvs" - taxa_prefix: "taxonomy" - combined_filename: "tax_counts.csv" - asv_filename: "asv_counts.csv" +dada2: + file_patterns: + mode: "paired" # paired | forward | reverse + + # Filter and trim - DADA2's filterAndTrim(): + filter_trim: + trunc_q: 2 + trunc_len: [220, 220] # [forward, reverse]; first value used for single-end mode + max_ee: [3, 3] # maximum expected errors in F and R reads + min_len: 175 + max_n: 0 + match_ids: true + rm_phix: true + + # Denoising - learnErrors() and dada(): + dada: + seed: 123 + nbases: 200000000 + max_consist: 15 + pool_method: "pseudo" # none | pseudo | true + + # Merging - mergePairs(), paired mode only: + merge: + min_overlap: 20 + max_mismatch: 0 + trim_overhang: true + + # ASV length filtering and chimera removal: + asv: + band_size_min: 200 # null to skip length filtering + band_size_max: 430 + denovo_method: "consensus" # consensus | pooled | per-sample + + # Taxonomy - assignTaxonomy() against the configured database: + taxonomy: + database: pr2 # key into config/databases.yml + multithread: 4 # threads for assignTaxonomy(); higher values increase memory use + min_boot: 0 # minimum bootstrap confidence to retain (0-100) + levels: + - "Domain" + - "Supergroup" + - "Division" + - "Subdivision" + - "Class" + - "Order" + - "Family" + - "Genus" + - "Species" + + # Optional: offload the memory-intensive assignTaxonomy() step to a remote + # server via SSH. Omit or set host to null to run locally. + # DISCLAIMER: You are solely responsible for ensuring you have authorisation + # to use the configured host. See config/defaults/pipeline.yml for the full disclaimer. + remote: + host: ~ # user@hostname + rscript: "Rscript" # path to Rscript on the server + staging_dir: "/absolute/path/on/server" + db_path: ~ # absolute path to database on server (null = transfer local copy) + + # Output filename prefixes (all written to dada2/Tables/): + output: + seq_table_prefix: "seqtab_nochim" + fasta_prefix: "asvs" + taxa_prefix: "taxonomy" ``` **Outputs written to `projects/{name}/{run}/dada2/Tables/`:** @@ -233,35 +224,38 @@ output: | `asv_counts.csv` | ASV sequences + per-sample counts (no taxonomy) | | `pipeline_stats.csv` | Read counts retained at each pipeline stage | -### Configuring vsearch (`projects/{name}/vsearch.yml`) +### Configuring vsearch (`vsearch:` in `pipeline.yml`) Controls the alignment thresholds used when assigning taxonomy against the reference database. ```yaml -optional_args: "--id 0.75 --query_cov 0.8" +vsearch: + optional_args: "--id 0.75 --query_cov 0.8" ``` `optional_args` is passed verbatim to `vsearch --usearch_global`. Key thresholds: - `--id` - minimum sequence identity (0-1); lower values recover more hits at the cost of specificity - `--query_cov` - minimum fraction of the query that must be aligned; filters partial matches -### Configuring cd-hit-est (`projects/{name}/cdhit.yml`) +### Configuring cd-hit-est (`cdhit:` in `pipeline.yml`) Clustering step that collapses near-identical ASVs before taxonomy assignment. ```yaml -optional_args: "-c 0.9" +cdhit: + optional_args: "-c 0.9" ``` `optional_args` is passed verbatim to `cd-hit-est`. `-c` sets the sequence identity threshold for clustering (default 0.9 = 90%). -### Configuring merge_taxa (`projects/{name}/merge_taxa.yml`) +### Configuring merge_taxa (`merge_taxa:` in `pipeline.yml`) Controls which filter configs are applied when merging taxonomy and count tables. `merged.csv` (unfiltered) is always written; each entry in `filters` produces an additional filtered CSV. ```yaml -filters: - - "protist_filter.yml" # -> merged/protist_filter.csv +merge_taxa: + filters: + - "protist_filter.yml" # -> merged/protist_filter.csv ``` Each entry is a filename relative to `config/filters/`. Remove all entries (or set `filters: []`) to produce only the unfiltered `merged.csv`. diff --git a/config/cdhit.yml b/config/cdhit.yml deleted file mode 100644 index 157b0b3..0000000 --- a/config/cdhit.yml +++ /dev/null @@ -1 +0,0 @@ -optional_args: "-c 0.9" diff --git a/config/cutadapt.yml b/config/cutadapt.yml deleted file mode 100644 index b3a1342..0000000 --- a/config/cutadapt.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. -primer_pairs: - - TarEuk - - Meta2 - -optional_args: "-m 200 --discard-untrimmed" diff --git a/config/defaults/cdhit.yml b/config/defaults/cdhit.yml deleted file mode 100644 index 157b0b3..0000000 --- a/config/defaults/cdhit.yml +++ /dev/null @@ -1 +0,0 @@ -optional_args: "-c 0.9" diff --git a/config/defaults/cutadapt.yml b/config/defaults/cutadapt.yml deleted file mode 100644 index 993e831..0000000 --- a/config/defaults/cutadapt.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. -primer_pairs: - - PrimerPair1 - -optional_args: "-m 200 --discard-untrimmed" diff --git a/config/defaults/dada2.yml b/config/defaults/dada2.yml deleted file mode 100644 index 8042504..0000000 --- a/config/defaults/dada2.yml +++ /dev/null @@ -1,65 +0,0 @@ -file_patterns: - forward: "_R1_trimmed.fastq.gz" - reverse: "_R2_trimmed.fastq.gz" - sample_name_split: "_" - sample_name_index: 1 - mode: "paired" # paired | forward | reverse - -filter_trim: - trunc_q: 2 - trunc_len: [220, 220] # [forward, reverse]; first value used for single-end - max_ee: [3, 3] - min_len: 175 - max_n: 0 - match_ids: true - rm_phix: true - -dada: - seed: 123 - nbases: 200000000 - max_consist: 15 - pool_method: "pseudo" # none | pseudo | true - -merge: - min_overlap: 20 - max_mismatch: 0 - trim_overhang: true - -asv: - band_size_min: 200 # null to skip length filtering - band_size_max: 430 - denovo_method: "consensus" - -taxonomy: - database: pr2 # key into config/databases.yml - multithread: 4 # higher values increase memory use significantly (mclapply forks one process per thread) - # DISCLAIMER: Configuring a remote host causes this pipeline to connect via SSH, - # transfer files, execute Rscript, and delete the staging directory on the remote - # server. You are solely responsible for ensuring you have authorisation to use - # the configured host and that staging_dir is a safe path to write to and delete. - # The authors accept no liability for data loss or misuse arising from this feature. - remote: # omit or set host to null to run locally - host: ~ # user@hostname - rscript: "Rscript" # path to Rscript on the server - staging_dir: "/absolute/path/on/server" # must be an explicit absolute path on the server - db_path: ~ # path to pre-installed database on server; null to transfer local copy - min_boot: 0 - levels: - - "Domain" - - "Supergroup" - - "Division" - - "Subdivision" - - "Class" - - "Order" - - "Family" - - "Genus" - - "Species" - -output: - seq_table_prefix: "seqtab_nochim" - fasta_prefix: "asvs" - taxa_prefix: "taxonomy" - combined_filename: "tax_counts.csv" - asv_filename: "asv_counts.csv" - -verbose: true diff --git a/config/defaults/default_configs.md b/config/defaults/default_configs.md new file mode 100644 index 0000000..d2931c6 --- /dev/null +++ b/config/defaults/default_configs.md @@ -0,0 +1,13 @@ +# Default Configuration + +In this directory is the baseline for every pipeline run on every +machine. It lists every key the pipeline understands and supplies a +sensible starting value for each one. + +Do not edit this file unless you are a contributer. It is the +software's own definition of what a complete configuration looks like. +Changes here will be overwritten on update. + +To change a setting, add it to the appropriate level above this one - +e.g: `config/pipeline.yml` for machine-wide changes, or a project- or +run-level file for narrower scope. diff --git a/config/defaults/merge_taxa.yml b/config/defaults/merge_taxa.yml deleted file mode 100644 index 7d4cd72..0000000 --- a/config/defaults/merge_taxa.yml +++ /dev/null @@ -1,6 +0,0 @@ -# List of filter configs to apply, each relative to config/filters/. -# Each entry produces a separate filtered CSV in the project's merged/ directory. -# The unfiltered merged.csv is always written regardless of this list. -# Set to [] or omit entries to produce merged.csv only. -filters: - - "protist_filter.yml" diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml new file mode 100644 index 0000000..ec56330 --- /dev/null +++ b/config/defaults/pipeline.yml @@ -0,0 +1,80 @@ +cutadapt: + # Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. + primer_pairs: + - PrimerPair1 + optional_args: "-m 200 --discard-untrimmed" + +dada2: + file_patterns: + mode: "paired" # paired | forward | reverse + + filter_trim: + trunc_q: 2 + trunc_len: [220, 220] # [forward, reverse]; first value used for single-end + max_ee: [3, 3] + min_len: 175 + max_n: 0 + match_ids: true + rm_phix: true + + dada: + seed: 123 + nbases: 200000000 + max_consist: 15 + pool_method: "pseudo" # none | pseudo | true + + merge: + min_overlap: 20 + max_mismatch: 0 + trim_overhang: true + + asv: + band_size_min: 200 # null to skip length filtering + band_size_max: 430 + denovo_method: "consensus" + + taxonomy: + database: pr2 # key into config/databases.yml + multithread: 4 # higher values increase memory use significantly + # DISCLAIMER: Configuring a remote host causes this pipeline to connect via SSH, + # transfer files, execute Rscript, and delete the staging directory on the remote + # server. You are solely responsible for ensuring you have authorisation to use + # the configured host and that staging_dir is a safe path to write to and delete. + # The authors accept no liability for data loss or misuse arising from this feature. + remote: # omit or set host to null to run locally + host: ~ # user@hostname + rscript: "Rscript" # path to Rscript on the server + staging_dir: "/absolute/path/on/server" + db_path: ~ # path to pre-installed database on server; null to transfer local copy + min_boot: 0 + levels: + - "Domain" + - "Supergroup" + - "Division" + - "Subdivision" + - "Class" + - "Order" + - "Family" + - "Genus" + - "Species" + + output: + seq_table_prefix: "seqtab_nochim" + fasta_prefix: "asvs" + taxa_prefix: "taxonomy" + + verbose: true + +vsearch: + optional_args: "--id 0.75 --query_cov 0.8" + +cdhit: + optional_args: "-c 0.9" + +merge_taxa: + # List of filter configs to apply, each relative to config/filters/. + # Each entry produces a separate filtered CSV in the project's merged/ directory. + # The unfiltered merged.csv is always written regardless of this list. + # Set to [] or omit entries to produce merged.csv only. + filters: + - "protist_filter.yml" diff --git a/config/defaults/primers.yml b/config/defaults/primers.yml new file mode 100644 index 0000000..303a0b4 --- /dev/null +++ b/config/defaults/primers.yml @@ -0,0 +1,30 @@ +# Kandaurova, E. (2025) ‘Metabarcoding of gut protists of Eurasian beaver’. +# Owens, L.A., Friant, S., Martorelli Di Genova, B., Knoll, L.J., Contreras, M., Noya-Alarcon, O., Dominguez-Bello, M.G., and Goldberg, T.L. (2024) ‘VESPA: an optimized protocol for accurate metabarcoding-based characterization of vertebrate eukaryotic endosymbiont and parasite assemblages’, Nature Communications, 15(1), 402, available: https://doi.org/10.1038/s41467-023-44521-3. + +Forward: + TarEukF: "CCAGCASCYGCGGTAATTCC" + ArchF: "YGCGGTAAYTCCAGCTC" + VESPAF: "AGCAGCCGCGGTAATTCC" + +Reverse: + TarEukR: "ACTTTCGTTCTTGATYRA" + Meta1R: "CYCCTACYYTMGYYCTKGA" + Meta2R: "DCTKTCGTYCTTGATYRA" + VESPAR: "TCCGTCAATTYCTTNAASTTTC" + +Pairs: + - TarEuk: + - TarEukF + - TarEukR + - Meta1: + - TarEukF + - Meta1R + - Meta2: + - TarEukF + - Meta2R + - Arch: + - ArchF + - TarEukR + - VESPA: + - VESPAF + - VESPAR \ No newline at end of file diff --git a/config/defaults/vsearch.yml b/config/defaults/vsearch.yml deleted file mode 100644 index e31b1d7..0000000 --- a/config/defaults/vsearch.yml +++ /dev/null @@ -1 +0,0 @@ -optional_args: "--id 0.75 --query_cov 0.8" diff --git a/config/global_configs.md b/config/global_configs.md new file mode 100644 index 0000000..a077b49 --- /dev/null +++ b/config/global_configs.md @@ -0,0 +1,17 @@ +# Global Configuration + +In this directory is the machine-level override. Anything written here +applies to every study run on this installation, regardless of project. + +This is the right place for settings that reflect the environment: which +remote server handles taxonomy assignment, how many threads this machine +can spare, as well as primers available to be picked by cutadapt. If you +are a contributer, your configurations here will not be committed. + +Leave a key out entirely to inherit the default. Only write what you +deliberately want to change. + +Note that this is the lowest-precedence level after the defaults. Any +setting written here can be overridden at the project, group, or run +level, this simply serves as the fallback for every run that does not +explicitly override it. diff --git a/config/merge_taxa.yml b/config/merge_taxa.yml deleted file mode 100644 index 72821b7..0000000 --- a/config/merge_taxa.yml +++ /dev/null @@ -1,3 +0,0 @@ -# Filename of filter config relative to config/filters/. -# Leave empty or omit to produce merged.csv only (no filtering). -filter: "protist_filter.yml" diff --git a/config/primers.yml b/config/primers.yml deleted file mode 100644 index 53a0027..0000000 --- a/config/primers.yml +++ /dev/null @@ -1,27 +0,0 @@ -Forward: - TarEukF: "CCAGCASCYGCGGTAATTCC" - ArchF: "YGCGGTAAYTCCAGCTC" - VESPAF: "AGCAGCCGCGGTAATTCC" - -Reverse: - TarEukR: "ACTTTCGTTCTTGATYRA" - Meta1R: "CYCCTACYYTMGYYCTKGA" - Meta2R: "DCTKTCGTYCTTGATYRA" - VESPAR: "TCCGTCAATTYCTTNAASTTTC" - -Pairs: - - TarEuk: - - TarEukF - - TarEukR - - Meta1: - - TarEukF - - Meta1R - - Meta2: - - TarEukF - - Meta2R - - Arch: - - ArchF - - TarEukR - - VESPA: - - VESPAF - - VESPAR \ No newline at end of file diff --git a/config/vsearch.yml b/config/vsearch.yml deleted file mode 100644 index e31b1d7..0000000 --- a/config/vsearch.yml +++ /dev/null @@ -1 +0,0 @@ -optional_args: "--id 0.75 --query_cov 0.8" diff --git a/src/call_tools.jl b/src/call_tools.jl index 1c662e3..b8d3838 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -9,6 +9,7 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit using YAML using Logging using ..PipelineTypes + using ..Config # Run cmd_str via bash, capturing stdout+stderr to log_path. # On failure, prints the log to stderr before rethrowing so the error is visible. @@ -371,54 +372,64 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit function cutadapt(project::ProjectCtx; cutadapt_bin = tool_bin("cutadapt")) - config_path = joinpath(project.dir, "cutadapt.yml") + config_path = write_run_config(project) primers_path = joinpath(project.config_dir, "primers.yml") - cfg = YAML.load_file(config_path) + cfg = get(YAML.load_file(config_path), "cutadapt", Dict()) primer_pairs = cfg["primer_pairs"] optional_args = get(cfg, "optional_args", "-m 200 --discard-untrimmed") cutadapt_dir = joinpath(project.dir, "cutadapt") + hash_file = joinpath(cutadapt_dir, "config.hash") if isdir(cutadapt_dir) - trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) - cfg_mtime = max(mtime(config_path), mtime(primers_path)) - if !isempty(trimmed) && all(f -> mtime(joinpath(cutadapt_dir, f)) > cfg_mtime, trimmed) + trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) + if !isempty(trimmed) && + !_section_stale(config_path, "cutadapt", hash_file) && + all(f -> mtime(joinpath(cutadapt_dir, f)) > mtime(primers_path), trimmed) @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" return TrimmedReads(cutadapt_dir) end end + mkpath(cutadapt_dir) run_cutadapt(get_primer_args(primer_pairs, primers_path), optional_args, project.data_dir, cutadapt_dir, cutadapt_bin) + _write_section_hash(config_path, "cutadapt", hash_file) return TrimmedReads(cutadapt_dir) end function vsearch(project::ProjectCtx, input::HasFasta, reference_database::String; vsearch_bin = tool_bin("vsearch")) - config_path = joinpath(project.dir, "vsearch.yml") - cfg = YAML.load_file(config_path) + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "vsearch", Dict()) optional_args = get(cfg, "optional_args", "--id 0.75 --query_cov 0.8") - vsearch_dir = joinpath(project.dir, "vsearch") - tsv = joinpath(vsearch_dir, "taxonomy.tsv") - cfg_mtime = max(mtime(config_path), mtime(input.fasta)) - if isfile(tsv) && mtime(tsv) > cfg_mtime + vsearch_dir = joinpath(project.dir, "vsearch") + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + hash_file = joinpath(vsearch_dir, "config.hash") + if isfile(tsv) && + !_section_stale(config_path, "vsearch", hash_file) && + mtime(tsv) > mtime(input.fasta) @info "Skipping vsearch: $tsv up to date" return TaxonomyHits(tsv) end vsearch(input.fasta, reference_database, vsearch_dir; optional_args, vsearch_bin) + _write_section_hash(config_path, "vsearch", hash_file) return TaxonomyHits(tsv) end function cdhit(project::ProjectCtx, input::ASVResult; cdhit_bin = tool_bin("cd_hit_est")) - config_path = joinpath(project.dir, "cdhit.yml") - cfg = YAML.load_file(config_path) + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "cdhit", Dict()) optional_args = get(cfg, "optional_args", "-c 0.9") - cdhit_dir = joinpath(project.dir, "cdhit") - new_fasta = joinpath(cdhit_dir, basename(input.fasta)) - cfg_mtime = max(mtime(config_path), mtime(input.fasta)) - if isfile(new_fasta) && mtime(new_fasta) > cfg_mtime + cdhit_dir = joinpath(project.dir, "cdhit") + new_fasta = joinpath(cdhit_dir, basename(input.fasta)) + hash_file = joinpath(cdhit_dir, "config.hash") + if isfile(new_fasta) && + !_section_stale(config_path, "cdhit", hash_file) && + mtime(new_fasta) > mtime(input.fasta) @info "Skipping cdhit: $new_fasta up to date" return ASVResult(new_fasta, input.count_table, input.taxonomy) end new_fasta = cdhit(input.fasta, cdhit_dir; optional_args, cdhit_bin) + _write_section_hash(config_path, "cdhit", hash_file) return ASVResult(new_fasta, input.count_table, input.taxonomy) end end diff --git a/src/config.jl b/src/config.jl new file mode 100644 index 0000000..c630599 --- /dev/null +++ b/src/config.jl @@ -0,0 +1,165 @@ +module Config + +# Hierarchical config cascade and per-section content-hash helpers. +# +# Config cascade (global -> project -> run): +# config/defaults/pipeline.yml <- full defaults (source of truth) +# config/pipeline.yml <- machine-level overrides +# projects/{study}/pipeline.yml <- study-level overrides +# ...intermediate dirs... +# projects/{study}/{run}/pipeline.yml <- run-level overrides +# +# Each override file contains only intentional changes; omitted keys are +# inherited from the nearest ancestor. At the start of each run, all levels +# are deep-merged into a single Dict and written to: +# projects/{study}/{run}/run_config.yml +# +# Stage skip guards hash the relevant YAML section from run_config.yml so +# that any change at any level -- global, study, or run -- correctly +# invalidates downstream checkpoints. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export _section_stale, _write_section_hash, + load_merged_config, write_run_config + + using SHA, YAML + using ..PipelineTypes + + # Deep merge + # Recursively merge `patch` into `base`. Dict values are merged recursively; + # all other types (including Arrays) are replaced by the patch value. + function _deep_merge(base::Dict, patch::Dict) + result = copy(base) + for (k, v) in patch + result[k] = (haskey(result, k) && result[k] isa Dict && v isa Dict) ? + _deep_merge(result[k], v) : v + end + return result + end + + # Cascade path discovery + # Return the ordered list of pipeline.yml paths participating in the cascade, + # from least specific (defaults) to most specific (leaf project dir). + function _cascade_paths(config_dir::String, study_dir::String, project_dir::String) + paths = String[ + joinpath(config_dir, "defaults", "pipeline.yml"), + joinpath(config_dir, "pipeline.yml"), + ] + + study_norm = normpath(abspath(study_dir)) + project_norm = normpath(abspath(project_dir)) + + if study_norm == project_norm + push!(paths, joinpath(project_norm, "pipeline.yml")) + else + rel = relpath(project_norm, study_norm) + parts = splitpath(rel) + current = study_norm + push!(paths, joinpath(current, "pipeline.yml")) + for part in parts + current = joinpath(current, part) + push!(paths, joinpath(current, "pipeline.yml")) + end + end + + return paths + end + + # Config loading + # Merge all config levels in `paths` (ordered global -> specific). + # paths[1] must exist (the defaults file). Subsequent files are optional; + # missing files and files that parse as nothing/empty are skipped. + function load_merged_config(paths::Vector{String}) + isfile(paths[1]) || error("Default config not found: $(paths[1])") + base = something(YAML.load_file(paths[1]), Dict()) + base isa Dict || (base = Dict()) + for path in paths[2:end] + isfile(path) || continue + patch = YAML.load_file(path) + isnothing(patch) && continue + patch isa Dict || continue + isempty(patch) && continue + base = _deep_merge(base, patch) + end + return base + end + + load_merged_config(config_dir::String, study_dir::String, project_dir::String) = + load_merged_config(_cascade_paths(config_dir, study_dir, project_dir)) + + load_merged_config(project::ProjectCtx) = + load_merged_config(project.config_dir, project.study_dir, project.dir) + + # run_config.yml + # Merge all cascade levels and write the result to {project.dir}/run_config.yml. + # Regenerates only when a source file is newer than the existing run_config.yml. + # Returns the path to run_config.yml. + function write_run_config(project::ProjectCtx) + run_config_path = joinpath(project.dir, "run_config.yml") + primers_path = joinpath(project.config_dir, "primers.yml") + paths = _cascade_paths(project.config_dir, project.study_dir, project.dir) + source_paths = isfile(primers_path) ? vcat(paths, primers_path) : paths + if !isfile(run_config_path) || + any(p -> isfile(p) && mtime(p) > mtime(run_config_path), source_paths) + merged = load_merged_config(paths) + if isfile(primers_path) + primers = YAML.load_file(primers_path) + isnothing(primers) || (merged["primers"] = primers) + end + open(run_config_path, "w") do io + print(io, YAML.write(merged)) + end + end + return run_config_path + end + + # Section content hashes + # Produce a stable, canonical string from a YAML-loaded value. + # Dicts are sorted by key so insertion-order differences do not affect the hash. + function _canonical(x)::String + if x isa AbstractDict + pairs_sorted = sort([(string(k), _canonical(v)) for (k, v) in x]; by = p -> p[1]) + return "{" * join(["$(p[1]):$(p[2])" for p in pairs_sorted], ",") * "}" + elseif x isa AbstractVector + return "[" * join(map(_canonical, x), ",") * "]" + elseif isnothing(x) + return "null" + else + return string(x) + end + end + + function _section_hash(config_path::String, section::String)::String + cfg = YAML.load_file(config_path) + cfg isa Dict || return bytes2hex(sha256("")) + sec = get(cfg, section, Dict()) + bytes2hex(sha256(_canonical(sec))) + end + + """ + _section_stale(config_path, section, hash_file) -> Bool + + Return `true` if the named section of `config_path` has changed since + `hash_file` was last written, or if `hash_file` does not yet exist. + Pass `run_config.yml` as `config_path` to hash the fully-merged section. + """ + function _section_stale(config_path::String, section::String, hash_file::String)::Bool + !isfile(hash_file) && return true + stored = strip(read(hash_file, String)) + return _section_hash(config_path, section) != stored + end + + """ + _write_section_hash(config_path, section, hash_file) + + Write the current SHA-256 hash of the named section to `hash_file`. + Call this after a stage completes successfully. + """ + function _write_section_hash(config_path::String, section::String, hash_file::String) + write(hash_file, _section_hash(config_path, section)) + end + +end diff --git a/src/dada2.jl b/src/dada2.jl index bf5e51d..c6a6d99 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -25,12 +25,15 @@ module DADA2 # Commons Attribution 4.0 International License (CC BY 4.0): # https://creativecommons.org/licenses/by/4.0/. -export dada2, prefilter_qc, filter_trim, learn_errors, denoise, +export dada2, dada2_denoise, dada2_classify, + prefilter_qc, filter_trim, learn_errors, denoise, filter_length, chimera_removal, assign_taxonomy import Downloads using Logging, RCall, YAML using ..PipelineTypes + using ..Databases + using ..Config include("dada2/context.jl") # shared helpers and _pipeline_context include("dada2/qc.jl") # prefilter_qc, filter_trim @@ -80,11 +83,73 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, assign_taxonomy(config_path; progress, input_dir, workspace_root, taxonomy_db) end + """ + dada2_denoise(project, trimmed; progress) -> DenoisedASVs + + Run all DADA2 stages up to and including chimera removal. Returns a + `DenoisedASVs` wire type that can be passed to `dada2_classify()`. + + Skips automatically if `ckpt_chimera.RData` is newer than both `dada2.yml` + and the trimmed reads. Individual stage skip guards apply for partial runs. + """ + function dada2_denoise(project::ProjectCtx, trimmed::TrimmedReads; progress=nothing) + config_path = write_run_config(project) + workspace_root = joinpath(project.dir, "dada2") + chimera_ckpt = joinpath(workspace_root, "Checkpoints", "ckpt_chimera.RData") + hash_file = joinpath(workspace_root, "Checkpoints", "config.hash") + + trimmed_files = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(trimmed.dir)) + trimmed_mtime = isempty(trimmed_files) ? 0.0 : + maximum(mtime(joinpath(trimmed.dir, f)) for f in trimmed_files) + + if isfile(chimera_ckpt) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(chimera_ckpt) > trimmed_mtime + @info "Skipping dada2_denoise: ckpt_chimera.RData up to date" + else + R"rm(list=ls())" + prefilter_qc(config_path; progress, input_dir=trimmed.dir, workspace_root) + filter_trim(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + learn_errors(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + denoise(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + filter_length(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + chimera_removal(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + end + + return DenoisedASVs(chimera_ckpt, config_path, workspace_root, trimmed.dir) + end + + """ + dada2_classify(project, denoised; taxonomy_db, progress) -> ASVResult + + Run taxonomy assignment on a `DenoisedASVs` result and return an `ASVResult`. + Respects the mtime skip guard inside `assign_taxonomy()`. + """ + function dada2_classify(project::ProjectCtx, denoised::DenoisedASVs; + taxonomy_db=nothing, progress=nothing) + config_path = denoised.config_path + workspace_root = denoised.workspace_root + cfg = get(YAML.load_file(config_path), "dada2", Dict()) + out_cfg = get(cfg, "output", Dict()) + tables_dir = joinpath(workspace_root, "Tables") + result = ASVResult( + joinpath(tables_dir, get(out_cfg, "fasta_prefix", "asvs") * ".fasta"), + joinpath(tables_dir, get(out_cfg, "seq_table_prefix", "seqtab_nochim") * ".csv"), + joinpath(tables_dir, get(out_cfg, "taxa_prefix", "taxonomy") * ".csv") + ) + assign_taxonomy(config_path; + progress, + input_dir = denoised.input_dir, + workspace_root, + taxonomy_db) + return result + end + function dada2(project::ProjectCtx, trimmed::TrimmedReads; taxonomy_db=nothing, progress=nothing) - config_path = joinpath(project.dir, "dada2.yml") + config_path = write_run_config(project) workspace_root = joinpath(project.dir, "dada2") - cfg = YAML.load_file(config_path) + cfg = get(YAML.load_file(config_path), "dada2", Dict()) out_cfg = get(cfg, "output", Dict()) tables_dir = joinpath(workspace_root, "Tables") result = ASVResult( @@ -92,21 +157,18 @@ export dada2, prefilter_qc, filter_trim, learn_errors, denoise, joinpath(tables_dir, get(out_cfg, "seq_table_prefix", "seqtab_nochim") * ".csv"), joinpath(tables_dir, get(out_cfg, "taxa_prefix", "taxonomy") * ".csv") ) + hash_file = joinpath(workspace_root, "Checkpoints", "config.hash") trimmed_files = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(trimmed.dir)) trimmed_mtime = isempty(trimmed_files) ? 0.0 : maximum(mtime(joinpath(trimmed.dir, f)) for f in trimmed_files) - input_mtime = max(mtime(config_path), trimmed_mtime) if all(isfile, (result.fasta, result.count_table, result.taxonomy)) && - all(f -> mtime(f) > input_mtime, (result.fasta, result.count_table, result.taxonomy)) + !_section_stale(config_path, "dada2", hash_file) && + all(f -> mtime(f) > trimmed_mtime, (result.fasta, result.count_table, result.taxonomy)) @info "Skipping dada2: outputs up to date in $tables_dir" return result end - dada2(config_path; - input_dir = trimmed.dir, - workspace_root, - taxonomy_db, - progress) - return result + denoised = dada2_denoise(project, trimmed; progress) + return dada2_classify(project, denoised; taxonomy_db, progress) end end diff --git a/src/dada2/chimera.jl b/src/dada2/chimera.jl index 1175b98..5ae075a 100644 --- a/src/dada2/chimera.jl +++ b/src/dada2/chimera.jl @@ -29,14 +29,17 @@ length_ckpt = ctx.ckpts["length"] chimera_ckpt = ctx.ckpts["chimera"] - if isfile(chimera_ckpt) - input_mtime = max(mtime(config_path), mtime(filter_ckpt), mtime(length_ckpt)) - if mtime(chimera_ckpt) > input_mtime - @info "Skipping chimera_removal: checkpoint up to date" - return nothing - end + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(chimera_ckpt) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(chimera_ckpt) > mtime(filter_ckpt) && + mtime(chimera_ckpt) > mtime(length_ckpt) + @info "Skipping chimera_removal: checkpoint up to date" + return nothing end + R"rm(list=ls())" + _source_r_functions() seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") tables_dir = ctx.dirs["Tables"] @@ -99,6 +102,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(tables_dir, "pipeline_stats.csv"))") emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") diff --git a/src/dada2/context.jl b/src/dada2/context.jl index d98cc01..3e7c541 100644 --- a/src/dada2/context.jl +++ b/src/dada2/context.jl @@ -10,50 +10,13 @@ # Database helpers - # Download uri to db_dir/basename(uri) if not already cached. - # Respects the optional local: override in fmt_info. - function _download_db_if_needed(key, fmt_info, db_dir, emit) - local_p = get(fmt_info, "local", nothing) - if !isnothing(local_p) - local_p = string(local_p) - if !isempty(local_p) - isfile(local_p) && (emit("[$key] Using local file: $local_p"); return local_p) - @warn "[$key] Configured local path not found: $local_p - falling back to uri" - end - end - uri = fmt_info["uri"] - cached = joinpath(db_dir, basename(uri)) - if isfile(cached) - emit("[$key] Using cached: $cached") - else - emit("[$key] Downloading: $uri") - Downloads.download(uri, cached) - emit("[$key] Saved to: $cached") - end - return cached - end - - # Resolve the DADA2 taxonomy database. - # Reads from global config/databases.yml; falls back to pipeline config's - # databases: section for backward compatibility with old per-project configs. + # Resolve the DADA2 taxonomy database from config/databases.yml. function _resolve_taxonomy_db(cfg, emit) - db_key = string(cfg["taxonomy"]["database"]) - global_dbs = joinpath(@__DIR__, "..", "..", "config", "databases.yml") - - db_source = if isfile(global_dbs) - get(YAML.load_file(global_dbs), "databases", Dict()) - else - get(cfg, "databases", Dict()) # backward compat - end - - haskey(db_source, db_key) || - error("taxonomy.database = \"$db_key\" not found in databases config") - fmt_cfg = get(db_source[db_key], "dada2", nothing) - isnothing(fmt_cfg) && - error("databases.$db_key.dada2 is not configured in config/databases.yml") - db_dir = abspath(get(db_source, "dir", "./databases")) - mkpath(db_dir) - return _download_db_if_needed("$(db_key)_dada2", fmt_cfg, db_dir, emit) + db_key = string(cfg["taxonomy"]["database"]) + dbs_path = joinpath(@__DIR__, "..", "..", "config", "databases.yml") + isfile(dbs_path) || + error("config/databases.yml not found. Run new_project() first.") + return resolve_db(dbs_path, db_key, "dada2"; emit) end # Config @@ -124,9 +87,17 @@ dirs end + # R function loader + # Source the R helper functions into the current R session. Called by each + # stage after its mtime skip check, so R is not loaded for skipped stages. + function _source_r_functions() + functions_r = joinpath(@__DIR__, "dada2_functions.r") + R"source($functions_r)" + end + # Pipeline context - # Shared setup called at the start of every stage: sources R functions, loads - # and validates config, discovers files, handles the single-sample fallback, and + # Shared pure-Julia setup called at the start of every stage: loads and + # validates config, discovers files, handles the single-sample fallback, and # computes all path variables. Returns a NamedTuple so stage functions can # extract what they need without repeating boilerplate. # @@ -134,10 +105,7 @@ # input_dir - overrides cfg["workspace"]["input_dir"] # workspace_root - overrides cfg["workspace"]["root"] function _pipeline_context(config_path::String; input_dir=nothing, workspace_root=nothing) - functions_r = joinpath(@__DIR__, "dada2_functions.r") - R"source($functions_r)" - - cfg = YAML.load_file(config_path) + cfg = get(YAML.load_file(config_path), "dada2", Dict{String,Any}()) isnothing(input_dir) && error("input_dir must be provided") isnothing(workspace_root) && error("workspace_root must be provided") @@ -151,16 +119,13 @@ fwd_files, rev_files = find_fastq_files( input_dir, - cfg["file_patterns"]["forward"], - cfg["file_patterns"]["reverse"], + "_R1_trimmed.fastq.gz", + "_R2_trimmed.fastq.gz", mode) validate_sample_files(fwd_files, rev_files, mode) primary_files = isempty(fwd_files) ? rev_files : fwd_files - sample_names = extract_sample_names( - primary_files, - cfg["file_patterns"]["sample_name_split"], - cfg["file_patterns"]["sample_name_index"]) + sample_names = extract_sample_names(primary_files, "_", 1) # Single-sample fallback: dada() returns a bare object (not a list) for a # single input file, breaking makeSequenceTable() and sapply() downstream. diff --git a/src/dada2/denoise.jl b/src/dada2/denoise.jl index 162808d..a639e96 100644 --- a/src/dada2/denoise.jl +++ b/src/dada2/denoise.jl @@ -19,14 +19,16 @@ errors_ckpt = ctx.ckpts["errors"] filter_ckpt = ctx.ckpts["filter"] - if isfile(errors_ckpt) && isfile(filter_ckpt) - input_mtime = max(mtime(config_path), mtime(filter_ckpt)) - if mtime(errors_ckpt) > input_mtime - @info "Skipping learn_errors: checkpoint up to date" - return nothing - end + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(errors_ckpt) && isfile(filter_ckpt) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(errors_ckpt) > mtime(filter_ckpt) + @info "Skipping learn_errors: checkpoint up to date" + return nothing end + R"rm(list=ls())" + _source_r_functions() seed = get(ctx.cfg["dada"], "seed", 123) nbases = ctx.cfg["dada"]["nbases"] max_con = ctx.cfg["dada"]["max_consist"] @@ -57,6 +59,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "error_rates.pdf"))") emit("Checkpoint: $(ctx.ckpts["errors"])") emit("Log: $log_path") @@ -89,14 +92,16 @@ errors_ckpt = ctx.ckpts["errors"] denoise_ckpt = ctx.ckpts["denoise"] - if isfile(denoise_ckpt) - input_mtime = max(mtime(config_path), mtime(errors_ckpt)) - if mtime(denoise_ckpt) > input_mtime - @info "Skipping denoise: checkpoint up to date" - return nothing - end + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(denoise_ckpt) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(denoise_ckpt) > mtime(errors_ckpt) + @info "Skipping denoise: checkpoint up to date" + return nothing end + R"rm(list=ls())" + _source_r_functions() log_path = joinpath(ctx.dirs["Logs"], "denoise.log") open(log_path, "w") do io; println(io, "=== denoise ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" @@ -151,6 +156,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "length_distribution.pdf"))") emit("Checkpoint: $(ctx.ckpts["denoise"])") emit("Log: $log_path") @@ -180,14 +186,16 @@ denoise_ckpt = ctx.ckpts["denoise"] length_ckpt = ctx.ckpts["length"] - if isfile(length_ckpt) - input_mtime = max(mtime(config_path), mtime(denoise_ckpt)) - if mtime(length_ckpt) > input_mtime - @info "Skipping filter_length: checkpoint up to date" - return nothing - end + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(length_ckpt) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(length_ckpt) > mtime(denoise_ckpt) + @info "Skipping filter_length: checkpoint up to date" + return nothing end + R"rm(list=ls())" + _source_r_functions() log_path = joinpath(ctx.dirs["Logs"], "filter_length.log") open(log_path, "w") do io; println(io, "=== filter_length ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" @@ -217,6 +225,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Checkpoint: $(ctx.ckpts["length"])") emit("Log: $log_path") nothing diff --git a/src/dada2/qc.jl b/src/dada2/qc.jl index 17326d2..2b680a7 100644 --- a/src/dada2/qc.jl +++ b/src/dada2/qc.jl @@ -15,16 +15,17 @@ ctx = _pipeline_context(config_path; input_dir, workspace_root) unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") - if isfile(unfiltered_pdf) - all_inputs = vcat(ctx.fwd_files, ctx.rev_files) - input_mtime = isempty(all_inputs) ? mtime(config_path) : - max(mtime(config_path), maximum(mtime(f) for f in all_inputs)) - if mtime(unfiltered_pdf) > input_mtime + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(unfiltered_pdf) && !_section_stale(config_path, "dada2", hash_file) + all_inputs = vcat(ctx.fwd_files, ctx.rev_files) + if isempty(all_inputs) || all(f -> mtime(unfiltered_pdf) > mtime(f) for f in all_inputs) @info "Skipping prefilter_qc: quality_unfiltered.pdf up to date" return nothing end end + R"rm(list=ls())" + _source_r_functions() log_path = joinpath(ctx.dirs["Logs"], "prefilter_qc.log") open(log_path, "w") do io; println(io, "=== prefilter_qc ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" @@ -38,6 +39,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf"))") emit("Log: $log_path") nothing @@ -60,16 +62,17 @@ ctx = _pipeline_context(config_path; input_dir, workspace_root) filter_ckpt = ctx.ckpts["filter"] - if isfile(filter_ckpt) - all_inputs = vcat(ctx.fwd_files, ctx.rev_files) - input_mtime = isempty(all_inputs) ? mtime(config_path) : - max(mtime(config_path), maximum(mtime(f) for f in all_inputs)) - if mtime(filter_ckpt) > input_mtime + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(filter_ckpt) && !_section_stale(config_path, "dada2", hash_file) + all_inputs = vcat(ctx.fwd_files, ctx.rev_files) + if isempty(all_inputs) || all(f -> mtime(filter_ckpt) > mtime(f) for f in all_inputs) @info "Skipping filter_trim: checkpoint up to date" return nothing end end + R"rm(list=ls())" + _source_r_functions() ft = ctx.cfg["filter_trim"] trunc_len = ft["trunc_len"] max_ee = ft["max_ee"] @@ -127,6 +130,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end + _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_filtered.pdf"))") emit("Checkpoint: $(ctx.ckpts["filter"])") emit("Log: $log_path") diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index 72b0ff9..f9b9d43 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -44,10 +44,15 @@ # Use a ControlMaster socket so the password is entered once and all # subsequent ssh/scp calls reuse the existing connection silently. - ctl = "/tmp/ssh_mux_$run_id" - ssh = (args...) -> `ssh -o ControlMaster=auto -o ControlPath=$ctl -o ControlPersist=yes $args` - scp = (args...) -> `scp -o ControlMaster=auto -o ControlPath=$ctl -q $args` - + # ConnectTimeout: fail fast instead of hanging on unreachable hosts. + # NumberOfPasswordPrompts=1: fail immediately on wrong password. + # ServerAlive*: detect stale connections during long Rscript runs. + ctl = "/tmp/ssh_mux_$run_id" + ssh_opts = `-o ControlMaster=auto -o ControlPath=$ctl -o ControlPersist=yes -o ConnectTimeout=15 -o NumberOfPasswordPrompts=1 -o ServerAliveInterval=30 -o ServerAliveCountMax=3` + ssh = (args...) -> `ssh $ssh_opts $args` + scp = (args...) -> `scp $ssh_opts $args` + + emit(" Connecting to $host (enter SSH password if prompted)...") emit(" Setting up staging directory on $host") run(ssh(host, "mkdir -p $remote_tables")) @@ -68,6 +73,8 @@ levels_str = join(tax_levels, ",") verbose_str = verbose ? "true" : "false" + # Julia Bool true/false -> R TRUE/FALSE; integers pass through as-is. + mt_str = multithread isa Bool ? (multithread ? "TRUE" : "FALSE") : string(multithread) remote_cmd = "$rscript $staging_dir/taxonomy_remote.r " * "functions=$staging_dir/dada2_functions.r " * "ckpt=$staging_dir/ckpt_chimera.RData " * @@ -75,7 +82,7 @@ "tables=$remote_tables " * "save=$remote_ckpt " * "prefix=$taxa_prefix " * - "multithread=$multithread " * + "multithread=$mt_str " * "min_boot=$min_boot " * "levels=$levels_str " * "verbose=$verbose_str" @@ -114,12 +121,12 @@ chimera_ckpt = ctx.ckpts["chimera"] checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") - if isfile(checkpoint) - input_mtime = max(mtime(config_path), mtime(chimera_ckpt)) - if mtime(checkpoint) > input_mtime - @info "Skipping assign_taxonomy: checkpoint up to date" - return nothing - end + hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + if isfile(checkpoint) && + !_section_stale(config_path, "dada2", hash_file) && + mtime(checkpoint) > mtime(chimera_ckpt) + @info "Skipping assign_taxonomy: checkpoint up to date" + return nothing end # Drop all data objects accumulated from prior stages before the @@ -133,12 +140,13 @@ rm(.data_objs) gc() """ + _source_r_functions() seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") taxa_prefix = get(ctx.cfg["output"], "taxa_prefix", "taxonomy") - combined_file = get(ctx.cfg["output"], "combined_filename", "tax_counts.csv") - asv_file = get(ctx.cfg["output"], "asv_filename", "asv_counts.csv") + combined_file = "tax_counts.csv" + asv_file = "asv_counts.csv" tables_dir = ctx.dirs["Tables"] multithread = get(ctx.cfg["taxonomy"], "multithread", 4) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) @@ -197,6 +205,7 @@ emit("Log: $log_path") end + _write_section_hash(config_path, "dada2", hash_file) emit("Checkpoint: $checkpoint") emit("Pipeline complete. Outputs:") diff --git a/src/dada2/taxonomy_remote.r b/src/dada2/taxonomy_remote.r index 7b79820..f7bb08c 100644 --- a/src/dada2/taxonomy_remote.r +++ b/src/dada2/taxonomy_remote.r @@ -34,7 +34,11 @@ load(p[["ckpt"]]) dir.create(p[["tables"]], recursive = TRUE, showWarnings = FALSE) verbose <- tolower(p[["verbose"]]) == "true" -multithread <- as.integer(p[["multithread"]]) +multithread <- if (tolower(p[["multithread"]]) %in% c("true", "false")) { + tolower(p[["multithread"]]) == "true" +} else { + as.integer(p[["multithread"]]) +} min_boot <- as.integer(p[["min_boot"]]) levels <- strsplit(p[["levels"]], ",", fixed = TRUE)[[1]] diff --git a/src/databases.jl b/src/databases.jl index d3733c1..60a7f59 100644 --- a/src/databases.jl +++ b/src/databases.jl @@ -19,7 +19,7 @@ module Databases import Downloads using YAML, Logging -export ensure_databases +export ensure_databases, resolve_db """ ensure_databases(config_path) -> Dict{String,String} @@ -64,13 +64,41 @@ export ensure_databases return resolved end - function _resolve_entry(key, fmt_info, db_dir) + """ + resolve_db(config_path, db_name, fmt; emit=nothing) -> String + + Resolve a single database entry from a databases.yml file. + + `config_path` is the path to a databases.yml-style config, `db_name` is the + key under `databases:` (e.g., `"pr2"`), and `fmt` is the format sub-key + (e.g., `"dada2"` or `"vsearch"`). Returns the resolved absolute local path. + + Pass an `emit` function (e.g., from `_emitter`) to route log messages through + the pipeline's progress channel instead of the default `@info` logger. + """ + function resolve_db(config_path::String, db_name::String, fmt::String; emit=nothing) + cfg = YAML.load_file(config_path) + db_cfg = get(cfg, "databases", Dict()) + db_dir = abspath(get(db_cfg, "dir", "./databases")) + mkpath(db_dir) + + haskey(db_cfg, db_name) || + error("Database '$db_name' not found in $config_path") + fmt_cfg = get(db_cfg[db_name], fmt, nothing) + isnothing(fmt_cfg) && + error("databases.$db_name.$fmt is not configured in $config_path") + + _resolve_entry("$(db_name)_$(fmt)", fmt_cfg, db_dir; emit) + end + + function _resolve_entry(key, fmt_info, db_dir; emit=nothing) + log = isnothing(emit) ? msg -> @info(msg) : emit local_p = get(fmt_info, "local", nothing) if !isnothing(local_p) local_p = string(local_p) if !isempty(local_p) if isfile(local_p) - @info "[$key] Using local file: $local_p" + log("[$key] Using local file: $local_p") return local_p end @warn "[$key] Configured local path not found: $local_p - falling back to uri" @@ -83,11 +111,11 @@ export ensure_databases cached = joinpath(db_dir, basename(uri)) if isfile(cached) - @info "[$key] Using cached: $cached" + log("[$key] Using cached: $cached") else - @info "[$key] Downloading: $uri" + log("[$key] Downloading: $uri") Downloads.download(uri, cached) - @info "[$key] Saved to: $cached" + log("[$key] Saved to: $cached") end return cached end diff --git a/src/main.jl b/src/main.jl index 3dd6c49..7a34d17 100755 --- a/src/main.jl +++ b/src/main.jl @@ -1,6 +1,7 @@ #!/usr/bin/env julia include("types.jl") +include("config.jl") include("databases.jl") include("call_tools.jl") include("dada2.jl") @@ -9,7 +10,7 @@ include("project.jl") using CSV using YAML -using .PipelineTypes, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup +using .PipelineTypes, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup ## Instantiate parameters config_dir = "./config" diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index d6711e7..4bcfd2b 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -6,6 +6,7 @@ module TaxonomyTableTools using CSV, DataFrames, Logging, YAML using ..PipelineTypes +using ..Config export merge_taxonomy_counts, filter_table, merge_taxa @@ -274,16 +275,18 @@ export merge_taxonomy_counts, filter_table, merge_taxa end function merge_taxa(project::ProjectCtx, source::ASVResult, tax::TaxonomyHits) - config_path = joinpath(project.dir, "merge_taxa.yml") - cfg = YAML.load_file(config_path) + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "merge_taxa", Dict()) filter_list = get(cfg, "filters", [nothing]) merge_dir = joinpath(project.dir, "merged") + hash_file = joinpath(merge_dir, "config.hash") - data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy), mtime(config_path)) - merged_csv = joinpath(merge_dir, "merged.csv") + data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy)) + config_changed = _section_stale(config_path, "merge_taxa", hash_file) + merged_csv = joinpath(merge_dir, "merged.csv") # Determine what needs to be (re-)computed. - need_base = !isfile(merged_csv) || mtime(merged_csv) <= data_mtime + need_base = config_changed || !isfile(merged_csv) || mtime(merged_csv) <= data_mtime stale_filters = Pair{String,String}[] # stem => filter_path tables = Dict{String,String}("merged" => merged_csv) @@ -294,7 +297,8 @@ export merge_taxonomy_counts, filter_table, merge_taxa stem = splitext(filter_name)[1] output_csv = joinpath(merge_dir, stem * ".csv") tables[stem] = output_csv - if !isfile(output_csv) || mtime(output_csv) <= max(data_mtime, mtime(filter_path)) + if config_changed || !isfile(output_csv) || + mtime(output_csv) <= max(data_mtime, mtime(filter_path)) push!(stale_filters, stem => filter_path) else @info "Skipping merge_taxa filter '$stem': $output_csv up to date" @@ -321,6 +325,7 @@ export merge_taxonomy_counts, filter_table, merge_taxa @info "Written: $output_csv" end + _write_section_hash(config_path, "merge_taxa", hash_file) return MergedTables(tables) end diff --git a/src/project.jl b/src/project.jl index c98b958..3ec0c28 100644 --- a/src/project.jl +++ b/src/project.jl @@ -4,25 +4,117 @@ export new_project using ..PipelineTypes - # Walk upward from dirname(dir) to find the nearest ancestor within - # root_project that already has fname, then check config_dir, then - # fall back to defaults_dir. - function _find_config_source(dir::String, fname::String, - root_project::String, - config_dir::String, - defaults_dir::String) - root_norm = normpath(root_project) - candidate = normpath(dirname(dir)) - while candidate == root_norm || - startswith(candidate * "/", root_norm * "/") - path = joinpath(candidate, fname) - isfile(path) && return path - candidate == root_norm && break - candidate = normpath(dirname(candidate)) + # Header written to every new pipeline.yml override file. + const _PIPELINE_YML_HEADER = """ +# Pipeline configuration overrides +# Add only the sections/keys you want to change relative to the parent level. +# Omitted settings are inherited from the nearest ancestor or from +# config/defaults/pipeline.yml (which lists all available options). +# This file is merged at runtime; delete it to inherit everything from above. +""" + + # pipeline.yml stubs written at each project level. Comments explain the + # cascade role so users have context without needing a separate docs file. + + const _PROJECT_PIPELINE_YML = """ +# Pipeline configuration overrides - PROJECT LEVEL +# +# Settings written here serve as the default for every run under this +# study; any group- or run-level file can override them further. +# +# This is the right place for choices that define the study as a whole: +# which primer set was used, the target amplicon length range, or the +# taxonomy database for this organism group. +# +# Leave a key out entirely to inherit the global settings unchanged. Only +# write what you deliberately want to change. +# +# Note that this level is overridden by any group or run beneath it; it +# simply serves as the fallback for every run that does not explicitly +# override it. +""" + + const _GROUP_PIPELINE_YML = """ +# Pipeline configuration overrides - GROUP LEVEL +# +# Settings written here serve as the default for all runs nested beneath +# this directory; any run-level file can override them further. +# +# Use it when a subset of runs within the study share settings that differ +# from the study defaults - for example, if one batch of samples was +# sequenced with different conditions or from collected different locations. +# +# Leave a key out entirely to inherit the study settings unchanged. Only +# write what you deliberately want to change. +# +# Note that this level is overridden by any run beneath it; it simply +# serves as the fallback for every run that does not explicitly override it. +""" + + const _RUN_PIPELINE_YML = """ +# Pipeline configuration overrides - RUN LEVEL +# +# Settings written here apply to this single run only and take precedence +# over every level above. +# +# Use it when runs within the study differ from one another - for example, +# if one batch of samples was sequenced with different conditions or from +# collected different locations. +# +# Leave a key out entirely to inherit the group or study settings unchanged. +# Only write what you deliberately want to change. +# +# run_config.yml in this same directory is generated automatically at +# runtime. It shows the fully merged result of every cascade level and is +# the single place to see the complete configuration that was actually used. +# Do not edit it; it is overwritten on each run. +""" + + const _GLOBAL_CONFIGS_MD = """ +# Global Configuration + +In this directory is the machine-level override. Anything written here +applies to every study run on this installation, regardless of project. + +This is the right place for settings that reflect the environment: which +remote server handles taxonomy assignment, how many threads this machine +can spare, as well as primers available to be picked by cutadapt. + +Leave a key out entirely to inherit the default. Only write what you +deliberately want to change. + +Note that this is the lowest-precedence level after the defaults. Any +setting written here can be overridden at the project, group, or run +level, this simply serves as the fallback for every run that does not +explicitly override it. + +**WARNING: This file was regenerated by project.jl** +""" + + const _DEFAULT_CONFIGS_MD = """ +# Default Configuration + +In this directory is the baseline for every pipeline run on every +machine. It lists every key the pipeline understands and supplies a +sensible starting value for each one. + +Do not edit this file unless you are a contributer. It is the +software's own definition of what a complete configuration looks like. +Changes here will be overwritten on update. + +To change a setting, add it to the appropriate level above this one - +e.g: `config/pipeline.yml` for machine-wide changes, or a project- or +run-level file for narrower scope. + +**WARNING: This file was regenerated by project.jl** +""" + + # Write a file if it does not already exist. + function _create_if_absent(path::String, content::String) + if !isfile(path) + write(path, content) + @info "Created: $path" end - global_path = joinpath(config_dir, fname) - isfile(global_path) && return global_path - return joinpath(defaults_dir, fname) end """ @@ -32,18 +124,16 @@ export new_project directory structure found under `data_dir/{name}/`. Any directory within `data_dir/{name}/` that contains `.fastq.gz` files is - treated as a leaf run. Intermediate directories and leaf runs each receive - copies of the stage config templates, cascading downward: each level - inherits from the nearest ancestor that already has a given config, falling - back to `config_dir/defaults/` when no ancestor has it. + treated as a leaf run. The study root and all intermediate directories each + receive an empty `pipeline.yml` override stub and a `*_configs.md` file + explaining that level's role in the cascade. Users add only the settings + they want to change; everything else is inherited from the nearest ancestor, + falling back to `config/defaults/pipeline.yml`. - Config cascade (nearest-first): run -> study root -> intermediate levels - -> `config_dir/` (global) -> `config_dir/defaults/` (ultimate fallback). - All levels are bootstrapped at call time; `config_dir/` stage configs are - created alongside `databases.yml` and `tools.yml`. + `databases.yml` and `tools.yml` are copied from `config/defaults/` on first + run (they are not part of the cascade -- edit them in place). - Re-running `new_project` never overwrites existing configs. To reset a - level to its parent's settings, delete that config file and re-run. + Re-running `new_project` never overwrites existing files. Returns a `Vector{ProjectCtx}`, one per leaf run. """ @@ -58,11 +148,8 @@ export new_project isdir(root_data) || error("Data directory not found: $root_data") - stage_configs = ("dada2.yml", "cutadapt.yml", "vsearch.yml", - "cdhit.yml", "merge_taxa.yml") - - # Bootstrap global configs from defaults if missing. - for fname in ("databases.yml", "tools.yml", stage_configs...) + # Bootstrap standalone configs (not cascade-merged) from defaults. + for fname in ("databases.yml", "tools.yml", "primers.yml") dst = joinpath(config_dir, fname) if !isfile(dst) src = joinpath(defaults_dir, fname) @@ -72,6 +159,11 @@ export new_project end end + # Bootstrap config/ level docs and pipeline.yml stub. + _create_if_absent(joinpath(config_dir, "pipeline.yml"), _PIPELINE_YML_HEADER) + _create_if_absent(joinpath(config_dir, "global_configs.md"), _GLOBAL_CONFIGS_MD) + _create_if_absent(joinpath(defaults_dir, "default_configs.md"), _DEFAULT_CONFIGS_MD) + # Find leaf directories (those containing .fastq.gz), relative to root_data. leaf_relpaths = String[] for (dirpath, _, files) in walkdir(root_data) @@ -81,9 +173,10 @@ export new_project end isempty(leaf_relpaths) && error("No .fastq.gz files found under $root_data") - # Collect all directories to bootstrap (study root, intermediates, leaves). - # root_project is always included so it sits in the cascade between - # config/ and any deeper levels. + # Classify every project directory by its role in the cascade. + leaf_dirs = Set(rp == "." ? root_project : joinpath(root_project, rp) + for rp in leaf_relpaths) + all_dirs = Set{String}([root_project]) for rp in leaf_relpaths rp == "." && continue @@ -98,25 +191,22 @@ export new_project for dir in sorted_dirs mkpath(dir) - for fname in stage_configs - dst = joinpath(dir, fname) - if isfile(dst) - @info "Exists (skipping): $dst" - continue - end - src = _find_config_source(dir, fname, root_project, config_dir, defaults_dir) - isfile(src) || error("Default template not found: $src") - cp(src, dst) - @info "Created: $dst" + yml_content = if dir in leaf_dirs + _RUN_PIPELINE_YML + elseif dir == root_project + _PROJECT_PIPELINE_YML + else + _GROUP_PIPELINE_YML end + _create_if_absent(joinpath(dir, "pipeline.yml"), yml_content) end - # Return one ProjectCtx per leaf. + # Return one ProjectCtx per leaf, with study_dir = root_project. projects = ProjectCtx[] for rp in leaf_relpaths proj_dir = rp == "." ? root_project : joinpath(root_project, rp) fastq_dir = rp == "." ? root_data : joinpath(root_data, rp) - push!(projects, ProjectCtx(proj_dir, config_dir, fastq_dir)) + push!(projects, ProjectCtx(proj_dir, config_dir, fastq_dir, root_project)) end @info "$(length(projects)) project(s) ready under $root_project" return projects diff --git a/src/types.jl b/src/types.jl index 185fdb7..8a87686 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,6 +1,6 @@ module PipelineTypes - export HasFasta, ProjectCtx, TrimmedReads, ASVResult, TaxonomyHits, MergedTables + export HasFasta, ProjectCtx, TrimmedReads, ASVResult, DenoisedASVs, TaxonomyHits, MergedTables abstract type HasFasta end @@ -8,6 +8,7 @@ module PipelineTypes dir::String # projects/{.../run}/ - configs and stage output dirs live here config_dir::String # config/ data_dir::String # data/{.../run}/ - FASTQ input files + study_dir::String # projects/{study}/ - top of the per-project config cascade end struct TrimmedReads @@ -20,6 +21,13 @@ module PipelineTypes taxonomy::String # .../dada2/Tables/taxonomy.csv end + struct DenoisedASVs + chimera_ckpt::String # absolute path to ckpt_chimera.RData + config_path::String # path to run_config.yml (merged cascade) + workspace_root::String # path to dada2/ output directory + input_dir::String # path to trimmed reads directory + end + struct TaxonomyHits tsv::String # .../vsearch/taxonomy.tsv end From 1d7703de754a761bc0cf22aa828f58a69929cb0d Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Sat, 21 Feb 2026 22:28:01 +0100 Subject: [PATCH 028/175] Added ssh key option. --- README.md | 1 + config/defaults/pipeline.yml | 1 + src/config.jl | 20 ++++++++++++++++++-- src/dada2.jl | 6 ++---- src/dada2/chimera.jl | 6 +++--- src/dada2/denoise.jl | 18 +++++++++--------- src/dada2/qc.jl | 14 ++++++-------- src/dada2/taxonomy.jl | 30 ++++++++++++++++++------------ 8 files changed, 58 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 6898f9d..da41614 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ dada2: # to use the configured host. See config/defaults/pipeline.yml for the full disclaimer. remote: host: ~ # user@hostname + identity_file: ~ # path to SSH private key; null to use password auth rscript: "Rscript" # path to Rscript on the server staging_dir: "/absolute/path/on/server" db_path: ~ # absolute path to database on server (null = transfer local copy) diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index ec56330..b02ec5c 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -43,6 +43,7 @@ dada2: # The authors accept no liability for data loss or misuse arising from this feature. remote: # omit or set host to null to run locally host: ~ # user@hostname + identity_file: ~ # path to SSH private key; null to use password auth rscript: "Rscript" # path to Rscript on the server staging_dir: "/absolute/path/on/server" db_path: ~ # path to pre-installed database on server; null to transfer local copy diff --git a/src/config.jl b/src/config.jl index c630599..d322d40 100644 --- a/src/config.jl +++ b/src/config.jl @@ -132,11 +132,27 @@ export _section_stale, _write_section_hash, end end + # Walk a dotted key path ("dada2.filter_trim") into a nested Dict. + # Returns nothing if any key is missing or a non-Dict is encountered mid-path. + function _get_nested(cfg, path::AbstractString) + val = cfg + for k in split(path, ".") + val isa Dict || return nothing + val = get(val, k, nothing) + isnothing(val) && return nothing + end + return val + end + + # Section can be a dotted path ("dada2.filter_trim") or a comma-separated + # list of dotted paths ("dada2.dada,dada2.merge") whose canonical strings + # are joined before hashing. function _section_hash(config_path::String, section::String)::String cfg = YAML.load_file(config_path) cfg isa Dict || return bytes2hex(sha256("")) - sec = get(cfg, section, Dict()) - bytes2hex(sha256(_canonical(sec))) + combined = join([_canonical(_get_nested(cfg, strip(s))) + for s in split(section, ",")], "|") + bytes2hex(sha256(combined)) end """ diff --git a/src/dada2.jl b/src/dada2.jl index c6a6d99..6fb92f1 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -96,14 +96,13 @@ export dada2, dada2_denoise, dada2_classify, config_path = write_run_config(project) workspace_root = joinpath(project.dir, "dada2") chimera_ckpt = joinpath(workspace_root, "Checkpoints", "ckpt_chimera.RData") - hash_file = joinpath(workspace_root, "Checkpoints", "config.hash") trimmed_files = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(trimmed.dir)) trimmed_mtime = isempty(trimmed_files) ? 0.0 : maximum(mtime(joinpath(trimmed.dir, f)) for f in trimmed_files) if isfile(chimera_ckpt) && - !_section_stale(config_path, "dada2", hash_file) && + mtime(chimera_ckpt) > mtime(config_path) && mtime(chimera_ckpt) > trimmed_mtime @info "Skipping dada2_denoise: ckpt_chimera.RData up to date" else @@ -157,12 +156,11 @@ export dada2, dada2_denoise, dada2_classify, joinpath(tables_dir, get(out_cfg, "seq_table_prefix", "seqtab_nochim") * ".csv"), joinpath(tables_dir, get(out_cfg, "taxa_prefix", "taxonomy") * ".csv") ) - hash_file = joinpath(workspace_root, "Checkpoints", "config.hash") trimmed_files = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(trimmed.dir)) trimmed_mtime = isempty(trimmed_files) ? 0.0 : maximum(mtime(joinpath(trimmed.dir, f)) for f in trimmed_files) if all(isfile, (result.fasta, result.count_table, result.taxonomy)) && - !_section_stale(config_path, "dada2", hash_file) && + all(f -> mtime(f) > mtime(config_path), (result.fasta, result.count_table, result.taxonomy)) && all(f -> mtime(f) > trimmed_mtime, (result.fasta, result.count_table, result.taxonomy)) @info "Skipping dada2: outputs up to date in $tables_dir" return result diff --git a/src/dada2/chimera.jl b/src/dada2/chimera.jl index 5ae075a..d4f6c8b 100644 --- a/src/dada2/chimera.jl +++ b/src/dada2/chimera.jl @@ -29,9 +29,9 @@ length_ckpt = ctx.ckpts["length"] chimera_ckpt = ctx.ckpts["chimera"] - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + hash_file = joinpath(ctx.dirs["Checkpoints"], "chimera_removal.hash") if isfile(chimera_ckpt) && - !_section_stale(config_path, "dada2", hash_file) && + !_section_stale(config_path, "dada2.asv,dada2.output", hash_file) && mtime(chimera_ckpt) > mtime(filter_ckpt) && mtime(chimera_ckpt) > mtime(length_ckpt) @info "Skipping chimera_removal: checkpoint up to date" @@ -102,7 +102,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.asv,dada2.output", hash_file) emit("Written: $(joinpath(tables_dir, "pipeline_stats.csv"))") emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") diff --git a/src/dada2/denoise.jl b/src/dada2/denoise.jl index a639e96..2425b83 100644 --- a/src/dada2/denoise.jl +++ b/src/dada2/denoise.jl @@ -19,9 +19,9 @@ errors_ckpt = ctx.ckpts["errors"] filter_ckpt = ctx.ckpts["filter"] - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + hash_file = joinpath(ctx.dirs["Checkpoints"], "learn_errors.hash") if isfile(errors_ckpt) && isfile(filter_ckpt) && - !_section_stale(config_path, "dada2", hash_file) && + !_section_stale(config_path, "dada2.dada", hash_file) && mtime(errors_ckpt) > mtime(filter_ckpt) @info "Skipping learn_errors: checkpoint up to date" return nothing @@ -59,7 +59,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.dada", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "error_rates.pdf"))") emit("Checkpoint: $(ctx.ckpts["errors"])") emit("Log: $log_path") @@ -92,9 +92,9 @@ errors_ckpt = ctx.ckpts["errors"] denoise_ckpt = ctx.ckpts["denoise"] - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + hash_file = joinpath(ctx.dirs["Checkpoints"], "denoise.hash") if isfile(denoise_ckpt) && - !_section_stale(config_path, "dada2", hash_file) && + !_section_stale(config_path, "dada2.dada,dada2.merge", hash_file) && mtime(denoise_ckpt) > mtime(errors_ckpt) @info "Skipping denoise: checkpoint up to date" return nothing @@ -156,7 +156,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.dada,dada2.merge", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "length_distribution.pdf"))") emit("Checkpoint: $(ctx.ckpts["denoise"])") emit("Log: $log_path") @@ -186,9 +186,9 @@ denoise_ckpt = ctx.ckpts["denoise"] length_ckpt = ctx.ckpts["length"] - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + hash_file = joinpath(ctx.dirs["Checkpoints"], "filter_length.hash") if isfile(length_ckpt) && - !_section_stale(config_path, "dada2", hash_file) && + !_section_stale(config_path, "dada2.asv", hash_file) && mtime(length_ckpt) > mtime(denoise_ckpt) @info "Skipping filter_length: checkpoint up to date" return nothing @@ -225,7 +225,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.asv", hash_file) emit("Checkpoint: $(ctx.ckpts["length"])") emit("Log: $log_path") nothing diff --git a/src/dada2/qc.jl b/src/dada2/qc.jl index 2b680a7..747737a 100644 --- a/src/dada2/qc.jl +++ b/src/dada2/qc.jl @@ -15,10 +15,9 @@ ctx = _pipeline_context(config_path; input_dir, workspace_root) unfiltered_pdf = joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf") - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") - if isfile(unfiltered_pdf) && !_section_stale(config_path, "dada2", hash_file) + if isfile(unfiltered_pdf) all_inputs = vcat(ctx.fwd_files, ctx.rev_files) - if isempty(all_inputs) || all(f -> mtime(unfiltered_pdf) > mtime(f) for f in all_inputs) + if isempty(all_inputs) || all(mtime(unfiltered_pdf) > mtime(f) for f in all_inputs) @info "Skipping prefilter_qc: quality_unfiltered.pdf up to date" return nothing end @@ -39,7 +38,6 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_unfiltered.pdf"))") emit("Log: $log_path") nothing @@ -62,10 +60,10 @@ ctx = _pipeline_context(config_path; input_dir, workspace_root) filter_ckpt = ctx.ckpts["filter"] - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") - if isfile(filter_ckpt) && !_section_stale(config_path, "dada2", hash_file) + hash_file = joinpath(ctx.dirs["Checkpoints"], "filter_trim.hash") + if isfile(filter_ckpt) && !_section_stale(config_path, "dada2.filter_trim", hash_file) all_inputs = vcat(ctx.fwd_files, ctx.rev_files) - if isempty(all_inputs) || all(f -> mtime(filter_ckpt) > mtime(f) for f in all_inputs) + if isempty(all_inputs) || all(mtime(filter_ckpt) > mtime(f) for f in all_inputs) @info "Skipping filter_trim: checkpoint up to date" return nothing end @@ -130,7 +128,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.filter_trim", hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_filtered.pdf"))") emit("Checkpoint: $(ctx.ckpts["filter"])") emit("Log: $log_path") diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index f9b9d43..09c9c3e 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -21,10 +21,11 @@ checkpoint, taxa_prefix, multithread, min_boot, tax_levels, verbose, remote_cfg, log_path) - host = remote_cfg["host"] - rscript = get(remote_cfg, "rscript", "Rscript") - base_dir = get(remote_cfg, "staging_dir", nothing) - remote_db = get(remote_cfg, "db_path", nothing) + host = remote_cfg["host"] + rscript = get(remote_cfg, "rscript", "Rscript") + base_dir = get(remote_cfg, "staging_dir", nothing) + remote_db = get(remote_cfg, "db_path", nothing) + identity_file = get(remote_cfg, "identity_file", nothing) isnothing(base_dir) && error("taxonomy.remote.staging_dir must be set explicitly in config") @@ -42,17 +43,22 @@ remote_tables = "$staging_dir/Tables" remote_ckpt = "$staging_dir/checkpoint.RData" - # Use a ControlMaster socket so the password is entered once and all - # subsequent ssh/scp calls reuse the existing connection silently. + # Use a ControlMaster socket so the connection is established once and + # all subsequent ssh/scp calls reuse it silently. # ConnectTimeout: fail fast instead of hanging on unreachable hosts. # NumberOfPasswordPrompts=1: fail immediately on wrong password. # ServerAlive*: detect stale connections during long Rscript runs. - ctl = "/tmp/ssh_mux_$run_id" - ssh_opts = `-o ControlMaster=auto -o ControlPath=$ctl -o ControlPersist=yes -o ConnectTimeout=15 -o NumberOfPasswordPrompts=1 -o ServerAliveInterval=30 -o ServerAliveCountMax=3` + ctl = "/tmp/ssh_mux_$run_id" + id_opt = isnothing(identity_file) ? `` : `-i $identity_file` + ssh_opts = `$id_opt -o ControlMaster=auto -o ControlPath=$ctl -o ControlPersist=yes -o ConnectTimeout=15 -o NumberOfPasswordPrompts=1 -o ServerAliveInterval=30 -o ServerAliveCountMax=3` ssh = (args...) -> `ssh $ssh_opts $args` scp = (args...) -> `scp $ssh_opts $args` - emit(" Connecting to $host (enter SSH password if prompted)...") + if isnothing(identity_file) + emit(" Connecting to $host (enter SSH password if prompted)...") + else + emit(" Connecting to $host (key: $identity_file)...") + end emit(" Setting up staging directory on $host") run(ssh(host, "mkdir -p $remote_tables")) @@ -121,9 +127,9 @@ chimera_ckpt = ctx.ckpts["chimera"] checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") - hash_file = joinpath(ctx.dirs["Checkpoints"], "config.hash") + hash_file = joinpath(ctx.dirs["Checkpoints"], "assign_taxonomy.hash") if isfile(checkpoint) && - !_section_stale(config_path, "dada2", hash_file) && + !_section_stale(config_path, "dada2.taxonomy,dada2.output", hash_file) && mtime(checkpoint) > mtime(chimera_ckpt) @info "Skipping assign_taxonomy: checkpoint up to date" return nothing @@ -205,7 +211,7 @@ emit("Log: $log_path") end - _write_section_hash(config_path, "dada2", hash_file) + _write_section_hash(config_path, "dada2.taxonomy,dada2.output", hash_file) emit("Checkpoint: $checkpoint") emit("Pipeline complete. Outputs:") From 9edd3a19985526d14aa7142c408b45decd3a5b67 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Tue, 24 Feb 2026 23:14:03 +0100 Subject: [PATCH 029/175] Added SILVA database compatibility, filters, and analysis. --- Manifest.toml | 539 ++++++- Project.toml | 3 + README.md | 111 +- config/defaults/databases.yml | 42 + config/defaults/pipeline.yml | 39 +- config/filters/bacteria_archaea.pr2.yml | 13 + config/filters/bacteria_archaea.silva.yml | 13 + config/filters/environmental_protozoa.pr2.yml | 16 + .../filters/environmental_protozoa.silva.yml | 19 + config/filters/fungi.pr2.yml | 12 + config/filters/fungi.silva.yml | 13 + config/filters/helminths.pr2.yml | 14 + config/filters/metazoa.silva.yml | 14 + config/filters/parasitic_protozoa.pr2.yml | 17 + config/filters/parasitic_protozoa.silva.yml | 20 + config/filters/plants.silva.yml | 23 + config/filters/plants_invertebrates.pr2.yml | 24 + config/filters/protist.pr2.yml | 28 + config/filters/protist.silva.yml | 19 + config/filters/protist_filter.yml | 34 - config/filters/vertebrates.pr2.yml | 12 + install.jl | 2 +- src/analysis.jl | 1382 +++++++++++++++++ src/call_tools.jl | 197 ++- src/config.jl | 2 +- src/dada2.jl | 11 +- src/dada2/chimera.jl | 2 +- src/dada2/context.jl | 14 +- src/dada2/denoise.jl | 6 +- src/dada2/qc.jl | 4 +- src/dada2/taxonomy.jl | 47 +- src/databases.jl | 147 +- src/diversity.jl | 49 + src/log.jl | 176 +++ src/main.jl | 60 +- src/merge_and_filter_taxa.jl | 420 ++++- src/plots.jl | 664 ++++++++ src/project.jl | 7 +- src/types.jl | 12 +- 39 files changed, 3881 insertions(+), 346 deletions(-) create mode 100644 config/filters/bacteria_archaea.pr2.yml create mode 100644 config/filters/bacteria_archaea.silva.yml create mode 100644 config/filters/environmental_protozoa.pr2.yml create mode 100644 config/filters/environmental_protozoa.silva.yml create mode 100644 config/filters/fungi.pr2.yml create mode 100644 config/filters/fungi.silva.yml create mode 100644 config/filters/helminths.pr2.yml create mode 100644 config/filters/metazoa.silva.yml create mode 100644 config/filters/parasitic_protozoa.pr2.yml create mode 100644 config/filters/parasitic_protozoa.silva.yml create mode 100644 config/filters/plants.silva.yml create mode 100644 config/filters/plants_invertebrates.pr2.yml create mode 100644 config/filters/protist.pr2.yml create mode 100644 config/filters/protist.silva.yml delete mode 100644 config/filters/protist_filter.yml create mode 100644 config/filters/vertebrates.pr2.yml create mode 100644 src/analysis.jl create mode 100644 src/diversity.jl create mode 100644 src/log.jl create mode 100644 src/plots.jl diff --git a/Manifest.toml b/Manifest.toml index fa79302..16697b6 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.12.5" manifest_format = "2.0" -project_hash = "6b1ec8272fc2c5461cd314f6512513860a0b0719" +project_hash = "be47a58c2ce839315d6617d75a8b8dad1daa6ddc" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -31,12 +31,23 @@ weakdeps = ["SparseArrays", "StaticArrays"] AdaptSparseArraysExt = "SparseArrays" AdaptStaticArraysExt = "StaticArrays" +[[deps.AdaptivePredicates]] +git-tree-sha1 = "7e651ea8d262d2d74ce75fdf47c4d63c07dba7a6" +uuid = "35492f91-a3bd-45ad-95db-fcad7dcfedb7" +version = "1.2.0" + [[deps.AliasTables]] deps = ["PtrArrays", "Random"] git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" version = "1.1.3" +[[deps.Animations]] +deps = ["Colors"] +git-tree-sha1 = "e092fa223bf66a3c41f9c022bd074d916dc303e7" +uuid = "27a7e980-b3e6-11e9-2bcd-0b925532e340" +version = "0.4.2" + [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.2" @@ -57,16 +68,33 @@ version = "3.5.2+0" uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" +[[deps.Automa]] +deps = ["PrecompileTools", "SIMD", "TranscodingStreams"] +git-tree-sha1 = "a8f503e8e1a5f583fbef15a8440c8c7e32185df2" +uuid = "67c07d97-cdcb-5c2c-af73-a7f9c32a568b" +version = "1.1.0" + [[deps.AxisAlgorithms]] deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] git-tree-sha1 = "01b8ccb13d68535d73d2b0c23e39bd23155fb712" uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" version = "1.1.0" +[[deps.AxisArrays]] +deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] +git-tree-sha1 = "4126b08903b777c88edf1754288144a0492c05ad" +uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" +version = "0.4.8" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.BaseDirs]] +git-tree-sha1 = "bca794632b8a9bbe159d56bf9e31c422671b35e0" +uuid = "18cc8868-cbac-4acf-b575-c8ff214dc66f" +version = "1.3.2" + [[deps.BitFlags]] git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" @@ -78,12 +106,45 @@ git-tree-sha1 = "1b96ea4a01afe0ea4090c5c8039690672dd13f2e" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" version = "1.0.9+0" +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +version = "1.11.0" + +[[deps.CRlibm]] +deps = ["CRlibm_jll"] +git-tree-sha1 = "66188d9d103b92b6cd705214242e27f5737a1e5e" +uuid = "96374032-68de-5a5b-8d9e-752f78720389" +version = "1.0.2" + +[[deps.CRlibm_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e329286945d0cfc04456972ea732551869af1cfc" +uuid = "4e9b3aee-d8a1-5a3d-ad8b-7d824db253f0" +version = "1.0.1+0" + [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] git-tree-sha1 = "8d8e0b0f350b8e1c91420b5e64e5de774c2f0f4d" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" version = "0.10.16" +[[deps.Cairo]] +deps = ["Cairo_jll", "Colors", "Glib_jll", "Graphics", "Libdl", "Pango_jll"] +git-tree-sha1 = "71aa551c5c33f1a4415867fe06b7844faadb0ae9" +uuid = "159f3aea-2a34-519c-b102-8c37f9878175" +version = "1.1.1" + +[[deps.CairoMakie]] +deps = ["CRC32c", "Cairo", "Cairo_jll", "Colors", "FileIO", "FreeType", "GeometryBasics", "LinearAlgebra", "Makie", "PrecompileTools"] +git-tree-sha1 = "9bd45574379e50579a78774334f4a1f1238c0af5" +uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +version = "0.13.10" + [[deps.Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] git-tree-sha1 = "a21c5464519504e41e0cbc91f0188e8ca23d7440" @@ -134,6 +195,12 @@ git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" version = "0.7.8" +[[deps.ColorBrewer]] +deps = ["Colors", "JSON"] +git-tree-sha1 = "07da79661b919001e6863b81fc572497daa58349" +uuid = "a2cac450-b92f-5266-8821-25eda20663c8" +version = "0.4.2" + [[deps.ColorSchemes]] deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] git-tree-sha1 = "b0fd3f56fa442f81e0a47815c92245acfaaa4e34" @@ -183,9 +250,9 @@ version = "1.3.0+1" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" +git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.5.0" +version = "2.5.1" [[deps.Conda]] deps = ["Downloads", "JSON", "VersionParsing"] @@ -193,6 +260,17 @@ git-tree-sha1 = "8f06b0cfa4c514c7b9546756dbae91fcfbc92dc9" uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" version = "1.10.3" +[[deps.ConstructionBase]] +git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.6.0" +weakdeps = ["IntervalSets", "LinearAlgebra", "StaticArrays"] + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + [[deps.Contour]] git-tree-sha1 = "439e35b0b36e2e5881738abc8857bd92ad6ff9a8" uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" @@ -236,6 +314,12 @@ git-tree-sha1 = "473e9afc9cf30814eb67ffa5f2db7df82c3ad9fd" uuid = "ee1fde0b-3d02-5ea6-8484-8dfef6360eab" version = "1.16.2+0" +[[deps.DelaunayTriangulation]] +deps = ["AdaptivePredicates", "EnumX", "ExactPredicates", "Random"] +git-tree-sha1 = "c55f5a9fd67bdbc8e089b5a3111fe4292986a8e8" +uuid = "927a84f5-c5f4-47a5-9785-b46e178433df" +version = "1.6.6" + [[deps.DelimitedFiles]] deps = ["Mmap"] git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" @@ -284,12 +368,29 @@ deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.7.0" +[[deps.EarCut_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e3290f2d49e661fbd94046d7e3726ffcb2d41053" +uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" +version = "2.2.4+0" + +[[deps.EnumX]] +git-tree-sha1 = "7bebc8aad6ee6217c78c5ddcf7ed289d65d0263e" +uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" +version = "1.0.6" + [[deps.EpollShim_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "8a4be429317c42cfae6a7fc03c31bad1970c310d" uuid = "2702e6a9-849d-5ed8-8c21-79e8b8f9ee43" version = "0.0.20230411+1" +[[deps.ExactPredicates]] +deps = ["IntervalArithmetic", "Random", "StaticArrays"] +git-tree-sha1 = "83231673ea4d3d6008ac74dc5079e77ab2209d8f" +uuid = "429591f6-91af-11e9-00e2-59fbe8cec110" +version = "2.2.9" + [[deps.ExceptionUnwrapping]] deps = ["Test"] git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" @@ -307,6 +408,11 @@ git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.10" +[[deps.Extents]] +git-tree-sha1 = "b309b36a9e02fe7be71270dd8c0fd873625332b4" +uuid = "411431e0-e8b7-467b-b5e0-f676ba4f2910" +version = "0.1.6" + [[deps.FFMPEG]] deps = ["FFMPEG_jll"] git-tree-sha1 = "95ecf07c2eea562b5adbd0696af6db62c0f52560" @@ -315,9 +421,9 @@ version = "0.4.5" [[deps.FFMPEG_jll]] deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] -git-tree-sha1 = "01ba9d15e9eae375dc1eb9589df76b3572acd3f2" +git-tree-sha1 = "eaa040768ea663ca695d442be1bc97edfe6824f2" uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" -version = "8.0.1+0" +version = "6.1.3+0" [[deps.FFTA]] deps = ["AbstractFFTs", "DocStringExtensions", "LinearAlgebra", "MuladdMacro", "Primes", "Random", "Reexport"] @@ -325,6 +431,22 @@ git-tree-sha1 = "65e55303b72f4a567a51b174dd2c47496efeb95a" uuid = "b86e33f2-c0db-4aa1-a6e0-ab43e668529e" version = "0.3.1" +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "6522cfb3b8fe97bec632252263057996cbd3de20" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.18.0" +weakdeps = ["HTTP"] + + [deps.FileIO.extensions] + HTTPExt = "HTTP" + +[[deps.FilePaths]] +deps = ["FilePathsBase", "MacroTools", "Reexport", "Requires"] +git-tree-sha1 = "919d9412dbf53a2e6fe74af62a73ceed0bce0629" +uuid = "8fc22ac5-c921-52a6-82fd-178b2807b824" +version = "0.8.3" + [[deps.FilePathsBase]] deps = ["Compat", "Dates"] git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" @@ -370,12 +492,24 @@ git-tree-sha1 = "9c68794ef81b08086aeb32eeaf33531668d5f5fc" uuid = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8" version = "1.3.7" +[[deps.FreeType]] +deps = ["CEnum", "FreeType2_jll"] +git-tree-sha1 = "907369da0f8e80728ab49c1c7e09327bf0d6d999" +uuid = "b38be410-82b0-50bf-ab77-7b57e271db43" +version = "4.1.1" + [[deps.FreeType2_jll]] deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] git-tree-sha1 = "2c5512e11c791d1baed2049c5652441b28fc6a31" uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" version = "2.13.4+0" +[[deps.FreeTypeAbstraction]] +deps = ["BaseDirs", "ColorVectorSpace", "Colors", "FreeType", "GeometryBasics", "Mmap"] +git-tree-sha1 = "4ebb930ef4a43817991ba35db6317a05e59abd11" +uuid = "663a7486-cb36-511b-a19d-713bb74d65c9" +version = "0.10.8" + [[deps.FriBidi_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "7a214fdac5ed5f59a22c2d9a885a16da1c74bbc7" @@ -411,6 +545,18 @@ git-tree-sha1 = "7dd7173f7129a1b6f84e0f03e0890cd1189b0659" uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" version = "0.73.22+0" +[[deps.GeometryBasics]] +deps = ["EarCut_jll", "Extents", "IterTools", "LinearAlgebra", "PrecompileTools", "Random", "StaticArrays"] +git-tree-sha1 = "1f5a80f4ed9f5a4aada88fc2db456e637676414b" +uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +version = "0.5.10" + + [deps.GeometryBasics.extensions] + GeometryBasicsGeoInterfaceExt = "GeoInterface" + + [deps.GeometryBasics.weakdeps] + GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" + [[deps.GettextRuntime_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll"] git-tree-sha1 = "45288942190db7c5f760f59c04495064eedf9340" @@ -423,18 +569,36 @@ git-tree-sha1 = "38044a04637976140074d0b0621c1edf0eb531fd" uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" version = "9.55.1+0" +[[deps.Giflib_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6570366d757b50fabae9f4315ad74d2e40c0560a" +uuid = "59f7168a-df46-5410-90c8-f2779963d0ec" +version = "5.2.3+0" + [[deps.Glib_jll]] deps = ["Artifacts", "GettextRuntime_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] git-tree-sha1 = "24f6def62397474a297bfcec22384101609142ed" uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" version = "2.86.3+0" +[[deps.Graphics]] +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" +uuid = "a2bd30eb-e257-5431-a919-1863eab51364" +version = "1.1.3" + [[deps.Graphite2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "8a6dbda1fd736d60cc477d99f2e7a042acfa46e8" uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" version = "1.3.15+0" +[[deps.GridLayoutBase]] +deps = ["GeometryBasics", "InteractiveUtils", "Observables"] +git-tree-sha1 = "93d5c27c8de51687a2c70ec0716e6e76f298416f" +uuid = "3955a311-db13-416c-9275-1d80ed98e5e9" +version = "0.11.2" + [[deps.Grisu]] git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" @@ -458,6 +622,52 @@ git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" version = "0.3.28" +[[deps.ImageAxes]] +deps = ["AxisArrays", "ImageBase", "ImageCore", "Reexport", "SimpleTraits"] +git-tree-sha1 = "e12629406c6c4442539436581041d372d69c55ba" +uuid = "2803e5a7-5153-5ecf-9a86-9b4c37f5f5ac" +version = "0.6.12" + +[[deps.ImageBase]] +deps = ["ImageCore", "Reexport"] +git-tree-sha1 = "eb49b82c172811fd2c86759fa0553a2221feb909" +uuid = "c817782e-172a-44cc-b673-b171935fbb9e" +version = "0.1.7" + +[[deps.ImageCore]] +deps = ["ColorVectorSpace", "Colors", "FixedPointNumbers", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "PrecompileTools", "Reexport"] +git-tree-sha1 = "8c193230235bbcee22c8066b0374f63b5683c2d3" +uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" +version = "0.10.5" + +[[deps.ImageIO]] +deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs", "WebP"] +git-tree-sha1 = "696144904b76e1ca433b886b4e7edd067d76cbf7" +uuid = "82e4d734-157c-48bb-816b-45c225c6df19" +version = "0.6.9" + +[[deps.ImageMetadata]] +deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] +git-tree-sha1 = "2a81c3897be6fbcde0802a0ebe6796d0562f63ec" +uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" +version = "0.9.10" + +[[deps.Imath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "dcc8d0cd653e55213df9b75ebc6fe4a8d3254c65" +uuid = "905a6f67-0a94-5f89-b386-d35d92009cd1" +version = "3.2.2+0" + +[[deps.IndirectArrays]] +git-tree-sha1 = "012e604e1c7458645cb8b436f8fba789a51b257f" +uuid = "9b13fd28-a010-5f03-acff-a1bbcff69959" +version = "1.0.0" + +[[deps.Inflate]] +git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.5" + [[deps.InlineStrings]] git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" @@ -482,18 +692,59 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" [[deps.Interpolations]] -deps = ["Adapt", "AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] -git-tree-sha1 = "65d505fa4c0d7072990d659ef3fc086eb6da8208" +deps = ["Adapt", "AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "Requires", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] +git-tree-sha1 = "88a101217d7cb38a7b481ccd50d21876e1d1b0e0" uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" -version = "0.16.2" +version = "0.15.1" +weakdeps = ["Unitful"] [deps.Interpolations.extensions] - InterpolationsForwardDiffExt = "ForwardDiff" InterpolationsUnitfulExt = "Unitful" - [deps.Interpolations.weakdeps] +[[deps.IntervalArithmetic]] +deps = ["CRlibm", "MacroTools", "OpenBLASConsistentFPCSR_jll", "Printf", "Random", "RoundingEmulator"] +git-tree-sha1 = "02b61501dbe6da3b927cc25dacd7ce32390ee970" +uuid = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" +version = "1.0.2" + + [deps.IntervalArithmetic.extensions] + IntervalArithmeticArblibExt = "Arblib" + IntervalArithmeticDiffRulesExt = "DiffRules" + IntervalArithmeticForwardDiffExt = "ForwardDiff" + IntervalArithmeticIntervalSetsExt = "IntervalSets" + IntervalArithmeticLinearAlgebraExt = "LinearAlgebra" + IntervalArithmeticRecipesBaseExt = "RecipesBase" + IntervalArithmeticSparseArraysExt = "SparseArrays" + + [deps.IntervalArithmetic.weakdeps] + Arblib = "fb37089c-8514-4489-9461-98f9c8763369" + DiffRules = "b552c78f-8df3-52c6-915a-8e097449b14b" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" - Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.IntervalSets]] +git-tree-sha1 = "d966f85b3b7a8e49d034d27a189e9a4874b4391a" +uuid = "8197267c-284f-5f27-9208-e0e47529a953" +version = "0.7.13" +weakdeps = ["Random", "RecipesBase", "Statistics"] + + [deps.IntervalSets.extensions] + IntervalSetsRandomExt = "Random" + IntervalSetsRecipesBaseExt = "RecipesBase" + IntervalSetsStatisticsExt = "Statistics" + +[[deps.InverseFunctions]] +git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.17" +weakdeps = ["Dates", "Test"] + + [deps.InverseFunctions.extensions] + InverseFunctionsDatesExt = "Dates" + InverseFunctionsTestExt = "Test" [[deps.InvertedIndices]] git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe" @@ -505,6 +756,17 @@ git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" version = "0.2.6" +[[deps.Isoband]] +deps = ["isoband_jll"] +git-tree-sha1 = "f9b6d97355599074dc867318950adaa6f9946137" +uuid = "f1662d9f-8043-43de-a69a-05efc1cc6ff4" +version = "0.1.1" + +[[deps.IterTools]] +git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.10.0" + [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" @@ -534,6 +796,12 @@ version = "1.4.0" [deps.JSON.weakdeps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" +[[deps.JpegTurbo]] +deps = ["CEnum", "FileIO", "ImageCore", "JpegTurbo_jll", "TOML"] +git-tree-sha1 = "9496de8fb52c224a2e3f9ff403947674517317d9" +uuid = "b835a17e-a41a-41e7-81f0-2f016b05efe0" +version = "0.1.6" + [[deps.JpegTurbo_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "b6893345fd6658c8e475d40155789f4860ac3b21" @@ -598,6 +866,11 @@ version = "0.16.10" SymEngine = "123dc426-2d89-5057-bbad-38513e3affd8" tectonic_jll = "d7dd28d6-a5e6-559c-9131-7eb760cdacc5" +[[deps.LazyModules]] +git-tree-sha1 = "a560dd966b386ac9ae60bdd3a3d3a326062d3c3e" +uuid = "8cdb02fc-e678-4876-92c5-9defec4f444e" +version = "0.3.1" + [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" @@ -699,11 +972,34 @@ git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" version = "0.5.16" +[[deps.Makie]] +deps = ["Animations", "Base64", "CRC32c", "ColorBrewer", "ColorSchemes", "ColorTypes", "Colors", "Contour", "Dates", "DelaunayTriangulation", "Distributions", "DocStringExtensions", "Downloads", "FFMPEG_jll", "FileIO", "FilePaths", "FixedPointNumbers", "Format", "FreeType", "FreeTypeAbstraction", "GeometryBasics", "GridLayoutBase", "ImageBase", "ImageIO", "InteractiveUtils", "Interpolations", "IntervalSets", "InverseFunctions", "Isoband", "KernelDensity", "LaTeXStrings", "LinearAlgebra", "MacroTools", "MakieCore", "Markdown", "MathTeXEngine", "Observables", "OffsetArrays", "PNGFiles", "Packing", "PlotUtils", "PolygonOps", "PrecompileTools", "Printf", "REPL", "Random", "RelocatableFolders", "Scratch", "ShaderAbstractions", "Showoff", "SignedDistanceFields", "SparseArrays", "Statistics", "StatsBase", "StatsFuns", "StructArrays", "TriplotBase", "UnicodeFun", "Unitful"] +git-tree-sha1 = "1d7d16f0e02ec063becd7a140f619b2ffe5f2b11" +uuid = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +version = "0.22.10" + +[[deps.MakieCore]] +deps = ["ColorTypes", "GeometryBasics", "IntervalSets", "Observables"] +git-tree-sha1 = "c3159eb1e3aa3e409edbb71f4035ed8b1fc16e23" +uuid = "20f20a25-4f0e-4fdf-b5d1-57303727442b" +version = "0.9.5" + +[[deps.MappedArrays]] +git-tree-sha1 = "0ee4497a4e80dbd29c058fcee6493f5219556f40" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.4.3" + [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" +[[deps.MathTeXEngine]] +deps = ["AbstractTrees", "Automa", "DataStructures", "FreeTypeAbstraction", "GeometryBasics", "LaTeXStrings", "REPL", "RelocatableFolders", "UnicodeFun"] +git-tree-sha1 = "7eb8cdaa6f0e8081616367c10b31b9d9b34bb02a" +uuid = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" +version = "0.6.7" + [[deps.MbedTLS]] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" @@ -737,6 +1033,12 @@ git-tree-sha1 = "2c140d60d7cb82badf06d8783800d0bcd1a7daa2" uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" version = "0.8.1" +[[deps.MosaicViews]] +deps = ["MappedArrays", "OffsetArrays", "PaddedViews", "StackViews"] +git-tree-sha1 = "7b86a5d4d70a9f5cdf2dacb3cbe6d251d1a61dbe" +uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +version = "0.3.4" + [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2025.11.4" @@ -748,9 +1050,9 @@ version = "0.2.4" [[deps.MultivariateStats]] deps = ["Arpack", "Distributions", "LinearAlgebra", "SparseArrays", "Statistics", "StatsAPI", "StatsBase"] -git-tree-sha1 = "816620e3aac93e5b5359e4fdaf23ca4525b00ddf" +git-tree-sha1 = "7c3ff68a904d0f7404e5d2f7f5bc667934d8d616" uuid = "6f286f6a-111f-5878-ab1e-185364afe411" -version = "0.10.3" +version = "0.10.4" [[deps.NaNMath]] deps = ["OpenLibm_jll"] @@ -764,6 +1066,12 @@ git-tree-sha1 = "e2c3bba08dd6dedfe17a17889131b885b8c082f0" uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" version = "0.4.27" +[[deps.Netpbm]] +deps = ["FileIO", "ImageCore", "ImageMetadata"] +git-tree-sha1 = "d92b107dbb887293622df7697a2223f9f8176fcd" +uuid = "f09324ee-3d7c-5217-9330-fc30815ba969" +version = "1.1.1" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" @@ -788,11 +1096,29 @@ git-tree-sha1 = "b6aa4566bb7ae78498a5e68943863fa8b5231b59" uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" version = "1.3.6+0" +[[deps.OpenBLASConsistentFPCSR_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "f2b3b9e52a5eb6a3434c8cca67ad2dde011194f4" +uuid = "6cdc7f73-28fd-5e50-80fb-958a8875b1af" +version = "0.3.30+0" + [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.29+0" +[[deps.OpenEXR]] +deps = ["Colors", "FileIO", "OpenEXR_jll"] +git-tree-sha1 = "97db9e07fe2091882c765380ef58ec553074e9c7" +uuid = "52e1d378-f018-4a11-a4be-720524705ac7" +version = "0.3.3" + +[[deps.OpenEXR_jll]] +deps = ["Artifacts", "Imath_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "df9b7c88c2e7a2e77146223c526bf9e236d5f450" +uuid = "18a262bb-aa17-5467-a713-aee519bc75cb" +version = "3.4.4+0" + [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" @@ -841,6 +1167,24 @@ weakdeps = ["StatsBase"] [deps.PDMats.extensions] StatsBaseExt = "StatsBase" +[[deps.PNGFiles]] +deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] +git-tree-sha1 = "cf181f0b1e6a18dfeb0ee8acc4a9d1672499626c" +uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" +version = "0.4.4" + +[[deps.Packing]] +deps = ["GeometryBasics"] +git-tree-sha1 = "bc5bf2ea3d5351edf285a06b0016788a121ce92c" +uuid = "19eb6ba3-879d-56ad-ad62-d5c202156566" +version = "0.5.1" + +[[deps.PaddedViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" +uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" +version = "0.5.12" + [[deps.Pango_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"] git-tree-sha1 = "0662b083e11420952f2e62e17eddae7fc07d5997" @@ -868,6 +1212,12 @@ weakdeps = ["REPL"] [deps.Pkg.extensions] REPLExt = "REPL" +[[deps.PkgVersion]] +deps = ["Pkg"] +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" +uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" +version = "0.3.3" + [[deps.PlotThemes]] deps = ["PlotUtils", "Statistics"] git-tree-sha1 = "41031ef3a1be6f5bbbf3e8073f210556daeae5ca" @@ -882,9 +1232,9 @@ version = "1.4.4" [[deps.Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "TOML", "UUIDs", "UnicodeFun", "Unzip"] -git-tree-sha1 = "1cc8ad0762e59e713ee3ef28f9b78b2c9f4ca078" +git-tree-sha1 = "cb20a4eacda080e517e4deb9cfb6c7c518131265" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.41.5" +version = "1.41.6" [deps.Plots.extensions] FileIOExt = "FileIO" @@ -900,6 +1250,11 @@ version = "1.41.5" ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" +[[deps.PolygonOps]] +git-tree-sha1 = "77b3d3605fc1cd0b42d95eba87dfcd2bf67d5ff6" +uuid = "647866c9-e3ac-4575-94e7-e3d426903924" +version = "0.1.2" + [[deps.PooledArrays]] deps = ["DataAPI", "Future"] git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" @@ -941,10 +1296,22 @@ deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" +[[deps.ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.11.0" + [[deps.PtrArrays]] -git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" +git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" -version = "1.3.0" +version = "1.4.0" + +[[deps.QOI]] +deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] +git-tree-sha1 = "472daaa816895cb7aee81658d4e7aec901fa1106" +uuid = "4b34888f-f399-49d4-9bb3-47ed5cae4e65" +version = "1.0.2" [[deps.Qt6Base_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Vulkan_Loader_jll", "Xorg_libSM_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_cursor_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "libinput_jll", "xkbcommon_jll"] @@ -987,13 +1354,11 @@ deps = ["CategoricalArrays", "Conda", "DataFrames", "DataStructures", "Dates", " git-tree-sha1 = "0ea46f30de5b17d7bd8eaaadb431b0a9ae494a48" uuid = "6f49c342-dc21-5d91-9882-a32aef131414" version = "0.14.12" +weakdeps = ["AxisArrays"] [deps.RCall.extensions] RCallAxisArraysExt = ["AxisArrays"] - [deps.RCall.weakdeps] - AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" - [[deps.REPL]] deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -1004,6 +1369,11 @@ deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" version = "1.11.0" +[[deps.RangeArrays]] +git-tree-sha1 = "b9039e93773ddcfc828f12aadf7115b4b4d225f5" +uuid = "b3c3ace0-ae52-54e7-9d0b-2c1406fd6b9d" +version = "0.3.2" + [[deps.Ratios]] deps = ["Requires"] git-tree-sha1 = "1342a47bf3260ee108163042310d26f2be5ec90b" @@ -1055,10 +1425,21 @@ git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8" uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" version = "0.5.1+0" +[[deps.RoundingEmulator]] +git-tree-sha1 = "40b9edad2e5287e05bd413a38f61a8ff55b9557b" +uuid = "5eaf0fd0-dfba-4ccb-bf02-d820a40db705" +version = "0.2.1" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" +[[deps.SIMD]] +deps = ["PrecompileTools"] +git-tree-sha1 = "e24dc23107d426a096d3eae6c165b921e74c18e4" +uuid = "fdea26ae-647d-5447-a871-4b548cad5224" +version = "3.7.2" + [[deps.Scratch]] deps = ["Dates"] git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" @@ -1075,6 +1456,12 @@ version = "1.4.9" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.ShaderAbstractions]] +deps = ["ColorTypes", "FixedPointNumbers", "GeometryBasics", "LinearAlgebra", "Observables", "StaticArrays"] +git-tree-sha1 = "818554664a2e01fc3784becb2eb3a82326a604b6" +uuid = "65257c39-d410-5151-9873-9b3e5be5013e" +version = "0.5.0" + [[deps.SharedArrays]] deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" @@ -1091,11 +1478,29 @@ git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" version = "1.0.3" +[[deps.SignedDistanceFields]] +deps = ["Statistics"] +git-tree-sha1 = "3949ad92e1c9d2ff0cd4a1317d5ecbba682f4b92" +uuid = "73760f76-fbc4-59ce-8f25-708e95d2df96" +version = "0.4.1" + [[deps.SimpleBufferStream]] git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" version = "1.2.0" +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "be8eeac05ec97d379347584fa9fe2f5f76795bcb" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.5" + +[[deps.Sixel]] +deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] +git-tree-sha1 = "0494aed9501e7fb65daba895fb7fd57cc38bc743" +uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" +version = "0.1.5" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" @@ -1127,11 +1532,17 @@ git-tree-sha1 = "4f96c596b8c8258cc7d3b19797854d368f243ddc" uuid = "860ef19b-820b-49d6-a774-d7a799459cd3" version = "1.0.4" +[[deps.StackViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "be1cf4eb0ac528d96f5115b4ed80c26a8d8ae621" +uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" +version = "0.1.2" + [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "eee1b9ad8b29ef0d936e3ec9838c7ec089620308" +git-tree-sha1 = "0f529006004a8be48f1be25f3451186579392d47" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.16" +version = "1.9.17" weakdeps = ["ChainRulesCore", "Statistics"] [deps.StaticArrays.extensions] @@ -1170,15 +1581,12 @@ deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Re git-tree-sha1 = "91f091a8716a6bb38417a6e6f274602a19aaa685" uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" version = "1.5.2" +weakdeps = ["ChainRulesCore", "InverseFunctions"] [deps.StatsFuns.extensions] StatsFunsChainRulesCoreExt = "ChainRulesCore" StatsFunsInverseFunctionsExt = "InverseFunctions" - [deps.StatsFuns.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - [[deps.StatsModels]] deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsAPI", "StatsBase", "StatsFuns", "Tables"] git-tree-sha1 = "08786db4a1346d17d0a8d952d2e66fd00fa18192" @@ -1203,6 +1611,27 @@ git-tree-sha1 = "a3c1536470bf8c5e02096ad4853606d7c8f62721" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.4.2" +[[deps.StructArrays]] +deps = ["ConstructionBase", "DataAPI", "Tables"] +git-tree-sha1 = "a2c37d815bf00575332b7bd0389f771cb7987214" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.7.2" + + [deps.StructArrays.extensions] + StructArraysAdaptExt = "Adapt" + StructArraysGPUArraysCoreExt = ["GPUArraysCore", "KernelAbstractions"] + StructArraysLinearAlgebraExt = "LinearAlgebra" + StructArraysSparseArraysExt = "SparseArrays" + StructArraysStaticArraysExt = "StaticArrays" + + [deps.StructArrays.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" + KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + [[deps.StructUtils]] deps = ["Dates", "UUIDs"] git-tree-sha1 = "28145feabf717c5d65c1d5e09747ee7b1ff3ed13" @@ -1275,6 +1704,12 @@ deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" version = "1.11.0" +[[deps.TiffImages]] +deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "PrecompileTools", "ProgressMeter", "SIMD", "UUIDs"] +git-tree-sha1 = "98b9352a24cb6a2066f9ababcc6802de9aed8ad8" +uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" +version = "0.11.6" + [[deps.TimeZones]] deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "Scratch", "TZJData", "Unicode", "p7zip_jll"] git-tree-sha1 = "d422301b2a1e294e3e4214061e44f338cafe18a2" @@ -1290,6 +1725,11 @@ git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" +[[deps.TriplotBase]] +git-tree-sha1 = "4d4ed7f294cda19382ff7de4c137d24d16adc89b" +uuid = "981d1d27-644d-49a2-9326-4793e63143c3" +version = "0.1.0" + [[deps.URIs]] git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" @@ -1310,6 +1750,29 @@ git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" version = "0.4.1" +[[deps.Unitful]] +deps = ["Dates", "LinearAlgebra", "Random"] +git-tree-sha1 = "57e1b2c9de4bd6f40ecb9de4ac1797b81970d008" +uuid = "1986cc42-f94f-5a68-af5c-568840ba703d" +version = "1.28.0" + + [deps.Unitful.extensions] + ConstructionBaseUnitfulExt = "ConstructionBase" + ForwardDiffExt = "ForwardDiff" + InverseFunctionsUnitfulExt = "InverseFunctions" + LatexifyExt = ["Latexify", "LaTeXStrings"] + NaNMathExt = "NaNMath" + PrintfExt = "Printf" + + [deps.Unitful.weakdeps] + ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" + Latexify = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" + NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" + Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" + [[deps.Unzip]] git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78" uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d" @@ -1338,6 +1801,12 @@ git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" version = "1.4.2" +[[deps.WebP]] +deps = ["CEnum", "ColorTypes", "FileIO", "FixedPointNumbers", "ImageCore", "libwebp_jll"] +git-tree-sha1 = "aa1ca3c47f119fbdae8770c29820e5e6119b83f2" +uuid = "e3aaa7dc-3e4b-44e0-be63-ffb868ccd7c1" +version = "0.1.3" + [[deps.Widgets]] deps = ["Colors", "Dates", "Observables", "OrderedCollections"] git-tree-sha1 = "e9aeb174f95385de31e70bd15fa066a505ea82b9" @@ -1533,6 +2002,12 @@ git-tree-sha1 = "b6a34e0e0960190ac2a4363a1bd003504772d631" uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" version = "0.61.1+0" +[[deps.isoband_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51b5eeb3f98367157a7a12a1fb0aa5328946c03c" +uuid = "9a68df92-36a6-505f-a73e-abb412b6bfb4" +version = "0.2.3+0" + [[deps.libaom_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "371cc681c00a3ccc3fbc5c0fb91f58ba9bec1ecf" @@ -1580,12 +2055,24 @@ git-tree-sha1 = "e015f211ebb898c8180887012b938f3851e719ac" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" version = "1.6.55+0" +[[deps.libsixel_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "libpng_jll"] +git-tree-sha1 = "c1733e347283df07689d71d61e14be986e49e47a" +uuid = "075b6546-f08a-558a-be8f-8157d0f608a5" +version = "1.10.5+0" + [[deps.libvorbis_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll"] git-tree-sha1 = "11e1772e7f3cc987e9d3de991dd4f6b2602663a5" uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" version = "1.3.8+0" +[[deps.libwebp_jll]] +deps = ["Artifacts", "Giflib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libglvnd_jll", "Libtiff_jll", "libpng_jll"] +git-tree-sha1 = "4e4282c4d846e11dce56d74fa8040130b7a95cb3" +uuid = "c5f90fcd-3b7e-5836-afba-fc50a0988cb2" +version = "1.6.0+0" + [[deps.mtdev_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "b4d631fd51f2e9cdd93724ae25b2efc198b059b1" diff --git a/Project.toml b/Project.toml index 2ffa552..09da061 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +1,7 @@ [deps] +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -10,6 +12,7 @@ TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] +CairoMakie = "0.12, 0.13" CSV = "0.10" DataFrames = "1.8" RCall = "0.14" diff --git a/README.md b/README.md index da41614..0db4ce6 100644 --- a/README.md +++ b/README.md @@ -183,16 +183,8 @@ dada2: database: pr2 # key into config/databases.yml multithread: 4 # threads for assignTaxonomy(); higher values increase memory use min_boot: 0 # minimum bootstrap confidence to retain (0-100) - levels: - - "Domain" - - "Supergroup" - - "Division" - - "Subdivision" - - "Class" - - "Order" - - "Family" - - "Genus" - - "Species" + # Taxonomy rank names are read from databases.yml (the `levels:` key under + # each database entry). Do not set them here. # Optional: offload the memory-intensive assignTaxonomy() step to a remote # server via SSH. Omit or set host to null to run locally. @@ -261,51 +253,80 @@ merge_taxa: Each entry is a filename relative to `config/filters/`. Remove all entries (or set `filters: []`) to produce only the unfiltered `merged.csv`. -### Configuring taxonomic filtering (`config/filters/protist_filter.yml`) +### Configuring taxonomic filtering (`config/filters/`) -The filter file controls the `filter_table()` step, which removes non-target taxa and remaps Supergroup labels for consistency with PR2 division names. +Each file in `config/filters/` defines one biological group to extract from the merged table. Filters are applied after the taxonomy/count merge and produce one additional CSV per entry in `merge_taxa.filters`. -Place filter configs in `config/filters/` and reference them by filename in `merge_taxa.yml`. The default `protist_filter.yml` targets eukaryotic protists from PR2-annotated data: +#### Database-specific filters + +Filter files are named `{category}.{database}.yml` and carry a `databases:` key so that each filter is only applied when the active database matches: ```yaml -# Division -> Supergroup remapping. -# This block overrides Supergroup with the Division value where they should be equivalent. -mappings: - Rhizaria: Rhizaria - Alveolata: Alveolata - Stramenopiles: Stramenopiles - Hemimastigophora: Hemimastigophora - Discoba: Discoba - Metamonada: Metamonada - Telonemia: Telonemia - Ancyromonadida: Ancyromonadida - -# Exclusion filters: rows where the named column contains the pattern are removed. +# fungi.pr2.yml +databases: [pr2] + filters: - - column: Domain - pattern: Bacteria - - column: Domain - pattern: Archaea - - column: Domain - pattern: Eukaryota:plas - - column: Domain - pattern: Eukaryota:mito - - column: Supergroup - pattern: TSAR:chro - - column: Subdivision - pattern: Metazoa - column: Subdivision pattern: Fungi - - column: Division - pattern: Rhodophyta - - column: Class - pattern: Embryophyceae + action: keep # keep rows matching the pattern (default action is exclude) + +remove_empty: + - Subdivision +``` + +```yaml +# fungi.silva.yml +databases: [silva] + +filters: + - column: Family + pattern: Nucletmycea + action: keep -# Remove rows with an empty or unassigned Domain field. -remove_empty_domain: true +remove_empty: + - Family ``` -> **Database compatibility:** Column names and patterns above are tuned for [PR2](https://pr2-database.org/). If you use a different reference database, update the column names to match that database's rank structure and adjust the `levels` list in `dada2.yml` accordingly. +Both files can be listed in `merge_taxa.filters` simultaneously; the pipeline silently skips whichever does not apply to the active database. + +The following filter pairs ship in `config/filters/`: + +| Category | PR2 match | SILVA match | +|----------|-----------|-------------| +| `bacteria_archaea` | `Domain` = Bacteria\|Archaea | `Kingdom` = Bacteria\|Archaea | +| `environmental_protozoa` | `Subdivision` = Cercozoa\|Gyrista\|Ciliophora\|Chrompodellids | `Order` = Cercozoa\|Ciliophora\|Ochrophyta | +| `fungi` | `Subdivision` = Fungi | `Family` = Nucletmycea | +| `helminths` | `Class` = Nematoda (excl. *Miculenchus*) | `Family` = Holozoa ¹ | +| `parasitic_protozoa` | `Subdivision` = Apicomplexa\|Parabasalia\|Fornicata\|Bigyra | `Family` = Conoidasida\|Trichomonadea\|Blastocystis\|Proteromonadea | +| `plants_invertebrates` | Exclusion-based (PR2 ranks) | Exclusion-based (SILVA ranks) ¹ | +| `protist` | Exclusion-based (PR2 ranks) | Exclusion-based (SILVA ranks) | +| `vertebrates` | `Class` = Craniata | `Family` = Holozoa ¹ | + +> ¹ **SILVA metazoan resolution limit.** SILVA's 18S taxonomy uses only six ranks (Kingdom → Genus). The entire Metazoa lineage compresses into these ranks such that all animals — vertebrates, nematodes, annelids — resolve to `Family = Holozoa`, `Genus = Choanozoa`. There is no rank at which helminths and vertebrates can be distinguished. As a result: +> - `helminths.silva.yml` and `vertebrates.silva.yml` are functionally equivalent: both capture all metazoa. +> - `plants_invertebrates.silva.yml` retains metazoa alongside plants but cannot exclude vertebrates. +> +> Use PR2 (`database: pr2`) when helminth- or vertebrate-specific filtering is required. + +#### Filter file format + +```yaml +databases: [pr2] # omit to apply regardless of active database + +mappings: # optional column remapping applied before filters + - source_column: Division + target_column: Supergroup + values: { Rhizaria: Rhizaria, Alveolata: Alveolata } + +filters: + - column: Domain + pattern: "Bacteria|Archaea" + regex: true # false (default) = substring match + action: exclude # exclude (default) | keep + +remove_empty: # remove rows where this column is blank or "NA" + - Domain +``` ## Usage diff --git a/config/defaults/databases.yml b/config/defaults/databases.yml index 50f9470..e17a5f4 100644 --- a/config/defaults/databases.yml +++ b/config/defaults/databases.yml @@ -7,6 +7,48 @@ databases: dada2: uri: "https://github.com/pr2database/pr2database/releases/download/v5.0.0/pr2_version_5.0.0_SSU_dada2.fasta.gz" local: ~ # set to a local path to skip download + remote_path: ~ # pre-existing path on remote host (avoids transferring the file) vsearch: uri: "https://github.com/pr2database/pr2database/releases/download/v5.1.0.0/pr2_version_5.1.0_SSU_taxo_long.fasta.gz" local: ~ # set to a local path to skip download + levels: + - Domain + - Supergroup + - Division + - Subdivision + - Class + - Order + - Family + - Genus + - Species + vsearch_format: pr2 # pipe-separated: Acc|rRNA|Org|spec|Domain|...|Species + corrections: + - source: Division + target: Supergroup + values: + Rhizaria: Rhizaria + Alveolata: Alveolata + Stramenopiles: Stramenopiles + Hemimastigophora: Hemimastigophora + Discoba: Discoba + Metamonada: Metamonada + Telonemia: Telonemia + Ancyromonadida: Ancyromonadida + silva: + dada2: + uri: "https://zenodo.org/records/4587955/files/silva_nr99_v138.1_train_set.fa.gz" + local: ~ + remote_path: ~ # pre-existing path on remote host (avoids transferring the file) + vsearch: + uri: "https://www.arb-silva.de/fileadmin/silva_databases/release_138_1/Exports/SILVA_138.1_SSURef_NR99_tax_silva.fasta.gz" + local: ~ + reformat: silva_vsearch # embed taxonomy in sequence ID so vsearch returns it in target field + levels: + - Kingdom + - Phylum + - Class + - Order + - Family + - Genus + vsearch_format: silva # >Acc Kingdom;Phylum;...;Genus + corrections: [] diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index b02ec5c..1aa29b1 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -46,18 +46,7 @@ dada2: identity_file: ~ # path to SSH private key; null to use password auth rscript: "Rscript" # path to Rscript on the server staging_dir: "/absolute/path/on/server" - db_path: ~ # path to pre-installed database on server; null to transfer local copy min_boot: 0 - levels: - - "Domain" - - "Supergroup" - - "Division" - - "Subdivision" - - "Class" - - "Order" - - "Family" - - "Genus" - - "Species" output: seq_table_prefix: "seqtab_nochim" @@ -78,4 +67,30 @@ merge_taxa: # The unfiltered merged.csv is always written regardless of this list. # Set to [] or omit entries to produce merged.csv only. filters: - - "protist_filter.yml" + - "parasitic_protozoa.pr2.yml" + - "parasitic_protozoa.silva.yml" + - "environmental_protozoa.pr2.yml" + - "environmental_protozoa.silva.yml" + - "protist.pr2.yml" + - "protist.silva.yml" + - "helminths.pr2.yml" + - "bacteria_archaea.pr2.yml" + - "bacteria_archaea.silva.yml" + - "fungi.pr2.yml" + - "fungi.silva.yml" + - "metazoa.silva.yml" + - "plants_invertebrates.pr2.yml" + - "plants.silva.yml" + - "vertebrates.pr2.yml" + +analysis: + taxa_bar: + top_n: 15 # collapse below this rank to "Other" + rank: ~ # null = lowest assigned rank; or specify e.g. "Class" + ranks: ~ # null = auto: [asv/lowest, penultimate level, antepenultimate level] + report_ranks: ~ # null = auto: last 3 levels + alpha: + metrics: ["richness", "shannon", "simpson"] + nmds: + distance: "bray_curtis" # only option for now + max_stress: 0.2 # warn if NMDS stress exceeds this value diff --git a/config/filters/bacteria_archaea.pr2.yml b/config/filters/bacteria_archaea.pr2.yml new file mode 100644 index 0000000..58b7b4c --- /dev/null +++ b/config/filters/bacteria_archaea.pr2.yml @@ -0,0 +1,13 @@ +# Bacteria + Archaea filter — retains all prokaryotic ASVs. +# Covers Bacteroidetes, Firmicutes, Tenericutes, Cyanobacteria, and +# any Archaea if present. +databases: [pr2] + +filters: + - column: Domain + pattern: "Bacteria|Archaea" + regex: true + action: keep + +remove_empty: + - Domain diff --git a/config/filters/bacteria_archaea.silva.yml b/config/filters/bacteria_archaea.silva.yml new file mode 100644 index 0000000..b5bc15e --- /dev/null +++ b/config/filters/bacteria_archaea.silva.yml @@ -0,0 +1,13 @@ +# Bacteria + Archaea filter — retains all prokaryotic ASVs. +# Covers Firmicutes, Bacteroidota, Cyanobacteria, Proteobacteria, +# Spirochaetota, and all Archaea (Euryarchaeota, Crenarchaeota, etc.). +databases: [silva] + +filters: + - column: Kingdom + pattern: "Bacteria|Archaea" + regex: true + action: keep + +remove_empty: + - Kingdom diff --git a/config/filters/environmental_protozoa.pr2.yml b/config/filters/environmental_protozoa.pr2.yml new file mode 100644 index 0000000..ab16eb8 --- /dev/null +++ b/config/filters/environmental_protozoa.pr2.yml @@ -0,0 +1,16 @@ +# Environmental (free-living) protozoa. +# +# Cercozoa: soil amoeboflagellates (Cercomonas, Sandona, etc.) +# Gyrista: Chrysophyceae (Spumella), Xanthophyceae (Botrydiopsis) +# Ciliophora: Colpodea (soil ciliate), Litostomatea +# Chrompodellids: Colpodellida (free-living predatory flagellate) +databases: [pr2] + +filters: + - column: Subdivision + pattern: "Cercozoa|Gyrista|Ciliophora|Chrompodellids" + regex: true + action: keep + +remove_empty: + - Subdivision diff --git a/config/filters/environmental_protozoa.silva.yml b/config/filters/environmental_protozoa.silva.yml new file mode 100644 index 0000000..28870e4 --- /dev/null +++ b/config/filters/environmental_protozoa.silva.yml @@ -0,0 +1,19 @@ +# Environmental (free-living) protozoa. +# +# SILVA equivalents of PR2 groups: +# Cercozoa -> Order: Cercozoa (Cercomonas, Heteromita; under Class Rhizaria) +# Ciliophora -> Order: Ciliophora (Litostomatea, Colpodea; under Class Alveolata) +# Gyrista -> Order: Ochrophyta (Chrysophyceae, Xanthophyceae; under Class Stramenopiles) +# +# Note: Chrompodellids (Colpodellida) cannot be separated in SILVA's +# 6-rank hierarchy; they fall under Alveolata without a distinct Order. +databases: [silva] + +filters: + - column: Order + pattern: "Cercozoa|Ciliophora|Ochrophyta" + regex: true + action: keep + +remove_empty: + - Order diff --git a/config/filters/fungi.pr2.yml b/config/filters/fungi.pr2.yml new file mode 100644 index 0000000..324e488 --- /dev/null +++ b/config/filters/fungi.pr2.yml @@ -0,0 +1,12 @@ +# Fungi filter — retains all fungal ASVs. +# PR2 taxonomy: Opisthokonta > Fungi > (Basidiomycota, Ascomycota, +# Mucoromycota, Chytridiomycota, Blastocladiomycota, Neocallimastigomycota). +databases: [pr2] + +filters: + - column: Subdivision + pattern: Fungi + action: keep + +remove_empty: + - Subdivision diff --git a/config/filters/fungi.silva.yml b/config/filters/fungi.silva.yml new file mode 100644 index 0000000..72f682e --- /dev/null +++ b/config/filters/fungi.silva.yml @@ -0,0 +1,13 @@ +# Fungi filter — retains all fungal ASVs. +# SILVA taxonomy: Amorphea > Obazoa > Opisthokonta > Nucletmycea. +# Nucletmycea is the Family-level clade containing all Fungi in SILVA's +# 6-rank eukaryotic hierarchy. +databases: [silva] + +filters: + - column: Family + pattern: Nucletmycea + action: keep + +remove_empty: + - Family diff --git a/config/filters/helminths.pr2.yml b/config/filters/helminths.pr2.yml new file mode 100644 index 0000000..c38618a --- /dev/null +++ b/config/filters/helminths.pr2.yml @@ -0,0 +1,14 @@ +# Helminth filter — retains parasitic nematodes only. +# Keeps Nematoda (Trichuris, Strongyloides) but excludes Miculenchus +# (a moss-associated tylenchid, not a medical helminth). +databases: [pr2] + +filters: + - column: Class + pattern: Nematoda + action: keep + - column: Genus + pattern: Miculenchus + +remove_empty: + - Class diff --git a/config/filters/metazoa.silva.yml b/config/filters/metazoa.silva.yml new file mode 100644 index 0000000..d25b478 --- /dev/null +++ b/config/filters/metazoa.silva.yml @@ -0,0 +1,14 @@ +# Host contamination filter — retains metazoan sequences (SILVA limitation). +# +# SILVA's 6-rank eukaryotic hierarchy resolves all metazoa (vertebrates, +# nematodes, annelids) to Family=Holozoa. This filter cannot distinguish +# vertebrates from invertebrates. Use PR2 for vertebrate-specific filtering. +databases: [silva] + +filters: + - column: Family + pattern: Holozoa + action: keep + +remove_empty: + - Family diff --git a/config/filters/parasitic_protozoa.pr2.yml b/config/filters/parasitic_protozoa.pr2.yml new file mode 100644 index 0000000..319e57f --- /dev/null +++ b/config/filters/parasitic_protozoa.pr2.yml @@ -0,0 +1,17 @@ +# Parasitic protozoa — host-associated protists. +# +# Apicomplexa: Eimeria, Isospora (gut coccidians); Monocystis, +# Amoebogregarina, Leidyana (invertebrate gregarines) +# Parabasalia: Simplicimonas similis, Hexamastix kirbyi (gut flagellates) +# Fornicata: Iotanema spirale, Hexamita sp. (gut diplomonads) +# Bigyra: Blastocystis, Proteromonas lacertae (gut commensals/parasites) +databases: [pr2] + +filters: + - column: Subdivision + pattern: "Apicomplexa|Parabasalia|Fornicata|Bigyra" + regex: true + action: keep + +remove_empty: + - Subdivision diff --git a/config/filters/parasitic_protozoa.silva.yml b/config/filters/parasitic_protozoa.silva.yml new file mode 100644 index 0000000..05f5872 --- /dev/null +++ b/config/filters/parasitic_protozoa.silva.yml @@ -0,0 +1,20 @@ +# Parasitic protozoa — host-associated protists. +# +# SILVA equivalents of PR2 groups (matched at Family level): +# Apicomplexa -> Family: Conoidasida (Coccidia, Gregarinasina) +# Parabasalia -> Family: Trichomonadea (Hexamastix, Simplicimonas) +# Bigyra -> Family: Blastocystis + Proteromonadea +# +# Note: Fornicata (diplomonads like Iotanema, Hexamita) do not appear in +# this SILVA dataset at a distinguishable Family. If present, they would +# be under Class=Metamonada but lack a unique Family-level label here. +databases: [silva] + +filters: + - column: Family + pattern: "Conoidasida|Trichomonadea|Blastocystis|Proteromonadea" + regex: true + action: keep + +remove_empty: + - Family diff --git a/config/filters/plants.silva.yml b/config/filters/plants.silva.yml new file mode 100644 index 0000000..67df81e --- /dev/null +++ b/config/filters/plants.silva.yml @@ -0,0 +1,23 @@ +# Plants + invertebrates (dietary / environmental contamination). +# Strategy: exclude all other major groups. +# +# Note: SILVA cannot distinguish vertebrates from invertebrates — all +# metazoa resolve to Family=Holozoa. This filter retains all metazoa +# alongside plants. Use PR2 for vertebrate-specific exclusion. +databases: [silva] + +filters: + - column: Kingdom + pattern: "Bacteria|Archaea" + regex: true + - column: Family + pattern: Nucletmycea + - column: Family + pattern: "Conoidasida|Trichomonadea|Blastocystis|Proteromonadea" + regex: true + - column: Order + pattern: "Cercozoa|Ciliophora|Ochrophyta" + regex: true + +remove_empty: + - Kingdom diff --git a/config/filters/plants_invertebrates.pr2.yml b/config/filters/plants_invertebrates.pr2.yml new file mode 100644 index 0000000..a90a0dc --- /dev/null +++ b/config/filters/plants_invertebrates.pr2.yml @@ -0,0 +1,24 @@ +# Plants + non-helminth invertebrates (dietary / environmental contamination). +# Keeps: Streptophyta, Chlorophyta (plants & green algae), +# Annelida (Achaeta), Miculenchus (moss-associated nematode). +# Strategy: exclude all other groups rather than OR-ing keeps. +databases: [pr2] + +filters: + - column: Domain + pattern: "Bacteria|Archaea" + regex: true + - column: Domain + pattern: "Eukaryota:plas|Eukaryota:mito" + regex: true + - column: Subdivision + pattern: "Fungi|Apicomplexa|Ciliophora|Chrompodellids|Parabasalia|Fornicata|Cercozoa|Bigyra|Gyrista|Euglenozoa|Eumycetozoa|Foraminifera|Rhodophyta_X|Prasinodermophyta_X|Discoba_X" + regex: true + - column: Class + pattern: Craniata + - column: Genus + pattern: "Trichuris|Strongyloides" + regex: true + +remove_empty: + - Domain diff --git a/config/filters/protist.pr2.yml b/config/filters/protist.pr2.yml new file mode 100644 index 0000000..e0aed5b --- /dev/null +++ b/config/filters/protist.pr2.yml @@ -0,0 +1,28 @@ +# Protist filter — retains free-living and parasitic protists. +# Only applied when the active database is in the 'databases' list below. +databases: [pr2] +# Excludes: Bacteria, Archaea, organellar sequences, chromista sensu stricto, +# Metazoa, Fungi, Rhodophyta, Embryophyta (land plants). +# +# PR2 Division→Supergroup corrections are applied globally during +# merge_taxonomy_counts, so no mappings are needed here. + +filters: + - column: Domain + pattern: "Bacteria|Archaea" + regex: true + - column: Domain + pattern: "Eukaryota:plas|Eukaryota:mito" + regex: true + - column: Supergroup + pattern: "TSAR:chro" + - column: Subdivision + pattern: "Metazoa|Fungi" + regex: true + - column: Division + pattern: Rhodophyta + - column: Class + pattern: Embryophyceae + +remove_empty: + - Domain diff --git a/config/filters/protist.silva.yml b/config/filters/protist.silva.yml new file mode 100644 index 0000000..551b541 --- /dev/null +++ b/config/filters/protist.silva.yml @@ -0,0 +1,19 @@ +# Protist filter — retains free-living and parasitic protists. +# Only applied when the active database is SILVA. +# +# Excludes: Bacteria, Archaea, plants (Chloroplastida), fungi +# (Nucletmycea), and metazoa (Holozoa). +databases: [silva] + +filters: + - column: Kingdom + pattern: "Bacteria|Archaea" + regex: true + - column: Class + pattern: Chloroplastida + - column: Family + pattern: "Nucletmycea|Holozoa" + regex: true + +remove_empty: + - Kingdom diff --git a/config/filters/protist_filter.yml b/config/filters/protist_filter.yml deleted file mode 100644 index e2d8daf..0000000 --- a/config/filters/protist_filter.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Mappings from Division -> Supergroup -mappings: - Rhizaria: Rhizaria - Alveolata: Alveolata - Stramenopiles: Stramenopiles - Hemimastigophora: Hemimastigophora - Discoba: Discoba - Metamonada: Metamonada - Telonemia: Telonemia - Ancyromonadida: Ancyromonadida - -# Taxonomic filters: exclude rows where these columns contain these substrings -filters: - - column: Domain - pattern: Bacteria - - column: Domain - pattern: Archaea - - column: Domain - pattern: Eukaryota:plas - - column: Domain - pattern: Eukaryota:mito - - column: Supergroup - pattern: TSAR:chro - - column: Subdivision - pattern: Metazoa - - column: Subdivision - pattern: Fungi - - column: Division - pattern: Rhodophyta - - column: Class - pattern: Embryophyceae - -# Optional: whether to remove empty/unassigned domains by default -remove_empty_domain: true \ No newline at end of file diff --git a/config/filters/vertebrates.pr2.yml b/config/filters/vertebrates.pr2.yml new file mode 100644 index 0000000..fd5e0c7 --- /dev/null +++ b/config/filters/vertebrates.pr2.yml @@ -0,0 +1,12 @@ +# Host contamination filter — retains vertebrate sequences. +# PR2 taxonomy: Subdivision=Metazoa > Class=Craniata. +# Craniata covers all vertebrates (mammals, birds, fish, etc.). +databases: [pr2] + +filters: + - column: Class + pattern: Craniata + action: keep + +remove_empty: + - Class diff --git a/install.jl b/install.jl index d98830e..1ea9e62 100644 --- a/install.jl +++ b/install.jl @@ -439,7 +439,7 @@ function main() # R packages println() println(" --- R packages -----------------------------------------------------") - r_packages = ["dada2", "tidyverse"] + r_packages = ["dada2", "tidyverse", "vegan"] if prompt_yn(" Install/check R packages (dada2, tidyverse)?") install_r_packages(r_packages) end diff --git a/src/analysis.jl b/src/analysis.jl new file mode 100644 index 0000000..77ff9fb --- /dev/null +++ b/src/analysis.jl @@ -0,0 +1,1382 @@ +module Analysis + +# Orchestration module for post-pipeline analysis. +# +# Three analysis levels: +# analyse_run - per-run: pipeline summary CSV, taxa bar chart, +# filter composition, alpha diversity (PDFs) +# analyse_group - per-group: multi-run comparison charts, NMDS +# analyse_study - study-wide: cross-group NMDS, alpha comparison, +# PERMANOVA (when metadata.csv is present) +# +# analyse_run is called inside the @threads loop (no R calls). +# analyse_study is called once after the loop and dispatches to +# analyse_group internally (may call R for NMDS/PERMANOVA). +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export analyse_run, analyse_study, load_metadata + + using CSV, DataFrames, Dates, Logging, Statistics, YAML, RCall + using ..PipelineTypes, ..PipelineLog, ..Config, ..DiversityMetrics, ..PipelinePlots + + ## Column identification + """ + _sample_cols(df, db_meta) -> Vector{String} + + Return column names that hold per-sample ASV counts by excluding known + taxonomy / metadata columns and any column ending with `_dada2` or + `_boot`. + """ + function _sample_cols(df::DataFrame, db_meta::DatabaseMeta) + filter(names(df)) do col + col ∉ db_meta.noncounts && + !endswith(col, "_dada2") && + !endswith(col, "_boot") && + !endswith(col, "_vsearch") && + !isempty(col) + end + end + + ## CSV cache + # Read a CSV once and reuse. Stores (DataFrame, sample_cols). + const _CSVCache = Dict{String, Tuple{DataFrame, Vector{String}}} + + function _cached_read(cache::_CSVCache, path::String, db_meta::DatabaseMeta) + haskey(cache, path) && return cache[path] + df = CSV.read(path, DataFrame) + scols = _sample_cols(df, db_meta) + # Ensure sample columns are numeric (DADA2 tax_counts.csv may write them as strings). + for col in scols + if eltype(df[!, col]) <: AbstractString || eltype(df[!, col]) == Union{Missing, String} + df[!, col] = [ismissing(v) ? missing : parse(Float64, v) for v in df[!, col]] + elseif !(eltype(df[!, col]) <: Union{Missing, Number}) + df[!, col] = passmissing(x -> Float64(x)).(df[!, col]) + end + end + cache[path] = (df, scols) + return df, scols + end + + # Taxonomy methods: vsearch uses standard column names, dada2 uses _dada2 suffix. + const _TAX_METHODS = ["vsearch", "dada2"] + + """ + _dada2_taxonomy_view(df, levels) -> DataFrame + + Return a copy where `_dada2` taxonomy columns are renamed to standard names + (vsearch columns get `_vsearch` suffix). Allows plotting/reporting code to + work unchanged on DADA2 taxonomy. + """ + function _dada2_taxonomy_view(df::DataFrame, levels::Vector{String}) + vdf = copy(df) + for rank in levels + dada2_col = rank * "_dada2" + dada2_col in names(vdf) || continue + if rank in names(vdf) + rename!(vdf, rank => rank * "_vsearch") + end + rename!(vdf, dada2_col => rank) + end + return vdf + end + + # Check whether a DataFrame has any DADA2 taxonomy columns. + _has_dada2(df::DataFrame, levels::Vector{String}) = any(c -> endswith(c, "_dada2") && + any(r -> startswith(c, r), levels), names(df)) + + # For a given method, return the appropriate source keys from a MergedTables. + # vsearch: keys without _dada2 suffix (existing behaviour) + # dada2: for each filter stem, use "{stem}_dada2" key; for "merged" use "merged" (same CSV, both taxonomies) + function _method_source_keys(merged::MergedTables, method::String) + if method == "dada2" + keys_out = String[] + for k in sort(collect(keys(merged.tables))) + k == "merged" && continue + endswith(k, "_dada2") && push!(keys_out, k) + end + push!(keys_out, "merged") + return keys_out + else + return sort([k for k in keys(merged.tables) + if k != "merged" && !endswith(k, "_dada2")]) |> + ks -> vcat(ks, ["merged"]) + end + end + + # Get the CSV path and apply taxonomy view for a method. + function _method_df(raw_df::DataFrame, method::String, levels::Vector{String}) + method == "dada2" ? _dada2_taxonomy_view(raw_df, levels) : raw_df + end + + # Source dirname for method-qualified keys (strip _dada2 suffix for directory names). + _method_source_dirname(key::String) = _source_dirname( + endswith(key, "_dada2") ? key[1:end-6] : key) + + # Lowest assigned taxonomic rank for a row. + function _lowest_rank_label(row, levels::Vector{String}) + label = "Unclassified" + for rank in levels + if hasproperty(row, Symbol(rank)) + val = row[Symbol(rank)] + if !ismissing(val) && !isempty(strip(string(val))) + label = string(val) + end + end + end + return label + end + + ## Stem-to-column matching + # Match pipeline_stats sample stems to full FASTQ column names. + # Returns a Dict mapping each stem to the matching column name + # (or the stem itself if no match is found). + function _stem_to_colname(stems::AbstractVector, col_names::Vector{String}) + mapping = Dict{String, String}() + for stem in stems + s = String(stem) + matched = false + for col in col_names + if startswith(col, s * "_") || col == s + mapping[s] = col + matched = true + break + end + end + matched || (mapping[s] = s) + end + return mapping + end + + ## Source label helper + _source_label(key::String) = key == "merged" ? "unfiltered" : key + + ## Taxa rank iteration + # Derive taxa chart ranks from database levels and optional config override. + function _taxa_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) + configured = get(get(analysis_cfg, "taxa_bar", Dict()), "ranks", nothing) + if !isnothing(configured) && configured isa Vector + return Tuple{String, Union{String,Nothing}}[ + (lowercase(string(r)), string(r) == "asv" ? nothing : string(r)) for r in configured + ] + end + n = length(levels) + ranks = Tuple{String, Union{String,Nothing}}[("asv", nothing)] + n >= 1 && push!(ranks, (lowercase(levels[end]), levels[end])) + n >= 2 && push!(ranks, (lowercase(levels[end-1]), levels[end-1])) + return ranks + end + + # Derive report ranks from database levels and optional config override. + function _report_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) + configured = get(get(analysis_cfg, "taxa_bar", Dict()), "report_ranks", nothing) + if !isnothing(configured) && configured isa Vector + return [(string(r), string(r)) for r in configured] + end + n = length(levels) + ranks = Tuple{String,String}[] + n >= 3 && push!(ranks, (levels[end-2], levels[end-2])) + n >= 2 && push!(ranks, (levels[end-1], levels[end-1])) + n >= 1 && push!(ranks, (levels[end], levels[end])) + return ranks + end + + # Source key -> directory name for figures. + _source_dirname(key::String) = key == "merged" ? "unfiltered" : key + + function _generate_taxa_charts(df::DataFrame, scols::Vector{String}, + figures_dir::String, top_n::Int, + subtitle::Union{Nothing,String}, + source_key::String; + ranks, rank_order::Vector{String}) + src_dir = _source_dirname(source_key) + for (rankdir, rank) in ranks + dir = joinpath(figures_dir, src_dir, rankdir) + mkpath(dir) + taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar.pdf"); + top_n, rank, relative=true, subtitle, rank_order) + taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar_absolute.pdf"); + top_n, rank, relative=false, subtitle, rank_order) + end + end + + ## Top-taxa report section + function _top_taxa_section(df::DataFrame, scols::Vector{String}, + rank_name::String, rank_col::String; n::Int=20) + sym = Symbol(rank_col) + hasproperty(df, sym) || return "" + + # Label each row, sum counts across all samples. + labels = String[] + totals = Float64[] + for row in eachrow(df) + val = row[sym] + label = (ismissing(val) || isempty(strip(string(val)))) ? "Unclassified" : string(val) + push!(labels, label) + push!(totals, sum(col -> begin + v = row[Symbol(col)] + ismissing(v) ? 0.0 : Float64(v) + end, scols)) + end + + # Aggregate by label. + agg = Dict{String, Float64}() + for (l, t) in zip(labels, totals) + agg[l] = get(agg, l, 0.0) + t + end + + sorted = sort(collect(agg); by=last, rev=true) + grand_total = sum(last, sorted; init=0.0) + + buf = IOBuffer() + print(buf, rpad("Rank", 4), rpad(rank_name, 30), rpad("Reads", 14), "Percent\n") + for (i, (label, count)) in enumerate(sorted) + i > n && break + pct = grand_total > 0 ? round(100.0 * count / grand_total; digits=1) : 0.0 + print(buf, rpad(string(i), 4), rpad(label, 30), + rpad(string(Int(count)), 14), "$(pct)%\n") + end + return String(take!(buf)) + end + + ## Report generator (dual: filtered + merged) + function _generate_report(df::DataFrame, scols::Vector{String}, + stats_df, merged_df::DataFrame, + src_key::String, report_path::String, + run_name::String; + stats_key::String=src_key, + report_ranks::Vector{Tuple{String,String}}=[("Family","Family"),("Genus","Genus"),("Species","Species")]) + src_label = _source_label(src_key) + report_sections = Pair{String, String}[] + + # Pipeline statistics table. + if !isnothing(stats_df) + buf = IOBuffer() + _print_stats_table(buf, stats_df) + if "input" in names(stats_df) + total_input = sum(stats_df.input) + # Use the reads column matching this report's stats key. + target_col = "reads_" * stats_key + if target_col in names(stats_df) + total_final = sum(stats_df[!, target_col]) + pct = total_input > 0 ? round(100.0 * total_final / total_input; digits=1) : 0.0 + println(buf, "\nOverall retention (input -> $(src_label)): $(pct)%") + end + end + push!(report_sections, "Pipeline Statistics" => String(take!(buf))) + end + + # ASV summary. + merged_asvs = nrow(merged_df) + filter_asvs = nrow(df) + buf = IOBuffer() + println(buf, "Total ASVs (merged): $merged_asvs") + if src_key != "merged" + println(buf, "ASVs after filter: $filter_asvs") + end + push!(report_sections, "ASV Summary" => String(take!(buf))) + + # Alpha diversity table. + alpha = _compute_alpha(df, scols) + buf = IOBuffer() + _print_alpha_table(buf, alpha) + push!(report_sections, "Alpha Diversity" => String(take!(buf))) + + # Top-20 taxa tables. + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(df, scols, rank_name, rank_col; n=20) + !isempty(section) && push!(report_sections, "Top 20 $(rank_name) ($(src_label))" => section) + end + + _write_report(report_path, + "Analysis Report: $run_name\n Source: $src_label", + report_sections) + end + + ## Text report writer + function _write_report(path::String, title::String, + sections::Vector{Pair{String, String}}) + open(path, "w") do io + sep = "=" ^ 64 + println(io, sep) + println(io, " ", title) + println(io, " Generated: ", Dates.format(now(), "yyyy-mm-dd HH:MM:SS")) + println(io, sep) + for (heading, body) in sections + println(io) + println(io, "--- ", heading, " ---") + println(io, body) + end + end + @info "Written: $path" + end + + ## Source table selection + # Pick the primary analysis source from a MergedTables: the first + # filter CSV (alphabetically) if any filters exist, else "merged". + function _source_key(merged::MergedTables) + "merged" + end + + ## Priority-based filter composition + # Assign each ASV to the first matching filter (by pipeline.yml order). + # Returns (filter_totals, sample_cols) or (nothing, nothing). + function _priority_filter_composition(merged::MergedTables, cache::_CSVCache, + db_meta::DatabaseMeta) + merged_csv = merged.tables["merged"] + isfile(merged_csv) || return nothing, nothing + merged_df, merged_scols = _cached_read(cache, merged_csv, db_meta) + isempty(merged_scols) && return nothing, nothing + + # Use filter_order for priority; fall back to sorted keys. + filter_keys = !isempty(merged.filter_order) ? merged.filter_order : + sort([k for k in keys(merged.tables) if k != "merged"]) + isempty(filter_keys) && return nothing, nothing + + # Per-sample totals from merged (unfiltered). + merged_totals = Float64[ + sum(v -> ismissing(v) ? 0.0 : Float64(v), merged_df[!, s]) + for s in merged_scols + ] + + # Load filter DataFrames. + filter_dfs = Dict{String, DataFrame}() + for fk in filter_keys + haskey(merged.tables, fk) || continue + fpath = merged.tables[fk] + (isfile(fpath) && filesize(fpath) > 0) || continue + filter_dfs[fk] = first(_cached_read(cache, fpath, db_meta)) + end + + # Track which SeqNames have been claimed. + claimed = Set{String}() + filter_totals = Dict{String, Vector{Float64}}() + + for fk in filter_keys + haskey(filter_dfs, fk) || continue + fdf = filter_dfs[fk] + totals = zeros(Float64, length(merged_scols)) + for row in eachrow(fdf) + seq = string(row.SeqName) + seq in claimed && continue + push!(claimed, seq) + for (i, s) in enumerate(merged_scols) + v = hasproperty(row, Symbol(s)) ? row[Symbol(s)] : missing + totals[i] += ismissing(v) ? 0.0 : Float64(v) + end + end + filter_totals[fk] = totals + end + + # Unclassified = merged totals minus all attributed. + attributed = zeros(Float64, length(merged_scols)) + for vals in values(filter_totals) + attributed .+= vals + end + unclassified = max.(merged_totals .- attributed, 0.0) + if any(>(0), unclassified) + filter_totals["Unclassified"] = unclassified + end + + return filter_totals, merged_scols + end + + ## Count-matrix builders + # Extract a samples x features count matrix from a single DataFrame. + # Rows = samples (one per element of `scols`), columns = ASV rows. + function _counts_matrix(df::DataFrame, scols::Vector{String}) + n_samples = length(scols) + n_features = nrow(df) + mat = zeros(Float64, n_samples, n_features) + for (j, row) in enumerate(eachrow(df)) + for (i, col) in enumerate(scols) + v = row[Symbol(col)] + mat[i, j] = ismissing(v) ? 0.0 : Float64(v) + end + end + return mat + end + + # Build a taxonomy-aggregated count matrix across multiple runs. + # + # ASV identifiers (seq1, seq2, ...) are local to each run and cannot + # be compared directly. This function aggregates counts to the + # lowest assigned taxonomic rank, producing a shared feature space + # suitable for between-run Bray-Curtis / NMDS. + # + # Returns (matrix, all_sample_names, taxon_labels). + function _build_combined_counts( + dfs::Vector{DataFrame}, + scols_per_df::Vector{Vector{String}}, + levels::Vector{String}, + ) + # taxon -> sample -> accumulated count + taxa_counts = Dict{String, Dict{String, Float64}}() + all_samples = String[] + + for (df, scols) in zip(dfs, scols_per_df) + append!(all_samples, scols) + for row in eachrow(df) + label = _lowest_rank_label(row, levels) + td = get!(taxa_counts, label, Dict{String, Float64}()) + for col in scols + v = row[Symbol(col)] + td[col] = get(td, col, 0.0) + (ismissing(v) ? 0.0 : Float64(v isa AbstractString ? parse(Float64, v) : v)) + end + end + end + + taxa_labels = sort(collect(keys(taxa_counts))) + n_samples = length(all_samples) + n_taxa = length(taxa_labels) + mat = zeros(Float64, n_samples, n_taxa) + + for (j, taxon) in enumerate(taxa_labels) + td = taxa_counts[taxon] + for (i, sample) in enumerate(all_samples) + mat[i, j] = get(td, sample, 0.0) + end + end + + return mat, all_samples, taxa_labels + end + + ## Alpha diversity helper + # Compute per-sample alpha diversity from a merged/filtered CSV. + function _compute_alpha(df::DataFrame, scols::Vector{String}) + out = DataFrame(sample=String[], richness=Int[], + shannon=Float64[], simpson=Float64[]) + for col in scols + counts = [ismissing(v) ? 0 : Int(v isa AbstractString ? parse(Int, v) : round(Int, Float64(v))) for v in df[!, col]] + push!(out, (col, richness(counts), shannon(counts), simpson(counts))) + end + return out + end + + # Metadata + """ + load_metadata(start_dir, study_dir) -> Union{DataFrame, Nothing} + + Walk from `start_dir` upward to `study_dir` (inclusive), returning the + first `metadata.csv` found as a DataFrame. Returns `nothing` when no + metadata file exists at any level. + """ + function load_metadata(start_dir::String, study_dir::String) + dir = abspath(start_dir) + stop = abspath(study_dir) + while true + csv = joinpath(dir, "metadata.csv") + isfile(csv) && return CSV.read(csv, DataFrame) + dir == stop && break + parent = dirname(dir) + parent == dir && break # filesystem root + dir = parent + end + return nothing + end + + # R: NMDS + PERMANOVA + # NMDS via vegan::metaMDS. + # `mat` is samples x features (community matrix). + # Returns (coords::Matrix{Float64}[nx2], stress::Float64). + # On failure returns a NaN-filled matrix and NaN stress. + function _run_nmds(mat::Matrix{Float64}, r_lock::ReentrantLock) + lock(r_lock) do + @rput mat + R""" + suppressPackageStartupMessages(library(vegan)) + set.seed(42) + nmds_res <- tryCatch( + metaMDS(mat, distance = "bray", k = 2, trymax = 200, + autotransform = FALSE, trace = 0), + error = function(e) NULL + ) + if (!is.null(nmds_res)) { + nmds_coords <- nmds_res$points + nmds_stress <- nmds_res$stress + } else { + nmds_coords <- matrix(NA_real_, nrow = nrow(mat), ncol = 2) + nmds_stress <- NA_real_ + } + """ + coords = rcopy(R"nmds_coords")::Matrix{Float64} + stress = rcopy(R"nmds_stress")::Float64 + return coords, stress + end + end + + # PERMANOVA via vegan::adonis2. + # Returns the captured text output, or `nothing` on failure. + function _run_permanova(mat::Matrix{Float64}, metadata::DataFrame, + r_lock::ReentrantLock) + covariates = [c for c in names(metadata) if lowercase(c) != "sample"] + isempty(covariates) && return nothing + formula_rhs = join(covariates, " + ") + + lock(r_lock) do + meta_r = copy(metadata) + @rput mat meta_r formula_rhs + R""" + suppressPackageStartupMessages(library(vegan)) + set.seed(42) + dist_mat <- vegdist(mat, method = "bray") + form <- as.formula(paste("dist_mat ~", formula_rhs)) + perm_res <- tryCatch( + adonis2(form, data = meta_r, permutations = 999), + error = function(e) NULL + ) + if (!is.null(perm_res)) { + perm_text <- paste(capture.output(print(perm_res)), collapse = "\n") + } else { + perm_text <- NA_character_ + } + """ + txt = rcopy(R"perm_text") + return (ismissing(txt) || txt == "NA") ? nothing : txt + end + end + + ## Pipeline summary CSV + function _pipeline_summary(project::ProjectCtx, merged::MergedTables, + db_meta::DatabaseMeta) + analysis_dir = joinpath(project.dir, "analysis") + summary_path = joinpath(analysis_dir, "pipeline_summary.csv") + + # Read DADA2's pipeline_stats.csv. + # R's write.csv writes row names as the first (unnamed) column. + stats_csv = joinpath(project.dir, "dada2", "Tables", "pipeline_stats.csv") + if !isfile(stats_csv) + @warn "pipeline_stats.csv not found at $stats_csv - skipping pipeline_summary" + return nothing + end + + stats = CSV.read(stats_csv, DataFrame) + first_col = names(stats)[1] + if first_col != "sample" + rename!(stats, first_col => "sample") + end + + # Add per-sample read totals from each merged table. + # Stems in pipeline_stats (e.g. "JIN-Nu-ves55") must be matched to + # full FASTQ column names (e.g. "JIN-Nu-ves55_R1_filt.fastq.gz"). + for key in sort(collect(keys(merged.tables))) + csv_path = merged.tables[key] + isfile(csv_path) || continue + df = CSV.read(csv_path, DataFrame) + scols = _sample_cols(df, db_meta) + + totals = Dict{String, Int}() + for col in scols + totals[col] = sum(v -> ismissing(v) ? 0 : Int(v), df[!, col]; init=0) + end + + # Match stems to full column names. + stem_map = _stem_to_colname(stats.sample, scols) + + # Prefix with "reads_" to avoid colliding with DADA2's own columns. + col_name = "reads_" * key + stats[!, col_name] = [get(totals, get(stem_map, String(s), ""), 0) + for s in stats.sample] + end + + mkpath(analysis_dir) + CSV.write(summary_path, stats) + @info "Written: $summary_path" + log_written(project, summary_path) + return stats + end + + ## Level 1 - Per-run analysis + """ + analyse_run(project, merged, asvs, db_meta; plot_lock=nothing) + + Produce per-run analysis outputs under `{project.dir}/analysis/`. + When `plot_lock` is provided, CairoMakie calls are serialized behind + it so multiple runs can prepare data in parallel. + """ + function analyse_run(project::ProjectCtx, merged::MergedTables, + asvs::ASVResult, db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(project.dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + merged_csv = merged.tables["merged"] + + filter_keys = sort([k for k in keys(merged.tables) if k != "merged"]) + has_filters = !isempty(filter_keys) + all_source_keys = vcat(filter_keys, ["merged"]) + + # Output paths. + summary_path = joinpath(analysis_dir, "pipeline_summary.csv") + stages_pdf = joinpath(figures_dir, "pipeline_stages.pdf") + alpha_pdf = joinpath(figures_dir, "alpha_diversity.pdf") + filter_pdf = joinpath(figures_dir, "filter_composition.pdf") + + # Read analysis config from the cascade (needed for skip guard and charts). + config_path = write_run_config(project) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + taxa_cfg = get(analysis_cfg, "taxa_bar", Dict()) + top_n = get(taxa_cfg, "top_n", 15) + taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(db_meta.levels, analysis_cfg) + + required_outputs = String[summary_path, stages_pdf, alpha_pdf] + has_filters && push!(required_outputs, filter_pdf) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + sd = _method_source_dirname(src) + for (rankdir, _) in taxa_ranks + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) + end + end + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + + # skip guard + merged_mtime = isfile(merged_csv) ? mtime(merged_csv) : time() + if all(isfile, required_outputs) && + all(f -> mtime(f) > merged_mtime, required_outputs) + @info "Skipping analyse_run: outputs up to date in $analysis_dir" + return + end + + mkpath(figures_dir) + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + # CSV cache + cache = _CSVCache() + + src_key = _source_key(merged) + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # pipeline summary + stats_df = _pipeline_summary(project, merged, db_meta) + + # pipeline stages plot + if !isnothing(stats_df) + _plot() do + pipeline_stats_plot(stats_df, stages_pdf; subtitle) + end + @info "Written: $stages_pdf" + log_written(project, stages_pdf) + end + + # taxa bar charts for all sources at multiple ranks (dual method) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + # For dada2 method with "merged" key, use merged.csv; + # for _dada2 suffixed keys, use the corresponding CSV. + csv_key = src + src_csv_path = get(merged.tables, csv_key, "") + if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 + # For "merged" key under dada2 method, fall back to merged.csv + src == "merged" || continue + src_csv_path = merged.tables["merged"] + (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue + end + raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) + isempty(src_scols) && continue + # Skip dada2 method if no dada2 taxonomy columns + method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue + view_df = _method_df(raw_df, method, db_meta.levels) + sd = _method_source_dirname(src) + src_sub = "Source: $sd ($method)" + method_fig_dir = joinpath(figures_dir, method) + _plot() do + _generate_taxa_charts(view_df, src_scols, method_fig_dir, + top_n, src_sub, src == "merged" ? "merged" : + (endswith(src, "_dada2") ? src[1:end-6] : src); + ranks=taxa_ranks, rank_order=db_meta.levels) + end + end + end + + # filter composition (priority-based) + if has_filters + filter_totals, comp_scols = _priority_filter_composition(merged, cache, db_meta) + if !isnothing(filter_totals) && !isempty(filter_totals) + _plot() do + filter_composition_plot(filter_totals, comp_scols, filter_pdf; + subtitle, colour_overrides=merged.filter_colours) + end + @info "Written: $filter_pdf" + log_written(project, filter_pdf) + end + end + + # alpha diversity (from primary source) + primary_csv = merged.tables[src_key] + if isfile(primary_csv) && filesize(primary_csv) > 0 + prim_df, prim_scols = _cached_read(cache, primary_csv, db_meta) + if !isempty(prim_scols) + alpha = _compute_alpha(prim_df, prim_scols) + _plot() do + alpha_diversity_plot(alpha, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(project, alpha_pdf) + end + end + + # analysis reports (dual method, one per source per method) + run_name = basename(project.dir) + merged_df_full, _ = _cached_read(cache, merged_csv, db_meta) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + csv_key = src + src_csv_path = get(merged.tables, csv_key, "") + if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 + src == "merged" || continue + src_csv_path = merged.tables["merged"] + (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue + end + raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) + isempty(src_scols) && continue + method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue + view_df = _method_df(raw_df, method, db_meta.levels) + sd = _method_source_dirname(src) + report_path = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + clean_src = src == "merged" ? "merged" : (endswith(src, "_dada2") ? src[1:end-6] : src) + _generate_report(view_df, src_scols, stats_df, merged_df_full, + clean_src, report_path, run_name; + stats_key=src, report_ranks=report_ranks) + log_written(project, report_path) + end + end + end + + # Format a pipeline stats DataFrame as a text table. + function _print_stats_table(io::IO, stats_df::DataFrame) + cols = names(stats_df) + # Header. + print(io, rpad("Sample", 20)) + for col in cols + col == "sample" && continue + print(io, rpad(col, 16)) + end + println(io) + # Rows. + for row in eachrow(stats_df) + print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) + for col in cols + col == "sample" && continue + v = row[Symbol(col)] + print(io, rpad(ismissing(v) ? "0" : string(Int(v)), 16)) + end + println(io) + end + end + + # Format an alpha diversity DataFrame as a text table. + function _print_alpha_table(io::IO, alpha_df::DataFrame) + print(io, rpad("Sample", 20)) + println(io, rpad("Richness", 12), rpad("Shannon", 12), "Simpson") + for row in eachrow(alpha_df) + print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) + print(io, rpad(string(row.richness), 12)) + print(io, rpad(string(round(row.shannon; digits=3)), 12)) + println(io, round(row.simpson; digits=3)) + end + end + + ## Level 2 - Per-group analysis + function _analyse_group(group_dir::String, + members::Vector{Tuple{ProjectCtx, MergedTables}}, + r_lock::ReentrantLock, + db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(group_dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + + alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") + nmds_pdf = joinpath(figures_dir, "nmds.pdf") + filter_pdf = joinpath(figures_dir, "filter_composition.pdf") + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + # Determine source keys. + src_key = _source_key(members[1][2]) + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # Derive taxa/report ranks from db_meta. + analysis_cfg = Dict() + try + config_path = write_run_config(members[1][1]) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + catch; end + taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(db_meta.levels, analysis_cfg) + + filter_keys = sort([k for k in keys(members[1][2].tables) if k != "merged"]) + has_any_filters = any(pair -> length(pair[2].tables) > 1, members) + all_source_keys = vcat(filter_keys, ["merged"]) + + # skip guard + newest_merged = maximum( + mtime(m.tables["merged"]) + for (_, m) in members if isfile(m.tables["merged"]); + init=0.0 + ) + required_outputs = String[alpha_pdf, nmds_pdf] + has_any_filters && push!(required_outputs, filter_pdf) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + sd = _method_source_dirname(src) + for (rankdir, _) in taxa_ranks + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison_absolute.pdf")) + end + end + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + + if all(isfile, required_outputs) && + all(f -> mtime(f) > newest_merged, required_outputs) + @info "Skipping analyse_group: outputs up to date in $figures_dir" + return + end + + reset_log(group_dir) + mkpath(figures_dir) + + # CSV cache + cache = _CSVCache() + + # helper: collect per-run data for a given source key + function _collect_source_data(source::String) + dfs = DataFrame[] + scols = Vector{String}[] + names_ = String[] + labels = String[] + all_s = String[] + for (proj, merged) in members + haskey(merged.tables, source) || continue + csv = merged.tables[source] + isfile(csv) || continue + df, sc = _cached_read(cache, csv, db_meta) + isempty(sc) && continue + rn = basename(proj.dir) + push!(dfs, df); push!(scols, sc); push!(names_, rn) + append!(all_s, sc); append!(labels, fill(rn, length(sc))) + end + return dfs, scols, names_, labels, all_s + end + + # Primary source data (for NMDS, alpha, filter composition). + run_dfs, run_scols, run_names, run_labels, all_scols = + _collect_source_data(src_key) + isempty(run_dfs) && return + + # taxa bar & group comparison for ALL sources (dual method) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + s_dfs, s_scols, s_names, _, s_all = _collect_source_data( + endswith(src, "_dada2") ? src : src) + # For "merged" under dada2, use the same "merged" key + if isempty(s_dfs) && method == "dada2" && src == "merged" + s_dfs, s_scols, s_names, _, s_all = _collect_source_data("merged") + end + isempty(s_dfs) && continue + # Skip dada2 if no dada2 columns + method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue + view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + sd = _method_source_dirname(src) + src_sub = "Source: $sd ($method)" + method_fig_dir = joinpath(figures_dir, method) + # Use cleaned source key for directory naming + clean_src = src == "merged" ? "merged" : + (endswith(src, "_dada2") ? src[1:end-6] : src) + _plot() do + _generate_taxa_charts(combined, s_all, method_fig_dir, 15, src_sub, clean_src; + ranks=taxa_ranks, rank_order=db_meta.levels) + for (rankdir, rank) in taxa_ranks + dir = joinpath(method_fig_dir, _source_dirname(clean_src), rankdir) + mkpath(dir) + group_comparison_chart(view_dfs, s_scols, s_names, + joinpath(dir, "group_comparison.pdf"); + top_n=15, rank, relative=true, + subtitle=src_sub, + rank_order=db_meta.levels) + group_comparison_chart(view_dfs, s_scols, s_names, + joinpath(dir, "group_comparison_absolute.pdf"); + top_n=15, rank, relative=false, + subtitle=src_sub, + rank_order=db_meta.levels) + end + end + end + end + + # group alpha diversity (boxplot by run) + all_alpha = DataFrame[] + alpha_labels = String[] + for (df, scols, rname) in zip(run_dfs, run_scols, run_names) + alpha = _compute_alpha(df, scols) + push!(all_alpha, alpha) + push!(alpha_labels, rname) + end + if !isempty(all_alpha) + _plot() do + alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(group_dir, alpha_pdf) + end + + # group NMDS + mat, _, _ = _build_combined_counts(run_dfs, run_scols, db_meta.levels) + nmds_stress = NaN + if size(mat, 1) >= 3 + coords, stress = _run_nmds(mat, r_lock) + if !any(isnan, coords) + nmds_cfg = Dict() + try + config_path = write_run_config(members[1][1]) + nmds_cfg = get(get(YAML.load_file(config_path), "analysis", Dict()), + "nmds", Dict()) + catch; end + max_stress = get(nmds_cfg, "max_stress", 0.2) + if !isnan(stress) && stress > max_stress + @warn "NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress for group $(basename(group_dir))" + pipeline_log(group_dir, "WARN: NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress") + end + nmds_stress = stress + _plot() do + nmds_plot(coords, all_scols, nmds_pdf; + colour_by=run_labels, stress=stress, subtitle) + end + @info "Written: $nmds_pdf" + log_written(group_dir, nmds_pdf) + else + @warn "NMDS failed for group $(basename(group_dir))" + pipeline_log(group_dir, "WARN: NMDS failed for group $(basename(group_dir))") + end + else + @warn "Too few samples ($(size(mat, 1))) for NMDS in group $(basename(group_dir))" + end + + # group filter composition + if has_any_filters + group_filter_totals = Dict{String, Vector{Float64}}() + group_sample_names = String[] + group_filter_colours = Dict{String,String}() + for (proj, merged) in members + ft, scols_m = _priority_filter_composition(merged, cache, db_meta) + isnothing(ft) && continue + append!(group_sample_names, scols_m) + merge!(group_filter_colours, merged.filter_colours) + for (fk, vals) in ft + existing = get(group_filter_totals, fk, Float64[]) + group_filter_totals[fk] = vcat(existing, vals) + end + end + + if haskey(group_filter_totals, "Unclassified") && + !any(>(0), group_filter_totals["Unclassified"]) + delete!(group_filter_totals, "Unclassified") + end + + if !isempty(group_filter_totals) && !isempty(group_sample_names) + _plot() do + filter_composition_plot(group_filter_totals, group_sample_names, filter_pdf; + subtitle, colour_overrides=group_filter_colours) + end + @info "Written: $filter_pdf" + log_written(group_dir, filter_pdf) + end + end + + # analysis reports (dual method, per source, each with top-20 tables) + # Shared sections: Per-Run Summary + NMDS. + shared_sections = Pair{String, String}[] + + buf = IOBuffer() + print(buf, rpad("Run", 20)) + println(buf, rpad("Samples", 10), rpad("Mean richness", 16), + rpad("Mean Shannon", 16), "Mean Simpson") + for (adf, rname) in zip(all_alpha, alpha_labels) + n = nrow(adf) + mr = round(mean(adf.richness); digits=0) + ms = round(mean(adf.shannon); digits=3) + mp = round(mean(adf.simpson); digits=3) + if n > 1 + sr = round(std(adf.richness); digits=0) + ss = round(std(adf.shannon); digits=2) + sp = round(std(adf.simpson); digits=2) + print(buf, rpad(rname, 20)) + println(buf, rpad(string(n), 10), + rpad("$(Int(mr)) +/- $(Int(sr))", 16), + rpad("$ms +/- $ss", 16), + "$mp +/- $sp") + else + print(buf, rpad(rname, 20)) + println(buf, rpad(string(n), 10), + rpad(string(Int(mr)), 16), + rpad(string(ms), 16), + string(mp)) + end + end + push!(shared_sections, "Per-Run Summary" => String(take!(buf))) + + if !isnan(nmds_stress) + quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" + push!(shared_sections, "NMDS" => + "Stress: $(round(nmds_stress; digits=3)) ($quality)") + end + + # Write one report per source per method. + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + s_dfs, s_scols, _, _, s_all = _collect_source_data( + endswith(src, "_dada2") ? src : src) + if isempty(s_dfs) && method == "dada2" && src == "merged" + s_dfs, s_scols, _, _, s_all = _collect_source_data("merged") + end + isempty(s_dfs) && continue + method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue + view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + sd = _method_source_dirname(src) + sl = sd == "unfiltered" ? "unfiltered" : sd + + sections = copy(shared_sections) + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(combined, s_all, rank_name, rank_col; n=20) + !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) + end + + rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + _write_report(rpath, + "Comparison Report\n Runs: $(join(run_names, ", "))\n Source: $sl ($method)", + sections) + log_written(group_dir, rpath) + end + end + end + + ## Level 3 - Study-level analysis + function _analyse_study_level(study_dir::String, + valid::Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}, + r_lock::ReentrantLock, + default_db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(study_dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + nmds_pdf = joinpath(figures_dir, "nmds.pdf") + alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") + perm_txt = joinpath(analysis_dir, "permanova.txt") + + src_key = _source_key(valid[1][2]) + filter_keys = sort([k for k in keys(valid[1][2].tables) if k != "merged"]) + + # Derive taxa/report ranks from the first project's db_meta. + analysis_cfg = Dict() + try + config_path = write_run_config(valid[1][1]) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + catch; end + taxa_ranks = _taxa_ranks(default_db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(default_db_meta.levels, analysis_cfg) + + # skip guard + newest_merged = maximum( + mtime(m.tables["merged"]) + for (_, m) in valid if isfile(m.tables["merged"]); + init=0.0 + ) + required_outputs = [nmds_pdf, alpha_pdf] + for method in _TAX_METHODS + msrc_keys = _method_source_keys(valid[1][2], method) + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + if all(isfile, required_outputs) && + all(f -> mtime(f) > newest_merged, required_outputs) + @info "Skipping study-level analysis: outputs up to date" + return + end + + reset_log(study_dir) + mkpath(figures_dir) + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + cache = _CSVCache() + + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # group projects by parent dir + groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() + for (proj, merged, dm) in valid + gdir = dirname(proj.dir) + push!(get!(groups, gdir, []), (proj, merged, dm)) + end + + # collect data across all groups + run_dfs = DataFrame[] + run_scols = Vector{String}[] + all_scols = String[] + group_labels = String[] # group name per sample + run_labels = String[] # run name per sample (for shape) + group_names = String[] + + for (gdir, members) in groups + gname = basename(gdir) + push!(group_names, gname) + for (proj, merged, dm) in members + src_csv = merged.tables[_source_key(merged)] + isfile(src_csv) || continue + df, scols = _cached_read(cache, src_csv, dm) + isempty(scols) && continue + rname = basename(proj.dir) + push!(run_dfs, df) + push!(run_scols, scols) + append!(all_scols, scols) + append!(group_labels, fill(gname, length(scols))) + append!(run_labels, fill(rname, length(scols))) + end + end + + isempty(run_dfs) && return + + # study NMDS + mat, _, _ = _build_combined_counts(run_dfs, run_scols, default_db_meta.levels) + nmds_stress = NaN + if size(mat, 1) >= 3 + coords, stress = _run_nmds(mat, r_lock) + if !any(isnan, coords) + nmds_stress = stress + _plot() do + nmds_plot(coords, all_scols, nmds_pdf; + colour_by=group_labels, shape_by=run_labels, + colour_label="Group", shape_label="Run", + stress=stress, subtitle) + end + @info "Written: $nmds_pdf" + log_written(study_dir, nmds_pdf) + else + @warn "Study-level NMDS failed" + pipeline_log(study_dir, "WARN: Study-level NMDS failed") + end + else + @warn "Too few samples ($(size(mat, 1))) for study-level NMDS" + end + + # study alpha boxplot by group + all_alpha = DataFrame[] + alpha_labels = String[] + for (gdir, members) in groups + gname = basename(gdir) + combined_alpha = DataFrame(sample=String[], richness=Int[], + shannon=Float64[], simpson=Float64[]) + for (_, merged, dm) in members + src_csv = merged.tables[_source_key(merged)] + isfile(src_csv) || continue + df, scols = _cached_read(cache, src_csv, dm) + isempty(scols) && continue + append!(combined_alpha, _compute_alpha(df, scols)) + end + if nrow(combined_alpha) > 0 + push!(all_alpha, combined_alpha) + push!(alpha_labels, gname) + end + end + + if !isempty(all_alpha) + _plot() do + alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(study_dir, alpha_pdf) + end + + # PERMANOVA (only when metadata.csv exists) + metadata = load_metadata(study_dir, study_dir) + if !isnothing(metadata) && size(mat, 1) >= 3 + sample_col = "sample" in names(metadata) ? "sample" : + "Sample" in names(metadata) ? "Sample" : nothing + if !isnothing(sample_col) + meta_idx = indexin(all_scols, String.(metadata[!, sample_col])) + if all(!isnothing, meta_idx) + meta_matched = metadata[collect(meta_idx), :] + perm_result = _run_permanova(mat, meta_matched, r_lock) + if !isnothing(perm_result) + mkpath(dirname(perm_txt)) + write(perm_txt, perm_result) + @info "Written: $perm_txt" + log_written(study_dir, perm_txt) + end + else + @warn "Not all samples found in metadata.csv - skipping PERMANOVA" + end + else + @warn "metadata.csv has no 'sample' column - skipping PERMANOVA" + end + end + + # dual analysis reports + report_sections = Pair{String, String}[] + + # Group overview. + buf = IOBuffer() + println(buf, "Groups: ", join(group_names, ", ")) + println(buf, "Total samples: ", length(all_scols)) + push!(report_sections, "Overview" => String(take!(buf))) + + # NMDS info. + if !isnan(nmds_stress) + quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" + push!(report_sections, "NMDS" => + "Stress: $(round(nmds_stress; digits=3)) ($quality)") + end + + # Alpha diversity by group. + if !isempty(all_alpha) + buf = IOBuffer() + print(buf, rpad("Group", 20)) + println(buf, rpad("Samples", 10), rpad("Mean richness", 16), + rpad("Mean Shannon", 16), "Mean Simpson") + for (adf, gname) in zip(all_alpha, alpha_labels) + n = nrow(adf) + mr = round(mean(adf.richness); digits=0) + ms = round(mean(adf.shannon); digits=3) + mp = round(mean(adf.simpson); digits=3) + print(buf, rpad(gname, 20)) + if n > 1 + sr = round(std(adf.richness); digits=0) + ss = round(std(adf.shannon); digits=2) + sp = round(std(adf.simpson); digits=2) + println(buf, rpad(string(n), 10), + rpad("$(Int(mr)) +/- $(Int(sr))", 16), + rpad("$ms +/- $ss", 16), + "$mp +/- $sp") + else + println(buf, rpad(string(n), 10), + rpad(string(Int(mr)), 16), + rpad(string(ms), 16), + string(mp)) + end + end + push!(report_sections, "Alpha Diversity by Group" => String(take!(buf))) + end + + # Write one report per source per method. + for method in _TAX_METHODS + msrc_keys = _method_source_keys(valid[1][2], method) + for src in msrc_keys + src_run_dfs = DataFrame[] + src_all_scols = String[] + # Determine the table key to look up + table_key = src + for (gdir_inner, members_inner) in groups + for (proj, merged, dm) in members_inner + csv = get(merged.tables, table_key, "") + # For "merged" under dada2, fall back to "merged" + if isempty(csv) && method == "dada2" && src == "merged" + csv = get(merged.tables, "merged", "") + end + (isempty(csv) || !isfile(csv)) && continue + df, sc = _cached_read(cache, csv, dm) + isempty(sc) && continue + push!(src_run_dfs, df) + append!(src_all_scols, sc) + end + end + isempty(src_run_dfs) && continue + method == "dada2" && !_has_dada2(src_run_dfs[1], default_db_meta.levels) && continue + + view_dfs = [_method_df(df, method, default_db_meta.levels) for df in src_run_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + sd = _method_source_dirname(src) + sl = sd == "unfiltered" ? "unfiltered" : sd + sections = copy(report_sections) + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(combined, src_all_scols, rank_name, rank_col; n=20) + !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) + end + + rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + _write_report(rpath, + "Study Report\n Source: $sl ($method)", + sections) + log_written(study_dir, rpath) + end + end + end + + # Public: analyse_study (entry point from main.jl) + """ + analyse_study(projects, merged_results) + + Run group-level and study-level analysis after the per-run `@threads` + loop has completed. + + Groups projects by `dirname(project.dir)` (= group directory). + Calls `_analyse_group` for each group with ≥2 runs, then + `_analyse_study_level` when there are ≥2 groups. + """ + function analyse_study(projects::Vector{ProjectCtx}, + merged_results::Vector{<:Union{MergedTables, Nothing}}, + db_metas::Vector{DatabaseMeta}; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + isempty(projects) && return + + r_lock = ReentrantLock() + study_dir = projects[1].study_dir + + # Pair projects with results and db_metas, filtering out failures. + valid = Tuple{ProjectCtx, MergedTables, DatabaseMeta}[ + (projects[i], merged_results[i], db_metas[i]) + for i in eachindex(projects) + if i <= length(merged_results) && !isnothing(merged_results[i]) + ] + isempty(valid) && return + + # Group by parent directory. + groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() + for (proj, merged, dm) in valid + gdir = dirname(proj.dir) + push!(get!(groups, gdir, []), (proj, merged, dm)) + end + + # Per-group analysis (skipped for single-run groups). + # Each group uses the db_meta of its first member (all runs in a group share a DB). + for (gdir, members) in groups + if length(members) > 1 + group_db_meta = members[1][3] + group_members = [(p, m) for (p, m, _) in members] + _analyse_group(gdir, group_members, r_lock, group_db_meta; plot_lock) + end + end + + # Study-level analysis (only meaningful with ≥2 genuine groups). + if length(groups) >= 2 + study_db_meta = valid[1][3] + _analyse_study_level(study_dir, valid, r_lock, study_db_meta; plot_lock) + end + end + +end diff --git a/src/call_tools.jl b/src/call_tools.jl index b8d3838..581843d 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -4,11 +4,12 @@ module Tools # # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). -export cutadapt, vsearch, multiqc, fastqc, cdhit +export cutadapt, vsearch, multiqc, cdhit using YAML using Logging using ..PipelineTypes + using ..PipelineLog using ..Config # Run cmd_str via bash, capturing stdout+stderr to log_path. @@ -49,12 +50,7 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit const _tools = load_tools() ## cutadapt - # Prevent duplication of primers. Must be instantiated outside get_primers() loop. Global scope may be an issue. - used_forward = String[] - used_reverse = String[] - - # Get primers based on input. Will not run without used_forward and used_reverse arrays being defined first. - function get_primers(input, primers_path) + function get_primers(input, primers_path, seen_fwd::Vector{String}, seen_rev::Vector{String}) data = YAML.load_file(primers_path) for pair in data["Pairs"] @@ -64,14 +60,14 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit forward = data["Forward"][pair_names[1]] reverse = data["Reverse"][pair_names[2]] - output_forward = in(forward, used_forward) ? "x" : forward - output_reverse = in(reverse, used_reverse) ? "x" : reverse + output_forward = in(forward, seen_fwd) ? "x" : forward + output_reverse = in(reverse, seen_rev) ? "x" : reverse if output_forward != "x" - push!(used_forward, output_forward) + push!(seen_fwd, output_forward) end if output_reverse != "x" - push!(used_reverse, output_reverse) + push!(seen_rev, output_reverse) end return (output_forward, output_reverse) @@ -82,22 +78,20 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit return "Invalid input. Please specify a valid primer pair in '$path'." end - # Formats cutadapt's arguments for the primer sets requested + # Formats cutadapt's arguments for the primer sets requested function get_primer_args(primer_pairs, primers_path) + seen_fwd = String[] + seen_rev = String[] args = "" for pair in primer_pairs - pair_tuple = get_primers(pair, primers_path) + pair_tuple = get_primers(pair, primers_path, seen_fwd, seen_rev) forward_primer = pair_tuple[1] reverse_primer = pair_tuple[2] args *= join([flag * primer * " " for (flag, primer) in (("-g ", forward_primer), ("-G ", reverse_primer)) if primer != "x"], "") end - # Reset global used primers. - used_forward = String[] - used_reverse = String[] - return chop(args) end @@ -213,6 +207,35 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit return TrimmedReads(cutadapt_dir) end + function cutadapt(project::ProjectCtx; + optional_args::Union{String,Nothing} = nothing, + cutadapt_bin = tool_bin("cutadapt")) + config_path = write_run_config(project) + primers_path = joinpath(project.config_dir, "primers.yml") + cfg = get(YAML.load_file(config_path), "cutadapt", Dict()) + primer_pairs = cfg["primer_pairs"] + optional_args = isnothing(optional_args) ? + get(cfg, "optional_args", "-m 200 --discard-untrimmed") : optional_args + cutadapt_dir = joinpath(project.dir, "cutadapt") + hash_file = joinpath(cutadapt_dir, "config.hash") + if isdir(cutadapt_dir) + trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) + if !isempty(trimmed) && + !_section_stale(config_path, "cutadapt", hash_file) && + all(f -> mtime(joinpath(cutadapt_dir, f)) > mtime(primers_path), trimmed) && + all(f -> filesize(joinpath(cutadapt_dir, f)) > 20, trimmed) # skip empty gzips + @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" + return TrimmedReads(cutadapt_dir) + end + end + mkpath(cutadapt_dir) + run_cutadapt(get_primer_args(primer_pairs, primers_path), optional_args, + project.data_dir, cutadapt_dir, cutadapt_bin) + _write_section_hash(config_path, "cutadapt", hash_file) + pipeline_log(project, "cutadapt complete") + return TrimmedReads(cutadapt_dir) + end + ## FastQC / MultiQC """ multiqc(fastq_in_dir, qc_dir; fastqc_args, multiqc_args, fastqc_bin, multiqc_bin) @@ -226,13 +249,13 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit the MultiQC summary report goes to `qc_dir/`. ## Keyword Arguments - - `fastqc_args` (optional, default: `"-t 20 --extract --delete --force"`): Additional arguments passed to `fastqc`. + - `fastqc_args` (optional, default: `"-t 20 --extract --delete"`): Additional arguments passed to `fastqc`. - `multiqc_args` (optional, default: `""`): Additional arguments passed to `multiqc`. - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. - `multiqc_bin` (optional): Path to the multiqc binary. Set via `config/tools.yml`. """ function multiqc(fastq_in_dir, qc_dir; - fastqc_args = "--force --verbose", + fastqc_args = "-t 20 --extract --delete", multiqc_args = "", fastqc_bin = tool_bin("fastqc"), multiqc_bin = tool_bin("multiqc")) @@ -264,34 +287,10 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit @info "MultiQC complete. Output: $qc_dir Log: $multiqc_log" end + ## vsearch """ - fastqc(fastq_in_file, fastqc_dir; optional_args, fastqc_bin) - - Run FastQC on a single FASTQ file. No MultiQC report is generated. - - ## Arguments - - `fastq_in_file`: Path to a single `.fastq` or `.fastq.gz` file. - - `fastqc_dir`: Output directory for the FastQC report. - - ## Keyword Arguments - - `optional_args` (optional, default: `"-t 20 --extract --delete"`): Additional arguments passed to `fastqc`. - - `fastqc_bin` (optional): Path to the fastqc binary. Set via `config/tools.yml`. - """ - function fastqc(fastq_in_file, fastqc_dir; optional_args = "-t 20 --extract --delete", fastqc_bin = tool_bin("fastqc")) - log_dir = joinpath(fastqc_dir, "logs") - mkpath(fastqc_dir) - mkpath(log_dir) - log_path = joinpath(log_dir, "fastqc.log") - @info "FastQC running on $fastq_in_file" - cmd = "$fastqc_bin $fastq_in_file -o $fastqc_dir $optional_args" - _run_logged(cmd, log_path) - @info "FastQC complete. Output: $fastqc_dir Log: $log_path" - end - - ## VSEARCH - """ vsearch(fasta_in_dir, reference_database; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") - + Requires `vsearch` installed. Runs `vsearch` command with any optional parameters to perform taxonomy assignment by local alignment against specified database. ## Arguments @@ -310,14 +309,53 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit outfile = joinpath(vsearch_dir, "taxonomy.tsv") log_path = joinpath(log_dir, "vsearch.log") @info "VSEARCH running: $fasta_in_dir against $(basename(reference_database))" - cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --blast6out $outfile $optional_args" + # --userout with query+target+id captures the full FASTA header (including + # description after the space), unlike --blast6out which truncates at the first space. + # This is required for databases like SILVA where taxonomy is in the description field. + cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --userout $outfile --userfields query+target+id $optional_args" _run_logged(cmd, log_path) @info "VSEARCH complete. Output: $outfile Log: $log_path" end + function vsearch(input::HasFasta, reference_database::String, vsearch_dir::String; + optional_args = "--id 0.75 --query_cov 0.8", + vsearch_bin = tool_bin("vsearch")) + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + if isfile(tsv) && mtime(tsv) > mtime(input.fasta) + @info "Skipping vsearch: $tsv up to date" + return TaxonomyHits(tsv) + end + vsearch(input.fasta, reference_database, vsearch_dir; + optional_args, vsearch_bin) + return TaxonomyHits(tsv) + end + + function vsearch(project::ProjectCtx, input::HasFasta, reference_database::String; + optional_args::Union{String,Nothing} = nothing, + vsearch_bin = tool_bin("vsearch")) + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "vsearch", Dict()) + optional_args = isnothing(optional_args) ? + get(cfg, "optional_args", "--id 0.75 --query_cov 0.8") : optional_args + vsearch_dir = joinpath(project.dir, "vsearch") + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + hash_file = joinpath(vsearch_dir, "config.hash") + if isfile(tsv) && + !_section_stale(config_path, "vsearch", hash_file) && + mtime(tsv) > mtime(input.fasta) + @info "Skipping vsearch: $tsv up to date" + return TaxonomyHits(tsv) + end + vsearch(input.fasta, reference_database, vsearch_dir; optional_args, vsearch_bin) + _write_section_hash(config_path, "vsearch", hash_file) + pipeline_log(project, "VSEARCH: $(input.fasta) against $(basename(reference_database))") + log_written(project, tsv) + return TaxonomyHits(tsv) + end + ## cd-hit-est """ - cdhit(fasta_in, cdhit_dir; optional_args, cdhit_bin) + cdhit(fasta_in, cdhit_dir; optional_args = "-c 0.9", cdhit_bin) Run cd-hit-est on a FASTA file, writing the clustered output to `cdhit_dir`. Returns the path to the output FASTA. @@ -343,19 +381,6 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit return fasta_out end - function vsearch(input::HasFasta, reference_database::String, vsearch_dir::String; - optional_args = "--id 0.75 --query_cov 0.8", - vsearch_bin = tool_bin("vsearch")) - tsv = joinpath(vsearch_dir, "taxonomy.tsv") - if isfile(tsv) && mtime(tsv) > mtime(input.fasta) - @info "Skipping vsearch: $tsv up to date" - return TaxonomyHits(tsv) - end - vsearch(input.fasta, reference_database, vsearch_dir; - optional_args, vsearch_bin) - return TaxonomyHits(tsv) - end - function cdhit(input::ASVResult, cdhit_dir::String; optional_args = "-c 0.9", cdhit_bin = tool_bin("cd_hit_est")) @@ -368,57 +393,13 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit return ASVResult(new_fasta, input.count_table, input.taxonomy) end - # ProjectCtx overloads - read parameters from per-project configs and fix output dirs. - - function cutadapt(project::ProjectCtx; - cutadapt_bin = tool_bin("cutadapt")) - config_path = write_run_config(project) - primers_path = joinpath(project.config_dir, "primers.yml") - cfg = get(YAML.load_file(config_path), "cutadapt", Dict()) - primer_pairs = cfg["primer_pairs"] - optional_args = get(cfg, "optional_args", "-m 200 --discard-untrimmed") - cutadapt_dir = joinpath(project.dir, "cutadapt") - hash_file = joinpath(cutadapt_dir, "config.hash") - if isdir(cutadapt_dir) - trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) - if !isempty(trimmed) && - !_section_stale(config_path, "cutadapt", hash_file) && - all(f -> mtime(joinpath(cutadapt_dir, f)) > mtime(primers_path), trimmed) - @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" - return TrimmedReads(cutadapt_dir) - end - end - mkpath(cutadapt_dir) - run_cutadapt(get_primer_args(primer_pairs, primers_path), optional_args, - project.data_dir, cutadapt_dir, cutadapt_bin) - _write_section_hash(config_path, "cutadapt", hash_file) - return TrimmedReads(cutadapt_dir) - end - - function vsearch(project::ProjectCtx, input::HasFasta, reference_database::String; - vsearch_bin = tool_bin("vsearch")) - config_path = write_run_config(project) - cfg = get(YAML.load_file(config_path), "vsearch", Dict()) - optional_args = get(cfg, "optional_args", "--id 0.75 --query_cov 0.8") - vsearch_dir = joinpath(project.dir, "vsearch") - tsv = joinpath(vsearch_dir, "taxonomy.tsv") - hash_file = joinpath(vsearch_dir, "config.hash") - if isfile(tsv) && - !_section_stale(config_path, "vsearch", hash_file) && - mtime(tsv) > mtime(input.fasta) - @info "Skipping vsearch: $tsv up to date" - return TaxonomyHits(tsv) - end - vsearch(input.fasta, reference_database, vsearch_dir; optional_args, vsearch_bin) - _write_section_hash(config_path, "vsearch", hash_file) - return TaxonomyHits(tsv) - end - function cdhit(project::ProjectCtx, input::ASVResult; + optional_args::Union{String,Nothing} = nothing, cdhit_bin = tool_bin("cd_hit_est")) config_path = write_run_config(project) cfg = get(YAML.load_file(config_path), "cdhit", Dict()) - optional_args = get(cfg, "optional_args", "-c 0.9") + optional_args = isnothing(optional_args) ? + get(cfg, "optional_args", "-c 0.9") : optional_args cdhit_dir = joinpath(project.dir, "cdhit") new_fasta = joinpath(cdhit_dir, basename(input.fasta)) hash_file = joinpath(cdhit_dir, "config.hash") @@ -430,6 +411,8 @@ export cutadapt, vsearch, multiqc, fastqc, cdhit end new_fasta = cdhit(input.fasta, cdhit_dir; optional_args, cdhit_bin) _write_section_hash(config_path, "cdhit", hash_file) + pipeline_log(project, "cd-hit-est complete") + log_written(project, new_fasta) return ASVResult(new_fasta, input.count_table, input.taxonomy) end end diff --git a/src/config.jl b/src/config.jl index d322d40..f4a89bb 100644 --- a/src/config.jl +++ b/src/config.jl @@ -15,7 +15,7 @@ module Config # projects/{study}/{run}/run_config.yml # # Stage skip guards hash the relevant YAML section from run_config.yml so -# that any change at any level -- global, study, or run -- correctly +# that any change at any level - global, study, or run - correctly # invalidates downstream checkpoints. # # © 2026 Joshua Benjamin Jewell. All rights reserved. diff --git a/src/dada2.jl b/src/dada2.jl index 6fb92f1..0f4dae8 100644 --- a/src/dada2.jl +++ b/src/dada2.jl @@ -32,6 +32,7 @@ export dada2, dada2_denoise, dada2_classify, import Downloads using Logging, RCall, YAML using ..PipelineTypes + using ..PipelineLog using ..Databases using ..Config @@ -113,6 +114,7 @@ export dada2, dada2_denoise, dada2_classify, denoise(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" filter_length(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" chimera_removal(config_path; progress, input_dir=trimmed.dir, workspace_root); R"gc()" + pipeline_log(project, "DADA2 denoising complete") end return DenoisedASVs(chimera_ckpt, config_path, workspace_root, trimmed.dir) @@ -141,6 +143,10 @@ export dada2, dada2_denoise, dada2_classify, input_dir = denoised.input_dir, workspace_root, taxonomy_db) + pipeline_log(project, "DADA2 taxonomy complete") + for f in (result.fasta, result.count_table, result.taxonomy) + isfile(f) && log_written(project, f) + end return result end @@ -165,8 +171,11 @@ export dada2, dada2_denoise, dada2_classify, @info "Skipping dada2: outputs up to date in $tables_dir" return result end + pipeline_log(project, "DADA2 pipeline started") denoised = dada2_denoise(project, trimmed; progress) - return dada2_classify(project, denoised; taxonomy_db, progress) + result = dada2_classify(project, denoised; taxonomy_db, progress) + pipeline_log(project, "DADA2 complete") + return result end end diff --git a/src/dada2/chimera.jl b/src/dada2/chimera.jl index d4f6c8b..b021b14 100644 --- a/src/dada2/chimera.jl +++ b/src/dada2/chimera.jl @@ -39,7 +39,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") tables_dir = ctx.dirs["Tables"] diff --git a/src/dada2/context.jl b/src/dada2/context.jl index 3e7c541..9f8a2a6 100644 --- a/src/dada2/context.jl +++ b/src/dada2/context.jl @@ -90,7 +90,15 @@ # R function loader # Source the R helper functions into the current R session. Called by each # stage after its mtime skip check, so R is not loaded for skipped stages. - function _source_r_functions() + # When `ctx` is provided, emits the single-sample warning (once per run). + const _single_sample_warned = Ref(false) + function _source_r_functions(ctx=nothing) + if !isnothing(ctx) && ctx.single_sample && !_single_sample_warned[] + _single_sample_warned[] = true + @warn "Only 1 sample found. Duplicating it to work around dada() " * + "returning a bare object for single-file input. The duplicate " * + "will be dropped from all outputs." + end functions_r = joinpath(@__DIR__, "dada2_functions.r") R"source($functions_r)" end @@ -133,9 +141,6 @@ # dropped in chimera_removal(). ASV calls are unaffected. single_sample = length(sample_names) == 1 if single_sample - @warn "Only 1 sample found. Duplicating it to work around dada() " * - "returning a bare object for single-file input. The duplicate " * - "will be dropped from all outputs." fwd_files = isempty(fwd_files) ? fwd_files : repeat(fwd_files, 2) rev_files = isempty(rev_files) ? rev_files : repeat(rev_files, 2) sample_names = [sample_names[1], sample_names[1] * "_dup"] @@ -164,3 +169,4 @@ fwd_files, rev_files, fwd_out, rev_out, in_fwd, out_fwd, in_rev_arg, out_rev_arg, ckpts) end + diff --git a/src/dada2/denoise.jl b/src/dada2/denoise.jl index 2425b83..eb7e6c7 100644 --- a/src/dada2/denoise.jl +++ b/src/dada2/denoise.jl @@ -28,7 +28,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) seed = get(ctx.cfg["dada"], "seed", 123) nbases = ctx.cfg["dada"]["nbases"] max_con = ctx.cfg["dada"]["max_consist"] @@ -101,7 +101,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) log_path = joinpath(ctx.dirs["Logs"], "denoise.log") open(log_path, "w") do io; println(io, "=== denoise ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" @@ -195,7 +195,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) log_path = joinpath(ctx.dirs["Logs"], "filter_length.log") open(log_path, "w") do io; println(io, "=== filter_length ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" diff --git a/src/dada2/qc.jl b/src/dada2/qc.jl index 747737a..df7d058 100644 --- a/src/dada2/qc.jl +++ b/src/dada2/qc.jl @@ -24,7 +24,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) log_path = joinpath(ctx.dirs["Logs"], "prefilter_qc.log") open(log_path, "w") do io; println(io, "=== prefilter_qc ===\nconfig: $config_path") end R"con <- file($log_path, open='at'); sink(con); sink(con, type='message')" @@ -70,7 +70,7 @@ end R"rm(list=ls())" - _source_r_functions() + _source_r_functions(ctx) ft = ctx.cfg["filter_trim"] trunc_len = ft["trunc_len"] max_ee = ft["max_ee"] diff --git a/src/dada2/taxonomy.jl b/src/dada2/taxonomy.jl index 09c9c3e..359f255 100644 --- a/src/dada2/taxonomy.jl +++ b/src/dada2/taxonomy.jl @@ -17,21 +17,20 @@ # and delete from, and that the remote server has sufficient resources. The authors # of this software accept no liability for unintended data loss, unauthorised access, # or any other consequences arising from misconfiguration or misuse of this feature. - function _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, + function _assign_taxonomy_remote(emit, chimera_ckpt, db_path, db_remote_path, tables_dir, checkpoint, taxa_prefix, multithread, min_boot, tax_levels, verbose, remote_cfg, log_path) host = remote_cfg["host"] rscript = get(remote_cfg, "rscript", "Rscript") base_dir = get(remote_cfg, "staging_dir", nothing) - remote_db = get(remote_cfg, "db_path", nothing) identity_file = get(remote_cfg, "identity_file", nothing) isnothing(base_dir) && error("taxonomy.remote.staging_dir must be set explicitly in config") - !isnothing(remote_db) && !startswith(string(remote_db), "/") && - error("taxonomy.remote.db_path must be an absolute path on the server " * - "(got: '$remote_db'). Do not include the hostname.") + !isnothing(db_remote_path) && !startswith(string(db_remote_path), "/") && + error("databases.yml dada2.remote_path must be an absolute path on the server " * + "(got: '$db_remote_path'). Do not include the hostname.") # Append a unique run ID so cleanup only ever touches this specific run's dir. run_id = string(floor(Int, time())) @@ -67,14 +66,18 @@ run(scp(functions_r, "$host:$staging_dir/dada2_functions.r")) run(scp(remote_r, "$host:$staging_dir/taxonomy_remote.r")) - remote_db_path = if !isnothing(remote_db) - emit(" Using remote database: $remote_db") - string(remote_db) - else + # If db_remote_path is set (from databases.yml dada2.remote_path), use it directly. + # Otherwise transfer the local db_path to the remote staging directory. + remote_db_resolved = if !isnothing(db_remote_path) + emit(" Using remote database: $db_remote_path") + db_remote_path + elseif !isnothing(db_path) db_basename = basename(db_path) emit(" Transferring database ($db_basename) to $host") run(scp(db_path, "$host:$staging_dir/$db_basename")) "$staging_dir/$db_basename" + else + error("No taxonomy database: set databases.yml dada2.remote_path or provide a local database") end levels_str = join(tax_levels, ",") @@ -84,7 +87,7 @@ remote_cmd = "$rscript $staging_dir/taxonomy_remote.r " * "functions=$staging_dir/dada2_functions.r " * "ckpt=$staging_dir/ckpt_chimera.RData " * - "db=$remote_db_path " * + "db=$remote_db_resolved " * "tables=$remote_tables " * "save=$remote_ckpt " * "prefix=$taxa_prefix " * @@ -146,7 +149,7 @@ rm(.data_objs) gc() """ - _source_r_functions() + _source_r_functions(ctx) seq_prefix = get(ctx.cfg["output"], "seq_table_prefix", "seqtab_nochim") fasta_prefix = get(ctx.cfg["output"], "fasta_prefix", "asvs") @@ -156,16 +159,26 @@ tables_dir = ctx.dirs["Tables"] multithread = get(ctx.cfg["taxonomy"], "multithread", 4) min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) - tax_levels = ctx.cfg["taxonomy"]["levels"] + # Read levels from databases.yml (single source of truth). + db_key = string(ctx.cfg["taxonomy"]["database"]) + dbs_path = joinpath(@__DIR__, "..", "..", "config", "databases.yml") + dbs_cfg = get(YAML.load_file(dbs_path), "databases", Dict()) + tax_levels = String[string(l) for l in get(get(dbs_cfg, db_key, Dict()), "levels", String[])] + isempty(tax_levels) && error("No levels defined for database '$db_key' in $dbs_path") remote_cfg = get(get(ctx.cfg, "taxonomy", Dict()), "remote", nothing) use_remote = !isnothing(remote_cfg) && !isnothing(get(remote_cfg, "host", nothing)) - # Skip local DB resolution when remote has its own db_path configured. - skip_local_db = use_remote && !isnothing(get(remote_cfg, "db_path", nothing)) - db_path = skip_local_db ? nothing : - isnothing(taxonomy_db) ? _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db + # Resolve the database path. Priority: + # 1. databases.yml dada2.remote_path - pre-existing file on remote (no transfer needed) + # 2. taxonomy_db argument - local file to transfer + # 3. _resolve_taxonomy_db - fallback local resolution + db_remote_path = get(get(get(dbs_cfg, db_key, Dict()), "dada2", Dict()), "remote_path", nothing) + if !isnothing(db_remote_path) + db_remote_path = string(db_remote_path) + end + db_path = isnothing(taxonomy_db) ? _resolve_taxonomy_db(ctx.cfg, emit) : taxonomy_db R"load($chimera_ckpt)" has_data = rcopy(R"isTRUE(sum(seq_table_nochim, na.rm=TRUE) > 0)") @@ -183,7 +196,7 @@ end elseif use_remote emit("Assigning taxonomy (remote: $(remote_cfg["host"]))") - _assign_taxonomy_remote(emit, chimera_ckpt, db_path, tables_dir, + _assign_taxonomy_remote(emit, chimera_ckpt, db_path, db_remote_path, tables_dir, checkpoint, taxa_prefix, multithread, min_boot, tax_levels, verbose, remote_cfg, log_path) diff --git a/src/databases.jl b/src/databases.jl index 60a7f59..6351451 100644 --- a/src/databases.jl +++ b/src/databases.jl @@ -17,9 +17,10 @@ module Databases # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). import Downloads -using YAML, Logging +using YAML, Logging, CodecZlib +using ..PipelineTypes -export ensure_databases, resolve_db +export ensure_databases, resolve_db, make_db_meta """ ensure_databases(config_path) -> Dict{String,String} @@ -57,6 +58,12 @@ export ensure_databases, resolve_db !(db_info isa AbstractDict) && continue for (fmt, fmt_info) in db_info !(fmt_info isa AbstractDict) && continue + # Skip entries where both uri and local are null/missing. + uri_val = get(fmt_info, "uri", nothing) + local_val = get(fmt_info, "local", nothing) + if isnothing(uri_val) && isnothing(local_val) + continue + end key = "$(db_name)_$(fmt)" resolved[key] = _resolve_entry(key, fmt_info, db_dir) end @@ -91,6 +98,47 @@ export ensure_databases, resolve_db _resolve_entry("$(db_name)_$(fmt)", fmt_cfg, db_dir; emit) end + # Fixed column names that are never sample counts, regardless of database. + const _FIXED_NONCOUNTS = Set([ + "SeqName", "Pident", "Accession", "rRNA", "Organellum", "specimen", + "Sequence", "sequence", "", "Column1", + ]) + + """ + make_db_meta(config_path, db_name) -> DatabaseMeta + + Construct a `DatabaseMeta` from the database entry in `config_path`. + Pre-computes `noncounts` as the union of taxonomy levels (including + `_dada2` suffixed variants) and fixed metadata column names. + """ + function make_db_meta(config_path::String, db_name::String) + cfg = YAML.load_file(config_path) + db_cfg = get(cfg, "databases", Dict()) + haskey(db_cfg, db_name) || + error("Database '$db_name' not found in $config_path") + entry = db_cfg[db_name] + + levels = String[string(l) for l in get(entry, "levels", String[])] + vsformat = string(get(entry, "vsearch_format", "generic")) + raw_corr = get(entry, "corrections", []) + corrections = Dict{String,Any}[] + if raw_corr isa Vector + for c in raw_corr + c isa AbstractDict && push!(corrections, Dict{String,Any}(string(k) => v for (k,v) in c)) + end + end + + noncounts = copy(_FIXED_NONCOUNTS) + for l in levels + push!(noncounts, l) + push!(noncounts, l * "_dada2") + push!(noncounts, l * "_vsearch") + push!(noncounts, l * "_boot") + end + + return DatabaseMeta(db_name, levels, vsformat, corrections, noncounts) + end + function _resolve_entry(key, fmt_info, db_dir; emit=nothing) log = isnothing(emit) ? msg -> @info(msg) : emit local_p = get(fmt_info, "local", nothing) @@ -117,7 +165,102 @@ export ensure_databases, resolve_db Downloads.download(uri, cached) log("[$key] Saved to: $cached") end + + # Post-processing: reformat database if requested. + reformat = get(fmt_info, "reformat", nothing) + if !isnothing(reformat) && !isempty(string(reformat)) + cached = _apply_reformat(key, string(reformat), cached, db_dir; log) + end + return cached end + """ + Apply a named reformat step to a downloaded database file. + Returns the path to the (possibly new) reformatted file. + """ + function _apply_reformat(key, reformat, src_path, db_dir; log=msg->@info(msg)) + if reformat == "silva_vsearch" + return _reformat_silva_vsearch(key, src_path, db_dir; log) + else + @warn "[$key] Unknown reformat '$reformat' - skipping" + return src_path + end + end + + """ + Reformat a SILVA vsearch FASTA so taxonomy is embedded in the sequence ID. + + SILVA headers are `>Accession Kingdom;Phylum;...;Genus` (taxonomy after a space). + vsearch only captures the sequence ID (before the first space), so taxonomy is lost. + + This function rewrites headers to `>Accession;Kingdom;Phylum;...;Genus` with spaces + within taxon names replaced by underscores, so vsearch returns the full taxonomy + string in the `target` field. + + The reformatted file is cached alongside the original; the original is kept intact. + """ + function _reformat_silva_vsearch(key, src_path, db_dir; log=msg->@info(msg)) + # Derive output filename from source + src_base = basename(src_path) + # Strip .gz if present to insert _reformatted before the extension + if endswith(src_base, ".fasta.gz") + out_base = src_base[1:end-9] * "_reformatted.fasta.gz" + elseif endswith(src_base, ".fa.gz") + out_base = src_base[1:end-6] * "_reformatted.fa.gz" + elseif endswith(src_base, ".fasta") + out_base = src_base[1:end-6] * "_reformatted.fasta" + else + out_base = src_base * "_reformatted" + end + out_path = joinpath(db_dir, out_base) + + if isfile(out_path) + log("[$key] Using cached reformatted SILVA: $out_path") + return out_path + end + + log("[$key] Reformatting SILVA FASTA for vsearch compatibility: $src_path -> $out_path") + + # Determine if gzipped + is_gz = endswith(src_path, ".gz") + is_out_gz = endswith(out_path, ".gz") + + open_in = is_gz ? () -> GzipDecompressorStream(open(src_path, "r")) : () -> open(src_path, "r") + open_out = is_out_gz ? () -> GzipCompressorStream(open(out_path, "w")) : () -> open(out_path, "w") + + in_io = open_in() + out_io = open_out() + n_seq = 0 + try + for line in eachline(in_io) + if startswith(line, '>') + # Header: ">Accession.start.end Kingdom;Phylum;...;Genus" + rest = line[2:end] + sp = findfirst(' ', rest) + if isnothing(sp) + # No space - already no taxonomy; pass through + write(out_io, line, '\n') + else + acc = rest[1:sp-1] + tax = rest[sp+1:end] + # Replace spaces within taxonomy with underscores, + # then join accession + taxonomy with semicolon. + tax_clean = replace(tax, ' ' => '_') + write(out_io, '>', acc, ';', tax_clean, '\n') + end + n_seq += 1 + else + write(out_io, line, '\n') + end + end + finally + close(out_io) + close(in_io) + end + + log("[$key] Reformatted $n_seq sequences -> $out_path") + return out_path + end + end \ No newline at end of file diff --git a/src/diversity.jl b/src/diversity.jl new file mode 100644 index 0000000..c9f6495 --- /dev/null +++ b/src/diversity.jl @@ -0,0 +1,49 @@ +module DiversityMetrics + +# Pure-Julia alpha and beta diversity metrics. +# +# No external dependencies beyond the Julia stdlib. NMDS ordination +# is delegated to R/vegan in the Analysis module - this module only +# provides the distance matrix that feeds it. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export richness, shannon, simpson + + """ + richness(counts) -> Int + + Observed richness: the number of non-zero features in `counts`. + """ + richness(counts) = count(!iszero, counts) + + """ + shannon(counts) -> Float64 + + Shannon diversity index H = -sum(p_i * ln(p_i)), where p_i is the + relative abundance of feature i. Zero-count features are ignored. + Returns 0.0 when `counts` sums to zero. + """ + function shannon(counts) + n = sum(counts) + n == 0 && return 0.0 + p = counts[counts .> 0] ./ n + return -sum(p .* log.(p)) + end + + """ + simpson(counts) -> Float64 + + Simpson diversity index 1 - sum(p_i^2), where p_i is the relative + abundance of feature i. Returns 0.0 when `counts` sums to zero. + """ + function simpson(counts) + n = sum(counts) + n == 0 && return 0.0 + p = counts[counts .> 0] ./ n + return 1.0 - sum(p .^ 2) + end + +end diff --git a/src/log.jl b/src/log.jl new file mode 100644 index 0000000..17bcb39 --- /dev/null +++ b/src/log.jl @@ -0,0 +1,176 @@ +module PipelineLog + +export pipeline_log, log_written, reset_log, finalise_log, write_combined_log + + using SHA, Dates + using ..PipelineTypes + + function reset_log(dir::String) + path = joinpath(dir, "pipeline.log") + open(path, "w") do io + println(io, "Pipeline log started ", Dates.format(now(), "yyyy-mm-dd HH:MM:SS")) + end + end + reset_log(project::ProjectCtx) = reset_log(project.dir) + + function pipeline_log(dir::String, msg::String) + open(joinpath(dir, "pipeline.log"), "a") do io + println(io, Dates.format(now(), "yyyy-mm-dd HH:MM:SS"), " ", msg) + end + end + pipeline_log(project::ProjectCtx, msg::String) = pipeline_log(project.dir, msg) + + function log_written(dir::String, path::String) + h = open(io -> bytes2hex(sha256(io)), path, "r") + rel = "./" * relpath(path, dir) + pipeline_log(dir, "Written: $rel sha256:$h") + end + log_written(project::ProjectCtx, path::String) = log_written(project.dir, path) + + # Tool-level log/stats/hash files to embed, relative to the project dir. + # Each entry is (stage_label, relative_path). Only existing files are included. + const _TOOL_LOG_FILES = [ + ("QC", "QC/logs/fastqc.log"), + ("QC", "QC/logs/multiqc.log"), + ("cutadapt", "cutadapt/logs/cutadapt_primer_trimming_stats.txt"), + ("cutadapt", "cutadapt/logs/cutadapt_trimmed_percentage.txt"), + ("cutadapt", "cutadapt/config.hash"), + ("DADA2", "dada2/Logs/prefilter_qc.log"), + ("DADA2", "dada2/Logs/filter_trim.log"), + ("DADA2", "dada2/Logs/learn_errors.log"), + ("DADA2", "dada2/Logs/denoise.log"), + ("DADA2", "dada2/Logs/filter_length.log"), + ("DADA2", "dada2/Logs/chimera_removal.log"), + ("DADA2", "dada2/Logs/assign_taxonomy.log"), + ("DADA2", "dada2/Checkpoints/filter_trim.hash"), + ("DADA2", "dada2/Checkpoints/learn_errors.hash"), + ("DADA2", "dada2/Checkpoints/denoise.hash"), + ("DADA2", "dada2/Checkpoints/filter_length.hash"), + ("DADA2", "dada2/Checkpoints/chimera_removal.hash"), + ("DADA2", "dada2/Checkpoints/assign_taxonomy.hash"), + ("VSEARCH", "vsearch/logs/vsearch.log"), + ("VSEARCH", "vsearch/config.hash"), + ("cd-hit", "cdhit/logs/cdhit.log"), + ("cd-hit", "cdhit/config.hash"), + ("merge", "merged/config.hash"), + ] + + """ + finalise_log(project::ProjectCtx) + + Append the contents of all tool-level log, stats, and hash files into + the run's `pipeline.log`. Call once per run after all stages complete. + """ + function finalise_log(project::ProjectCtx) + pipeline_log(project, "--- Tool logs appended below ---") + prev_stage = "" + for (stage, relpath_) in _TOOL_LOG_FILES + abspath_ = joinpath(project.dir, relpath_) + isfile(abspath_) || continue + if stage != prev_stage + pipeline_log(project, "") # blank separator + prev_stage = stage + end + _append_tool_file(project.dir, relpath_, abspath_) + end + pipeline_log(project, "--- End of tool logs ---") + end + + # Append a single tool file into the pipeline.log with a header/footer. + function _append_tool_file(dir::String, relpath_::String, abspath_::String) + contents = read(abspath_, String) + open(joinpath(dir, "pipeline.log"), "a") do io + println(io, ">>>>>> ", relpath_) + print(io, contents) + if !isempty(contents) && !endswith(contents, '\n') + println(io) + end + println(io, "<<<<<< ", relpath_) + end + end + + """ + write_combined_log(projects; study_dir) + + Finalise each run's log (appending tool-level files), then write a + combined log to `{study_dir}/combined_pipeline.log` that merges all + per-run `pipeline.log` files, each run's `run_config.yml`, plus any + group- and study-level logs. + """ + function write_combined_log(projects::Vector{ProjectCtx}; + study_dir::String = projects[1].study_dir) + # Finalise each run's pipeline.log with tool-level contents. + for project in projects + finalise_log(project) + end + + out_path = joinpath(study_dir, "combined_pipeline.log") + sep = "=" ^ 72 + + open(out_path, "w") do io + println(io, sep) + println(io, " Combined pipeline log") + println(io, " Generated: ", Dates.format(now(), "yyyy-mm-dd HH:MM:SS")) + println(io, " Study: ", study_dir) + println(io, " Runs: ", length(projects)) + println(io, sep) + + # Per-run sections + for project in projects + run_name = relpath(project.dir, study_dir) + println(io, "\n", sep) + println(io, " RUN: ", run_name) + println(io, sep) + + # run_config.yml + config_path = joinpath(project.dir, "run_config.yml") + if isfile(config_path) + println(io, "\n--- run_config.yml ---") + print(io, read(config_path, String)) + println(io) + end + + # pipeline.log (now includes tool logs) + log_path = joinpath(project.dir, "pipeline.log") + if isfile(log_path) + println(io, "--- pipeline.log ---") + print(io, read(log_path, String)) + end + end + + # Group-level logs (directories between runs and study root) + group_dirs = Set{String}() + for project in projects + gdir = dirname(project.dir) + while gdir != study_dir && startswith(gdir, study_dir) + push!(group_dirs, gdir) + gdir = dirname(gdir) + end + end + + for gdir in sort(collect(group_dirs)) + log_path = joinpath(gdir, "pipeline.log") + isfile(log_path) || continue + gname = relpath(gdir, study_dir) + println(io, "\n", sep) + println(io, " GROUP: ", gname) + println(io, sep) + println(io, "\n--- pipeline.log ---") + print(io, read(log_path, String)) + end + + # Study-level log + study_log = joinpath(study_dir, "pipeline.log") + if isfile(study_log) + println(io, "\n", sep) + println(io, " STUDY") + println(io, sep) + println(io, "\n--- pipeline.log ---") + print(io, read(study_log, String)) + end + end + + @info "Written: $out_path" + return out_path + end +end diff --git a/src/main.jl b/src/main.jl index 7a34d17..4e0c34e 100755 --- a/src/main.jl +++ b/src/main.jl @@ -1,16 +1,25 @@ #!/usr/bin/env julia +# Activate the project environment (Project.toml one directory up from src/). +import Pkg +Pkg.activate(joinpath(@__DIR__, ".."); io=devnull) + include("types.jl") +include("log.jl") include("config.jl") include("databases.jl") include("call_tools.jl") include("dada2.jl") include("merge_and_filter_taxa.jl") include("project.jl") +include("diversity.jl") +include("plots.jl") +include("analysis.jl") using CSV using YAML -using .PipelineTypes, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup +using .PipelineTypes, .PipelineLog, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup +using .DiversityMetrics, .PipelinePlots, .Analysis ## Instantiate parameters config_dir = "./config" @@ -21,17 +30,54 @@ dbs = ensure_databases(databases_config) ## Main const r_lock = ReentrantLock() -projects = new_project("Multi_v_Vespa") +projects = new_project("PR2_v_SILVA") + +merged_results = Vector{Union{MergedTables, Nothing}}(undef, length(projects)) +asvs_results = Vector{Union{ASVResult, Nothing}}(undef, length(projects)) +db_metas = Vector{DatabaseMeta}(undef, length(projects)) -Threads.@threads for project in projects +Threads.@threads for (i, project) in collect(enumerate(projects)) + reset_log(project) multiqc(project.data_dir, joinpath(project.dir, "QC")) + # Resolve per-project database from its config cascade. + run_cfg_path = write_run_config(project) + db_name = string(get(get(get(YAML.load_file(run_cfg_path), "dada2", Dict()), "taxonomy", Dict()), "database", "pr2")) + db_metas[i] = make_db_meta(databases_config, db_name) + trimmed = cutadapt(project) asvs = lock(r_lock) do - dada2(project, trimmed, taxonomy_db = dbs["pr2_dada2"]) + dada2(project, trimmed, taxonomy_db = dbs["$(db_name)_dada2"]) end - #asvs = lock(r_lock) do; cdhit(project, asvs); end # optional, uncomment to enable - tax = vsearch(project, asvs, dbs["pr2_vsearch"]) - merged = merge_taxa(project, asvs, tax) + asvs = lock(r_lock) do; cdhit(project, asvs; optional_args = "-c 1"); end + + if haskey(dbs, "$(db_name)_vsearch") + tax = vsearch(project, asvs, dbs["$(db_name)_vsearch"]) + merged = merge_taxa(project, asvs, tax, db_metas[i]) + else + @warn "No vsearch database for '$db_name' - using DADA2 tax_counts as merged for $(basename(project.dir))" + tax_counts = joinpath(project.dir, "dada2", "Tables", "tax_counts.csv") + merged = isfile(tax_counts) && filesize(tax_counts) > 0 ? + MergedTables(Dict("merged" => tax_counts), String[], Dict{String,String}()) : + nothing + end + merged_results[i] = merged + asvs_results[i] = asvs end + +# Analysis: data prep runs in parallel, CairoMakie calls serialized via plot_lock. +const plot_lock = ReentrantLock() + +Threads.@threads for (i, project) in collect(enumerate(projects)) + merged = merged_results[i] + asvs = asvs_results[i] + (isnothing(merged) || isnothing(asvs)) && continue + analyse_run(project, merged, asvs, db_metas[i]; plot_lock) +end + +if any(!isnothing, merged_results) + analyse_study(projects, merged_results, db_metas; plot_lock) +end + +write_combined_log(projects) diff --git a/src/merge_and_filter_taxa.jl b/src/merge_and_filter_taxa.jl index 4bcfd2b..8890f47 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_and_filter_taxa.jl @@ -6,9 +6,10 @@ module TaxonomyTableTools using CSV, DataFrames, Logging, YAML using ..PipelineTypes +using ..PipelineLog using ..Config -export merge_taxonomy_counts, filter_table, merge_taxa +export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa # Import vsearch taxonomy function import_vsearch(file::AbstractString) @@ -36,10 +37,22 @@ export merge_taxonomy_counts, filter_table, merge_taxa return DataFrame(SeqName=qseqid, Pident=pident, sseqtax=sseqtax) end - function build_taxonomy_table_rows(df::DataFrame) - header = ["SeqName","Pident","Accession","rRNA","Organellum","specimen", - "Domain","Supergroup","Division","Subdivision","Class","Order", - "Family","Genus","Species"] + function build_taxonomy_table_rows(df::DataFrame, db_meta::DatabaseMeta) + levels = db_meta.levels + if db_meta.vsearch_format == "pr2" + # PR2: pipe-separated with metadata prefix + meta_cols = ["SeqName", "Pident", "Accession", "rRNA", "Organellum", "specimen"] + header = vcat(meta_cols, levels) + n_meta = length(meta_cols) - 2 # fields parsed from sseqtax (excl SeqName, Pident) + elseif db_meta.vsearch_format == "silva" + header = vcat(["SeqName", "Pident"], levels) + n_meta = 0 + else + # Generic: semicolon-separated, mapped positionally to levels + header = vcat(["SeqName", "Pident"], levels) + n_meta = 0 + end + out_rows = Vector{Vector{String}}(undef, nrow(df)) for (i, r) in enumerate(eachrow(df)) @@ -48,18 +61,36 @@ export merge_taxonomy_counts, filter_table, merge_taxa data[2] = string(r.Pident) tax_str = r.sseqtax - if occursin('|', tax_str) + if db_meta.vsearch_format == "pr2" && occursin('|', tax_str) # Full PR2 format: Accession|rRNA|Organellum|specimen|Domain|...|Species parts = split(tax_str, '|') for j in 1:min(length(parts), length(header) - 2) data[j + 2] = parts[j] end + elseif db_meta.vsearch_format == "pr2" + # PR2 taxonomy-only: Domain;Supergroup;...;Species[;] + parts = filter(!isempty, split(tax_str, ';')) + tax_start = length(header) - length(levels) + 1 + for j in 1:min(length(parts), length(levels)) + data[tax_start + j - 1] = parts[j] + end + elseif db_meta.vsearch_format == "silva" + # SILVA (reformatted): vsearch target field is "Accession;Kingdom;Phylum;...;Genus" + # (spaces within taxon names replaced by underscores during DB reformatting). + # Split on ';' and skip the first field (accession). + all_parts = split(tax_str, ';') + # Drop leading accession field + parts = filter(!isempty, length(all_parts) > 1 ? all_parts[2:end] : all_parts) + tax_start = 3 # after SeqName, Pident + for j in 1:min(length(parts), length(levels)) + data[tax_start + j - 1] = replace(String(strip(parts[j])), '_' => ' ') + end else - # Taxonomy-only format: Domain;Supergroup;...;Species[;] - # No accession/rRNA/Organellum/specimen - those stay as empty strings + # Generic: semicolon-separated, mapped positionally to levels parts = filter(!isempty, split(tax_str, ';')) - for j in 1:min(length(parts), length(header) - 6) - data[j + 6] = parts[j] + tax_start = 3 + for j in 1:min(length(parts), length(levels)) + data[tax_start + j - 1] = String(strip(parts[j])) end end @@ -78,24 +109,27 @@ export merge_taxonomy_counts, filter_table, merge_taxa end """ - merge_taxonomy_counts(taxonomy_vsearch_path, counts_csv_path) + merge_taxonomy_counts(taxonomy_vsearch_path, counts_csv_path, db_meta) Reads taxonomy TSV from vsearch and counts CSV from DADA2 and returns a merged DataFrame. ## Arguments: - - `taxonomy_vsearch_path` (): Path to vsearch output file. + - `taxonomy_vsearch_path`: Path to vsearch output file. - `counts_csv_path`: Path to DADA2 idtax output file. + - `db_meta`: DatabaseMeta with levels, corrections, and vsearch_format. """ function merge_taxonomy_counts( taxonomy_vsearch_path, - counts_csv_path) + counts_csv_path, + db_meta::DatabaseMeta; + bootstraps_path=nothing) @info("Merging taxonomy counts.") # build taxonomy dataframe - FIXED VERSION imported = import_vsearch(taxonomy_vsearch_path) df_tax_raw = vsearch_to_df(imported) - header, rows = build_taxonomy_table_rows(df_tax_raw) + header, rows = build_taxonomy_table_rows(df_tax_raw, db_meta) # Create DataFrame properly by specifying all columns first df_taxonomy = DataFrame([Symbol(h) => String[] for h in header]) @@ -167,106 +201,181 @@ export merge_taxonomy_counts, filter_table, merge_taxa [String(c) => String(c) * "_dada2" for c in overlap]) end - # Left join instead of outerjoin to keep all taxonomy rows - merged_df = leftjoin(df_taxonomy, df_counts_prepared, on="SeqName") + # Full outer join to retain ASVs found by either method + merged_df = outerjoin(df_taxonomy, df_counts_prepared, on="SeqName") sort!(merged_df, "SeqName", by=seqnum) + + # Join bootstrap confidence values if a bootstraps file was provided. + if !isnothing(bootstraps_path) && isfile(bootstraps_path) && filesize(bootstraps_path) > 0 + df_boot = CSV.read(bootstraps_path, DataFrame) + # Keep only SeqName and *_boot columns; drop Sequence (already in merged_df). + boot_cols = filter(c -> c == "SeqName" || endswith(c, "_boot"), names(df_boot)) + select!(df_boot, boot_cols) + merged_df = leftjoin(merged_df, df_boot, on="SeqName") + @info "Joined $(length(boot_cols)-1) bootstrap columns from $bootstraps_path" + end + + # Fill NA/missing values at lower ranks with parent_X convention. + # e.g. if Family=Muribaculaceae and Genus=NA, Genus becomes "Muribaculaceae_X". + # Applied to both plain rank columns and _dada2 suffixed columns. + for suffix in ("", "_dada2") + cols = [l * suffix for l in db_meta.levels if (l * suffix) in names(merged_df)] + for i in 2:length(cols) + col = cols[i] + parent_col = cols[i-1] + merged_df[!, col] = [ + begin + v = merged_df[rn, col] + is_na = ismissing(v) || strip(string(v)) == "NA" || strip(string(v)) == "" + if is_na + p = merged_df[rn, parent_col] + (ismissing(p) || strip(string(p)) == "NA" || strip(string(p)) == "") ? + v : string(strip(string(p))) * "_X" + else + v + end + end + for rn in 1:nrow(merged_df) + ] + end + end + + # Apply database-specific taxonomy corrections from config. + for corr in db_meta.corrections + src_col = get(corr, "source", nothing) + tgt_col = get(corr, "target", nothing) + vals = get(corr, "values", Dict()) + (isnothing(src_col) || isnothing(tgt_col)) && continue + src_col, tgt_col = string(src_col), string(tgt_col) + if src_col in names(merged_df) && tgt_col in names(merged_df) && !isempty(vals) + mapping = Dict(string(k) => string(v) for (k,v) in vals) + merged_df[!, tgt_col] = [ + ismissing(d) ? s : get(mapping, string(d), ismissing(s) ? s : string(s)) + for (d, s) in zip(merged_df[!, src_col], merged_df[!, tgt_col]) + ] + end + end + @info "Merge complete. $(nrow(merged_df)) rows." return merged_df end """ - filter_table(merged_df; filters_yaml_path, remove_empty_domain_override) + filter_table(merged_df, filters_yaml_path) - Applies protist filtering to a merged DataFrame produced by merge_taxonomy_counts using rules defined in a YAML file. + Apply taxonomy filtering to a merged DataFrame using rules from a YAML file. ## Arguments: - `merged_df`: Input DataFrame with taxonomy columns. - - `filters_yaml_path` (default: "./inputs/protist_filter.yml"): Path to YAML file defining filters and mappings. - - `remove_empty_domain_override` (default: nothing): Overrides YAML setting if provided. + - `filters_yaml_path`: Path to YAML file defining mappings, filters, and remove_empty rules. - ## Expected YAML structure: + ## YAML structure: ``` YAML - mappings: { DivisionValue: SupergroupValue, ... } + mappings: + - source_column: Division + target_column: Supergroup + values: { Rhizaria: Rhizaria, ... } filters: - - column: Domain + - column: Domain pattern: Bacteria - remove_empty_domain: true + action: exclude # exclude (default) | keep + regex: false # false (default) = substring match + remove_empty: + - Domain ``` + + A flat `mappings` dict and `remove_empty_domain` bool are auto-detected as legacy format. """ + function _apply_single_mapping!(df::DataFrame, src_col::String, tgt_col::String, mapping) + isempty(mapping) && return + m = Dict(string(k) => string(v) for (k,v) in mapping) + if src_col in names(df) && tgt_col in names(df) + df[!, tgt_col] = [ + ismissing(s) ? t : get(m, string(s), t) + for (s, t) in zip(df[!, src_col], df[!, tgt_col]) + ] + elseif !isempty(m) + missing_cols = filter(c -> c ∉ names(df), [src_col, tgt_col]) + @warn "Column remapping skipped: $(join(missing_cols, ", ")) not in data. " * + "Available: $(join(names(df), ", "))" + end + end + + function _apply_mappings!(df::DataFrame, mapping_config) + if mapping_config isa AbstractDict + # Legacy: flat dict = Division -> Supergroup + _apply_single_mapping!(df, "Division", "Supergroup", mapping_config) + elseif mapping_config isa Vector + for entry in mapping_config + src = get(entry, "source_column", nothing) + tgt = get(entry, "target_column", nothing) + vals = get(entry, "values", Dict()) + (isnothing(src) || isnothing(tgt)) && continue + _apply_single_mapping!(df, string(src), string(tgt), vals) + end + end + end + function filter_table( merged_df::DataFrame, - filters_yaml_path::String; - remove_empty_domain_override::Union{Bool,Nothing} = nothing + filters_yaml_path::String ) - # Load YAML config config = YAML.load_file(filters_yaml_path) - df = deepcopy(merged_df) @info "Filtering table using configuration from $filters_yaml_path" - # Load mappings - mapping_config = get(config, "mappings", Dict()) - if !isempty(mapping_config) - @assert mapping_config isa AbstractDict "mappings must be a dictionary in YAML" - end - - mapping = Dict(string(k) => string(v) for (k,v) in mapping_config) - - # Apply Supergroup mapping if both columns exist - if "Division" in names(df) && "Supergroup" in names(df) && !isempty(mapping) - df.Supergroup = [ - ismissing(d) ? s : (haskey(mapping, string(d)) ? mapping[string(d)] : s) - for (d, s) in zip(df.Division, df.Supergroup) - ] - elseif !isempty(mapping) - missing_cols = filter(c -> c ∉ names(df), ["Division", "Supergroup"]) - @warn "Supergroup remapping skipped: column(s) not present in data: " * - "$(join(missing_cols, ", ")). The mappings block in your filter config " * - "may be tuned for a different reference database (e.g. PR2). " * - "Available columns: $(join(names(df), ", "))" - end - - # Load remove_empty_domain flag - remove_empty = get(config, "remove_empty_domain", true) - if remove_empty_domain_override !== nothing - remove_empty = remove_empty_domain_override + # ---- mappings (auto-detect old vs new format) ---- + _apply_mappings!(df, get(config, "mappings", Dict())) + + # ---- remove_empty (backwards compat with remove_empty_domain) ---- + remove_cols = if haskey(config, "remove_empty") + val = config["remove_empty"] + val isa Vector ? string.(val) : [string(val)] + elseif get(config, "remove_empty_domain", false) + ["Domain"] + else + String[] end - # Optionally remove blank/unassigned Domain entries - if remove_empty && "Domain" in names(df) - df = filter(row -> - !ismissing(row.Domain) && - !isempty(strip(string(row.Domain))) && - lowercase(strip(string(row.Domain))) != "blank", - df) - elseif remove_empty - @warn "remove_empty_domain skipped: 'Domain' column not present in data. " * - "This setting may be tuned for a different reference database (e.g. PR2). " * - "Available columns: $(join(names(df), ", "))" + for col in remove_cols + if col in names(df) + df = filter(row -> + !ismissing(row[col]) && + !isempty(strip(string(row[col]))) && + lowercase(strip(string(row[col]))) != "blank", + df) + else + @warn "remove_empty: column '$col' not in data. Available: $(join(names(df), ", "))" + end end - # Load and apply filters + # ---- filters (with action + regex support) ---- raw_filters = get(config, "filters", []) @assert raw_filters isa Vector "filters must be a list in YAML" for item in raw_filters @assert item isa Dict "Each filter must be a key-value map" - colname = get(item, "column", nothing) - pattern = get(item, "pattern", nothing) - @assert typeof(colname) <: AbstractString "filter column must be a string" - @assert typeof(pattern) <: AbstractString "filter pattern must be a string" + colname = string(get(item, "column", "")) + pattern = string(get(item, "pattern", "")) + action = lowercase(string(get(item, "action", "exclude"))) + use_regex = get(item, "regex", false) == true if !(colname in names(df)) @warn "Skipping filter: column '$colname' not present in data. " * - "This filter may be tuned for a different reference database (e.g. PR2). " * - "Available columns: $(join(names(df), ", "))" + "Available: $(join(names(df), ", "))" continue end - # Exclude rows where column contains the pattern (case-sensitive substring match) - df = filter(row -> ismissing(row[colname]) || !occursin(pattern, string(row[colname])), df) + matcher = use_regex ? (val -> occursin(Regex(pattern), string(val))) : + (val -> occursin(pattern, string(val))) + + if action == "keep" + df = filter(row -> ismissing(row[colname]) || matcher(row[colname]), df) + else # exclude (default) + df = filter(row -> ismissing(row[colname]) || !matcher(row[colname]), df) + end end @info "Filtering complete. $(nrow(df))/$(nrow(merged_df)) rows retained." @@ -274,14 +383,85 @@ export merge_taxonomy_counts, filter_table, merge_taxa return df end - function merge_taxa(project::ProjectCtx, source::ASVResult, tax::TaxonomyHits) + """ + _with_dada2_as_primary(df, levels) -> DataFrame + + Return a copy of `df` where `_dada2` taxonomy columns are renamed to + standard names (and original vsearch columns get a `_vsearch` suffix). + This lets `filter_table` operate on DADA2 taxonomy using unchanged logic. + """ + function _with_dada2_as_primary(df::DataFrame, levels::Vector{String}) + out = copy(df) + for col in levels + dada2_col = col * "_dada2" + dada2_col in names(out) || continue + if col in names(out) + rename!(out, col => col * "_vsearch") + end + rename!(out, dada2_col => col) + end + # Also swap Pident if a DADA2 equivalent exists + if "Pident_dada2" in names(out) && "Pident" in names(out) + rename!(out, "Pident" => "Pident_vsearch", "Pident_dada2" => "Pident") + end + return out + end + + """ + _restore_from_dada2_primary(df, levels) -> DataFrame + + Reverse of `_with_dada2_as_primary`: rename standard columns back to + `_dada2` and `_vsearch` columns back to standard names. + """ + function _restore_from_dada2_primary(df::DataFrame, levels::Vector{String}) + out = copy(df) + for col in levels + vsearch_col = col * "_vsearch" + # Current standard name holds DADA2 values - rename to _dada2 + if col in names(out) + rename!(out, col => col * "_dada2") + end + # Restore vsearch column + if vsearch_col in names(out) + rename!(out, vsearch_col => col) + end + end + if "Pident_vsearch" in names(out) + if "Pident" in names(out) + rename!(out, "Pident" => "Pident_dada2") + end + rename!(out, "Pident_vsearch" => "Pident") + end + return out + end + + """ + filter_table_dada2(merged_df, filters_yaml_path, levels) + + Apply taxonomy filtering using DADA2 columns instead of vsearch columns. + Temporarily swaps column names so `filter_table` logic applies to `_dada2` columns. + """ + function filter_table_dada2(merged_df::DataFrame, filters_yaml_path::String, + levels::Vector{String}) + swapped = _with_dada2_as_primary(merged_df, levels) + filtered = filter_table(swapped, filters_yaml_path) + return _restore_from_dada2_primary(filtered, levels) + end + + function merge_taxa(project::ProjectCtx, source::ASVResult, tax::TaxonomyHits, + db_meta::DatabaseMeta) config_path = write_run_config(project) cfg = get(YAML.load_file(config_path), "merge_taxa", Dict()) filter_list = get(cfg, "filters", [nothing]) merge_dir = joinpath(project.dir, "merged") hash_file = joinpath(merge_dir, "config.hash") - data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy)) + tax_counts_path = joinpath(dirname(source.taxonomy), "tax_counts.csv") + tax_prefix_ = splitext(basename(source.taxonomy))[1] + boot_path_ = joinpath(dirname(source.taxonomy), tax_prefix_ * "_bootstraps.csv") + data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy), + isfile(tax_counts_path) ? mtime(tax_counts_path) : 0.0, + isfile(boot_path_) ? mtime(boot_path_) : 0.0) config_changed = _section_stale(config_path, "merge_taxa", hash_file) merged_csv = joinpath(merge_dir, "merged.csv") @@ -290,13 +470,45 @@ export merge_taxonomy_counts, filter_table, merge_taxa stale_filters = Pair{String,String}[] # stem => filter_path tables = Dict{String,String}("merged" => merged_csv) + filter_colours = Dict{String,String}() + # Pre-compute applicable filter stems (respecting per-filter database restrictions). + filter_stems = String[] + for e in filter_list + isnothing(e) && continue + fp = joinpath(project.config_dir, "filters", string(e)) + if isfile(fp) + allowed = get(YAML.load_file(fp), "databases", nothing) + isnothing(allowed) || db_meta.name ∈ string.(allowed) || continue + end + push!(filter_stems, splitext(string(e))[1]) + end + for entry in filter_list isnothing(entry) && continue filter_name = string(entry) filter_path = joinpath(project.config_dir, "filters", filter_name) stem = splitext(filter_name)[1] output_csv = joinpath(merge_dir, stem * ".csv") + + # Skip filters that declare a databases list not including the active DB. + if isfile(filter_path) + fcfg = YAML.load_file(filter_path) + allowed_dbs = get(fcfg, "databases", nothing) + if !isnothing(allowed_dbs) && db_meta.name ∉ string.(allowed_dbs) + @info "Skipping merge_taxa filter '$stem': not applicable to database '$(db_meta.name)'" + continue + end + if haskey(fcfg, "colour") + filter_colours[stem] = string(fcfg["colour"]) + end + end + tables[stem] = output_csv + # Pre-populate the DADA2-filtered path so it is always present in the + # returned MergedTables, even when the function returns early via the + # skip guard below (the CSV may already exist from a previous run). + tables[stem * "_dada2"] = joinpath(merge_dir, stem * "_dada2" * ".csv") + if config_changed || !isfile(output_csv) || mtime(output_csv) <= max(data_mtime, mtime(filter_path)) push!(stale_filters, stem => filter_path) @@ -307,26 +519,74 @@ export merge_taxonomy_counts, filter_table, merge_taxa if !need_base && isempty(stale_filters) @info "Skipping merge_taxa: all outputs up to date in $merge_dir" - return MergedTables(tables) + return MergedTables(tables, filter_stems, filter_colours) end mkpath(merge_dir) - df = need_base ? merge_taxonomy_counts(tax.tsv, source.taxonomy) : + counts_path = (isfile(tax_counts_path) && filesize(tax_counts_path) > 0) ? + tax_counts_path : source.taxonomy + tables_dir = dirname(source.taxonomy) + tax_prefix = splitext(basename(source.taxonomy))[1] # e.g. "taxonomy" + boot_path = joinpath(tables_dir, tax_prefix * "_bootstraps.csv") + df = need_base ? merge_taxonomy_counts(tax.tsv, counts_path, db_meta; + bootstraps_path=boot_path) : CSV.read(merged_csv, DataFrame) if need_base CSV.write(merged_csv, df) @info "Written: $merged_csv" + pipeline_log(project, "Merge complete. $(nrow(df)) ASVs.") + log_written(project, merged_csv) end for (stem, filter_path) in stale_filters output_csv = tables[stem] - CSV.write(output_csv, filter_table(df, filter_path)) + filtered = filter_table(df, filter_path) + CSV.write(output_csv, filtered) @info "Written: $output_csv" + pipeline_log(project, "Filter '$stem': $(nrow(filtered))/$(nrow(df)) rows retained.") + log_written(project, output_csv) + end + + # ---- DADA2 filtered CSVs (same filters, applied to _dada2 columns) ---- + # Check whether any _dada2 taxonomy columns exist before attempting. + has_dada2 = any(endswith(c, "_dada2") for c in names(df) + if any(startswith(c, t) for t in db_meta.levels)) + + if has_dada2 + for entry in filter_list + isnothing(entry) && continue + filter_name = string(entry) + filter_path = joinpath(project.config_dir, "filters", filter_name) + stem = splitext(filter_name)[1] + dada2_stem = stem * "_dada2" + output_csv = joinpath(merge_dir, dada2_stem * ".csv") + + # Skip filters that declare a databases list not including the active DB. + if isfile(filter_path) + allowed_dbs = get(YAML.load_file(filter_path), "databases", nothing) + if !isnothing(allowed_dbs) && db_meta.name ∉ string.(allowed_dbs) + @info "Skipping merge_taxa DADA2 filter '$stem': not applicable to database '$(db_meta.name)'" + continue + end + end + + if !config_changed && isfile(output_csv) && + mtime(output_csv) > max(data_mtime, mtime(filter_path)) + @info "Skipping merge_taxa DADA2 filter '$stem': $output_csv up to date" + continue + end + + filtered = filter_table_dada2(df, filter_path, db_meta.levels) + CSV.write(output_csv, filtered) + @info "Written: $output_csv (DADA2 filter)" + pipeline_log(project, "Filter '$stem' (DADA2): $(nrow(filtered))/$(nrow(df)) rows retained.") + log_written(project, output_csv) + end end _write_section_hash(config_path, "merge_taxa", hash_file) - return MergedTables(tables) + return MergedTables(tables, filter_stems, filter_colours) end end \ No newline at end of file diff --git a/src/plots.jl b/src/plots.jl new file mode 100644 index 0000000..bc48282 --- /dev/null +++ b/src/plots.jl @@ -0,0 +1,664 @@ +module PipelinePlots + +# CairoMakie-based plotting for the analysis stage. +# +# All public functions write a PDF to the given path and return nothing. +# Module is named PipelinePlots (not Plots) to avoid shadowing Plots.jl. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# +# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export taxa_bar_chart, filter_composition_plot, alpha_diversity_plot, + nmds_plot, alpha_boxplot, pipeline_stats_plot, group_comparison_chart + + using CairoMakie, DataFrames + + CairoMakie.activate!(type = "pdf") + + # Colour palette + # Return `n` visually distinct colours. Starts from Makie's + # Wong palette (7 colours optimised for colour-blindness) then + # extends via golden-angle hue rotation. + function _palette(n::Int) + base = Makie.wong_colors() + n <= length(base) && return collect(base[1:n]) + colors = collect(base) + for i in (length(base) + 1):n + hue = mod(i * 137.508, 360.0) + # Simple HSV to RGB with S=0.65, V=0.85 + s, v = 0.65, 0.85 + c = v * s + x = c * (1.0 - abs(mod(hue / 60.0, 2.0) - 1.0)) + m = v - c + r, g, b = if hue < 60 + (c + m, x + m, m) + elseif hue < 120 + (x + m, c + m, m) + elseif hue < 180 + (m, c + m, x + m) + elseif hue < 240 + (m, x + m, c + m) + elseif hue < 300 + (x + m, m, c + m) + else + (c + m, m, x + m) + end + push!(colors, RGBAf(r, g, b, 1.0)) + end + return colors[1:n] + end + + ## Taxonomy label helpers + # Return the value of the most specific (lowest) non-empty taxonomy rank. + function _lowest_rank_label(row; rank_order::Vector{String}) + label = "Unclassified" + for rank in rank_order + if hasproperty(row, Symbol(rank)) + val = row[Symbol(rank)] + if !ismissing(val) && !isempty(strip(string(val))) + label = string(val) + end + end + end + return label + end + + # Return the value at a specified rank, or "Unclassified". + function _rank_label(row, rank::String) + sym = Symbol(rank) + if hasproperty(row, sym) + val = row[sym] + if !ismissing(val) && !isempty(strip(string(val))) + return string(val) + end + end + return "Unclassified" + end + + # Assign a taxon label to every row of `df`. + function _label_rows(df::DataFrame; rank=nothing, + rank_order::Vector{String})::Vector{String} + if isnothing(rank) + String[_lowest_rank_label(row; rank_order) for row in eachrow(df)] + else + String[_rank_label(row, rank) for row in eachrow(df)] + end + end + + ## Display-name helper + # Strip FASTQ suffixes so tick labels show sample stems only. + const _FASTQ_SUFFIX = r"_R[12](?:_filt|_trimmed)?\.fastq\.gz$" + _display_name(name::String) = replace(name, _FASTQ_SUFFIX => "") + _display_names(names::Vector{String}) = String[_display_name(n) for n in names] + + ## Subtitle helper + function _add_subtitle!(fig, subtitle) + isnothing(subtitle) && return + Label(fig[0, 1:end], subtitle; fontsize=11, color=:gray40) + end + + # Overrides Unclassified / Other as grey + const _GREY = RGBAf(0.7, 0.7, 0.7, 1.0) + + function _apply_grey_override!(colors::Vector, labels::Vector{String}) + for (i, l) in enumerate(labels) + if l in ("Unclassified", "Other") + colors[i] = _GREY + end + end + end + + ## Stacked-bar data builder + # From a label-per-row vector and sample columns, produce the + # aggregated counts matrix (labels x samples), final label list, + # and per-column normalisation (relative or absolute). + function _aggregate_taxa(df::DataFrame, sample_cols::Vector{String}, + labels::AbstractVector{<:AbstractString}; + top_n::Int=15, relative::Bool=true) + unique_labels = String.(unique(labels)) + label_idx = Dict(l => i for (i, l) in enumerate(unique_labels)) + n_labels = length(unique_labels) + n_samples = length(sample_cols) + + counts = zeros(Float64, n_labels, n_samples) + for (ri, row) in enumerate(eachrow(df)) + li = label_idx[labels[ri]] + for (j, col) in enumerate(sample_cols) + v = row[Symbol(col)] + counts[li, j] += ismissing(v) ? 0.0 : Float64(v isa AbstractString ? parse(Float64, v) : v) + end + end + + # Sort labels by total abundance (descending). + totals = vec(sum(counts, dims=2)) + order = sortperm(totals, rev=true) + + # Collapse beyond top_n into "Other". + if n_labels > top_n + keep = order[1:top_n] + other = vec(sum(counts[order[(top_n + 1):end], :], dims=1)) + counts = vcat(counts[keep, :], other') + final_labels = vcat(unique_labels[keep], ["Other"]) + else + counts = counts[order, :] + final_labels = unique_labels[order] + end + + # Optionally normalise to relative abundance. + if relative + col_sums = vec(sum(counts, dims=1)) + for j in 1:n_samples + col_sums[j] > 0 && (counts[:, j] ./= col_sums[j]) + end + end + + return counts, final_labels + end + + # Build the three parallel vectors (x, y, grp) for a stacked barplot. + function _long_format(counts::Matrix{Float64}) + n_grp, n_x = size(counts) + x = Int[] + y = Float64[] + grp = Int[] + for j in 1:n_x, i in 1:n_grp + push!(x, j) + push!(y, counts[i, j]) + push!(grp, i) + end + return x, y, grp + end + + ## Public: taxa_bar_chart + """ + taxa_bar_chart(df, sample_cols, output_pdf; top_n=15, rank=nothing, + relative=true, subtitle=nothing) + + Stacked bar chart of taxonomic composition per sample. + + `df` is a merged/filtered CSV read as a DataFrame. `sample_cols` lists + the column names that hold per-sample ASV counts. Taxa are labelled by + the lowest assigned taxonomic rank (or `rank` if specified) and the + `top_n` most abundant are shown; the rest are collapsed to "Other". + + When `relative=false`, shows absolute read counts instead of proportions. + """ + function taxa_bar_chart(df::DataFrame, sample_cols::Vector{String}, + output_pdf::String; top_n::Int=15, rank=nothing, + relative::Bool=true, + subtitle::Union{Nothing,String}=nothing, + rank_order::Vector{String}) + (nrow(df) == 0 || isempty(sample_cols)) && return nothing + + labels = _label_rows(df; rank, rank_order) + counts, final_labels = _aggregate_taxa(df, sample_cols, labels; + top_n, relative) + n_taxa = length(final_labels) + n_samples = length(sample_cols) + x, y, grp = _long_format(counts) + + colors = _palette(n_taxa) + _apply_grey_override!(colors, final_labels) + color_vec = [colors[g] for g in grp] + + display_names = _display_names(sample_cols) + ylabel = relative ? "Relative abundance" : "Read count" + title = relative ? "Taxonomic composition" : "Taxonomic composition (absolute)" + + fig = Figure(size = (max(800, n_samples * 55 + 250), 600)) + ax = Axis(fig[1, 1]; + xlabel = "Sample", + ylabel = ylabel, + xticks = (1:n_samples, display_names), + xticklabelrotation = π / 4, + title = title) + + barplot!(ax, x, y; stack=grp, color=color_vec) + relative && ylims!(ax, 0, 1) + + elements = [PolyElement(color=colors[i]) for i in 1:n_taxa] + Legend(fig[1, 2], elements, final_labels; framevisible=false, + nbanks=1, labelsize=11) + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: filter_composition_plot + """ + filter_composition_plot(filter_totals, sample_names, output_pdf; + subtitle=nothing) + + Stacked bar chart showing absolute read counts per sample, one colour + per biological-group filter (e.g. "protists", "bacteria", "Unclassified"). + + `filter_totals` maps filter name to a Vector{Float64} of per-sample totals. + """ + function filter_composition_plot(filter_totals::Dict{String, Vector{Float64}}, + sample_names::Vector{String}, + output_pdf::String; + subtitle::Union{Nothing,String}=nothing, + colour_overrides::Dict{String,String}=Dict{String,String}()) + isempty(filter_totals) && return nothing + n_samples = length(sample_names) + + display_names = _display_names(sample_names) + filter_names = sort(collect(keys(filter_totals))) + # Move "Unclassified" to the end if present. + if "Unclassified" in filter_names + filter!(!=( "Unclassified"), filter_names) + push!(filter_names, "Unclassified") + end + n_filters = length(filter_names) + + x = Int[] + y = Float64[] + grp = Int[] + for j in 1:n_samples, (i, fname) in enumerate(filter_names) + push!(x, j) + push!(y, filter_totals[fname][j]) + push!(grp, i) + end + + colors = _palette(n_filters) + _apply_grey_override!(colors, filter_names) + # Apply per-filter colour overrides from YAML configs. + for (i, fname) in enumerate(filter_names) + if haskey(colour_overrides, fname) + hex = colour_overrides[fname] + try + colors[i] = Makie.RGBAf(Makie.Colors.parse(Makie.Colors.Colorant, hex)) + catch + @warn "Invalid colour '$hex' for filter '$fname', using default" + end + end + end + color_vec = [colors[g] for g in grp] + + fig = Figure(size = (max(800, n_samples * 55 + 250), 600)) + ax = Axis(fig[1, 1]; + xlabel = "Sample", + ylabel = "Read count", + xticks = (1:n_samples, display_names), + xticklabelrotation = π / 4, + title = "Filter composition") + + barplot!(ax, x, y; stack=grp, color=color_vec) + + elements = [PolyElement(color=colors[i]) for i in 1:n_filters] + Legend(fig[1, 2], elements, filter_names; framevisible=false) + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: alpha_diversity_plot + """ + alpha_diversity_plot(alpha_df, output_pdf; subtitle=nothing) + + Three-panel bar chart (richness, Shannon, Simpson) with samples on X. + + `alpha_df` must have columns `:sample`, `:richness`, `:shannon`, `:simpson`. + """ + function alpha_diversity_plot(alpha_df::DataFrame, output_pdf::String; + subtitle::Union{Nothing,String}=nothing) + nrow(alpha_df) == 0 && return nothing + n = nrow(alpha_df) + samples = _display_names(String.(alpha_df.sample)) + + metrics = [ + ("Richness (observed ASVs)", Float64.(alpha_df.richness)), + ("Shannon index", Float64.(alpha_df.shannon)), + ("Simpson index", Float64.(alpha_df.simpson)), + ] + + fig = Figure(size = (max(700, n * 50), 900)) + for (row, (title, vals)) in enumerate(metrics) + ax = Axis(fig[row, 1]; + ylabel = title, + xticks = (1:n, samples), + xticklabelrotation = π / 4) + row == 1 && (ax.title = "Alpha diversity") + row < 3 && (ax.xticklabelsvisible = false; ax.xlabelvisible = false) + row == 3 && (ax.xlabel = "Sample") + barplot!(ax, 1:n, vals; color=Makie.wong_colors()[1]) + end + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: nmds_plot + # Marker shapes for dual-encoded NMDS. + const _MARKER_SHAPES = [:circle, :rect, :diamond, :utriangle, + :dtriangle, :cross, :star5, :hexagon] + + """ + nmds_plot(coords, labels, output_pdf; colour_by=nothing, + shape_by=nothing, colour_label="Group", + shape_label="Group", stress=nothing, subtitle=nothing) + + NMDS ordination scatter plot. + + `coords` is an nx2 matrix (from R `metaMDS\$points`). + `labels` are sample names. When `colour_by` is a `Vector{String}`, + points are coloured by group membership. When `shape_by` is also + provided, points are additionally encoded by marker shape with a + grouped legend. + """ + function nmds_plot(coords::Matrix{Float64}, labels::Vector{String}, + output_pdf::String; + colour_by::Union{Nothing, Vector{String}}=nothing, + shape_by::Union{Nothing, Vector{String}}=nothing, + colour_label::String="Group", + shape_label::String="Group", + stress::Union{Nothing, Float64}=nothing, + subtitle::Union{Nothing,String}=nothing) + n = size(coords, 1) + n == 0 && return nothing + + fig = Figure(size = (700, 650)) + ax = Axis(fig[1, 1]; + xlabel = "NMDS1", ylabel = "NMDS2", + title = "NMDS ordination") + + if isnothing(colour_by) && isnothing(shape_by) + # Plain scatter - no grouping. + scatter!(ax, coords[:, 1], coords[:, 2]; + markersize=10, color=Makie.wong_colors()[1]) + + elseif !isnothing(colour_by) && isnothing(shape_by) + # Colour-only grouping. + cgroups = unique(colour_by) + colors = _palette(length(cgroups)) + for (gi, g) in enumerate(cgroups) + mask = colour_by .== g + scatter!(ax, coords[mask, 1], coords[mask, 2]; + markersize=10, color=colors[gi], label=g) + end + Legend(fig[1, 2], ax; framevisible=false) + + else + # Dual-encoded: colour x shape. + cgroups = isnothing(colour_by) ? ["all"] : unique(colour_by) + sgroups = unique(shape_by) + colors = _palette(length(cgroups)) + cb = isnothing(colour_by) ? fill("all", n) : colour_by + + # Plot each combination. + for (ci, cg) in enumerate(cgroups), (si, sg) in enumerate(sgroups) + mask = (cb .== cg) .& (shape_by .== sg) + any(mask) || continue + mi = mod1(si, length(_MARKER_SHAPES)) + scatter!(ax, coords[mask, 1], coords[mask, 2]; + markersize=10, color=colors[ci], + marker=_MARKER_SHAPES[mi]) + end + + # Build grouped legend: elements, labels, titles. + colour_elements = [MarkerElement(color=colors[i], marker=:circle, + markersize=10) + for i in 1:length(cgroups)] + colour_labels = [string(g) for g in cgroups] + + shape_elements = [MarkerElement(color=:gray50, + marker=_MARKER_SHAPES[mod1(i, length(_MARKER_SHAPES))], + markersize=10) + for i in 1:length(sgroups)] + shape_labels = [string(g) for g in sgroups] + + Legend(fig[1, 2], + [colour_elements, shape_elements], + [colour_labels, shape_labels], + [colour_label, shape_label]; + framevisible=false) + end + + # Annotate stress value. + if !isnothing(stress) + text!(ax, 0.02, 0.98; + text = "stress = $(round(stress; digits=3))", + space = :relative, + align = (:left, :top), + fontsize = 12) + end + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: alpha_boxplot (group / study level) + """ + alpha_boxplot(all_alpha, group_labels, output_pdf; subtitle=nothing) + + Three-panel boxplot (richness, Shannon, Simpson) comparing groups. + + `all_alpha` is a `Vector{DataFrame}`, one per group, each with columns + `:sample`, `:richness`, `:shannon`, `:simpson`. + `group_labels` names each group. + """ + function alpha_boxplot(all_alpha::Vector{DataFrame}, + group_labels::Vector{String}, + output_pdf::String; + subtitle::Union{Nothing,String}=nothing) + isempty(all_alpha) && return nothing + n_groups = length(group_labels) + colors = _palette(n_groups) + + metrics = [:richness, :shannon, :simpson] + titles = ["Richness (observed ASVs)", "Shannon index", "Simpson index"] + + fig = Figure(size = (max(600, n_groups * 120), 900)) + for (row, (col, title)) in enumerate(zip(metrics, titles)) + ax = Axis(fig[row, 1]; + ylabel = title, + xticks = (1:n_groups, group_labels), + xticklabelrotation = π / 6) + row == 1 && (ax.title = "Alpha diversity comparison") + row < 3 && (ax.xticklabelsvisible = false; ax.xlabelvisible = false) + row == 3 && (ax.xlabel = "Group") + + positions = Int[] + values = Float64[] + cols_vec = RGBAf[] + for (gi, adf) in enumerate(all_alpha) + vals = Float64.(adf[!, col]) + append!(positions, fill(gi, length(vals))) + append!(values, vals) + append!(cols_vec, fill(colors[gi], length(vals))) + end + + boxplot!(ax, positions, values; color=cols_vec) + end + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: pipeline_stats_plot + # Readable stage labels for pipeline_stats columns. + const _STAGE_LABELS = Dict( + "input" => "Input", + "filtered" => "Filtered", + "denoisedF" => "Denoised (F)", + "denoisedR" => "Denoised (R)", + "merged" => "Merged", + "nochim" => "Chimera-free", + ) + + # Canonical stage order - extras (reads_*) are appended dynamically. + const _STAGE_ORDER = ["input", "filtered", "denoisedF", "denoisedR", + "merged", "nochim"] + + function _stage_label(col::String) + haskey(_STAGE_LABELS, col) && return _STAGE_LABELS[col] + # reads_protist_filter -> "After protist_filter" + if startswith(col, "reads_") + return "After " * replace(col[7:end], "_" => " ") + end + return col + end + + """ + pipeline_stats_plot(stats_df, output_pdf; subtitle=nothing) + + Grouped bar chart showing read counts at each pipeline stage per sample. + + `stats_df` must have a `:sample` column and one or more stage columns + (input, filtered, denoisedF, ..., reads_*). + """ + function pipeline_stats_plot(stats_df::DataFrame, output_pdf::String; + subtitle::Union{Nothing,String}=nothing) + nrow(stats_df) == 0 && return nothing + + all_cols = names(stats_df) + + # Select stage columns in canonical order, then reads_* sorted. + stage_cols = String[] + for col in _STAGE_ORDER + col in all_cols && push!(stage_cols, col) + end + reads_cols = sort([c for c in all_cols + if startswith(c, "reads_")]) + append!(stage_cols, reads_cols) + isempty(stage_cols) && return nothing + + sample_names = _display_names(String.(stats_df.sample)) + n_samples = length(sample_names) + n_stages = length(stage_cols) + stage_labels = [_stage_label(c) for c in stage_cols] + + colors = _palette(n_samples) + + # Build dodge-grouped barplot data. + xs = Int[] + ys = Float64[] + grps = Int[] + for (si, row) in enumerate(eachrow(stats_df)) + for (xi, col) in enumerate(stage_cols) + v = row[Symbol(col)] + push!(xs, xi) + push!(ys, ismissing(v) ? 0.0 : Float64(v)) + push!(grps, si) + end + end + + fig = Figure(size = (max(800, n_stages * 100 + 200), 600)) + ax = Axis(fig[1, 1]; + xlabel = "Pipeline stage", + ylabel = "Read count", + xticks = (1:n_stages, stage_labels), + xticklabelrotation = π / 4, + title = "Reads through pipeline stages") + + barplot!(ax, xs, ys; dodge=grps, + color=[colors[g] for g in grps]) + + elements = [PolyElement(color=colors[i]) for i in 1:n_samples] + Legend(fig[1, 2], elements, sample_names; framevisible=false, + labelsize=11) + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + + ## Public: group_comparison_chart + """ + group_comparison_chart(dfs_per_group, scols_per_group, group_names, + output_pdf; top_n=15, rank=nothing, + relative=true, subtitle=nothing) + + Stacked bar chart with one bar per group (aggregated reads across all + samples in the group). Useful for cross-group comparison at a glance. + """ + function group_comparison_chart(dfs_per_group::Vector{DataFrame}, + scols_per_group::Vector{Vector{String}}, + group_names::Vector{String}, + output_pdf::String; + top_n::Int=15, rank=nothing, + relative::Bool=true, + subtitle::Union{Nothing,String}=nothing, + rank_order::Vector{String}) + isempty(dfs_per_group) && return nothing + + # Combine all DataFrames and build labels. + combined_df = reduce((a, b) -> vcat(a, b; cols=:union), dfs_per_group) + labels = _label_rows(combined_df; rank, rank_order) + + # Build a synthetic count matrix: one column per group. + unique_labels = String.(unique(labels)) + label_idx = Dict(l => i for (i, l) in enumerate(unique_labels)) + n_labels = length(unique_labels) + n_groups = length(group_names) + + counts = zeros(Float64, n_labels, n_groups) + row_offset = 0 + for (gi, (df, scols)) in enumerate(zip(dfs_per_group, scols_per_group)) + for ri in 1:nrow(df) + li = label_idx[labels[row_offset + ri]] + for col in scols + v = df[ri, Symbol(col)] + counts[li, gi] += ismissing(v) ? 0.0 : Float64(v) + end + end + row_offset += nrow(df) + end + + # Sort by total abundance, collapse beyond top_n. + totals = vec(sum(counts, dims=2)) + order = sortperm(totals, rev=true) + if n_labels > top_n + keep = order[1:top_n] + other = vec(sum(counts[order[(top_n + 1):end], :], dims=1)) + counts = vcat(counts[keep, :], other') + final_labels = vcat(unique_labels[keep], ["Other"]) + else + counts = counts[order, :] + final_labels = unique_labels[order] + end + + if relative + col_sums = vec(sum(counts, dims=1)) + for j in 1:n_groups + col_sums[j] > 0 && (counts[:, j] ./= col_sums[j]) + end + end + + n_taxa = length(final_labels) + x, y, grp = _long_format(counts) + + colors = _palette(n_taxa) + _apply_grey_override!(colors, final_labels) + color_vec = [colors[g] for g in grp] + + ylabel = relative ? "Relative abundance" : "Read count" + title = relative ? "Group comparison" : "Group comparison (absolute)" + + fig = Figure(size = (max(600, n_groups * 80 + 250), 600)) + ax = Axis(fig[1, 1]; + xlabel = "Group", + ylabel = ylabel, + xticks = (1:n_groups, group_names), + xticklabelrotation = π / 4, + title = title) + + barplot!(ax, x, y; stack=grp, color=color_vec) + relative && ylims!(ax, 0, 1) + + elements = [PolyElement(color=colors[i]) for i in 1:n_taxa] + Legend(fig[1, 2], elements, final_labels; framevisible=false, + nbanks=1, labelsize=11) + + _add_subtitle!(fig, subtitle) + save(output_pdf, fig) + return nothing + end + +end diff --git a/src/project.jl b/src/project.jl index 3ec0c28..c70d3c0 100644 --- a/src/project.jl +++ b/src/project.jl @@ -3,6 +3,7 @@ module ProjectSetup export new_project using ..PipelineTypes + using ..PipelineLog # Header written to every new pipeline.yml override file. const _PIPELINE_YML_HEADER = """ @@ -131,7 +132,7 @@ run-level file for narrower scope. falling back to `config/defaults/pipeline.yml`. `databases.yml` and `tools.yml` are copied from `config/defaults/` on first - run (they are not part of the cascade -- edit them in place). + run (they are not part of the cascade - edit them in place). Re-running `new_project` never overwrites existing files. @@ -206,7 +207,9 @@ run-level file for narrower scope. for rp in leaf_relpaths proj_dir = rp == "." ? root_project : joinpath(root_project, rp) fastq_dir = rp == "." ? root_data : joinpath(root_data, rp) - push!(projects, ProjectCtx(proj_dir, config_dir, fastq_dir, root_project)) + ctx = ProjectCtx(proj_dir, config_dir, fastq_dir, root_project) + pipeline_log(ctx, "Project initialised") + push!(projects, ctx) end @info "$(length(projects)) project(s) ready under $root_project" return projects diff --git a/src/types.jl b/src/types.jl index 8a87686..4e6cf92 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,6 +1,6 @@ module PipelineTypes - export HasFasta, ProjectCtx, TrimmedReads, ASVResult, DenoisedASVs, TaxonomyHits, MergedTables + export HasFasta, ProjectCtx, TrimmedReads, ASVResult, DenoisedASVs, TaxonomyHits, MergedTables, DatabaseMeta abstract type HasFasta end @@ -32,8 +32,18 @@ module PipelineTypes tsv::String # .../vsearch/taxonomy.tsv end + struct DatabaseMeta + name::String + levels::Vector{String} + vsearch_format::String + corrections::Vector{Dict{String,Any}} + noncounts::Set{String} + end + struct MergedTables tables::Dict{String,String} # name => CSV path; always includes "merged" (unfiltered) + filter_order::Vector{String} # filter stems in pipeline.yml order (for priority) + filter_colours::Dict{String,String} # filter stem => hex colour override (from YAML "colour" key) end end From 471c882f9bdc9a724289dbc98cb7a2439eb7ec03 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Thu, 26 Feb 2026 18:07:00 +0100 Subject: [PATCH 030/175] Added declarative graph introspection, removed SILVA and references...it was getting messy and altogether not worth it for now. --- config/defaults/databases.yml | 18 - config/defaults/pipeline.yml | 33 +- config/filters/bacteria_archaea.silva.yml | 13 - .../filters/environmental_protozoa.silva.yml | 19 - config/filters/fungi.silva.yml | 13 - config/filters/metazoa.silva.yml | 14 - config/filters/parasitic_protozoa.silva.yml | 20 - config/filters/plants.silva.yml | 23 - config/filters/protist.silva.yml | 19 - install.jl | 31 +- src/analysis.jl | 1389 +---------------- src/analysis/composition.jl | 119 ++ src/analysis/group.jl | 285 ++++ src/analysis/helpers.jl | 186 +++ src/analysis/report.jl | 160 ++ src/analysis/run.jl | 168 ++ src/analysis/stats.jl | 154 ++ src/analysis/study.jl | 325 ++++ src/call_tools.jl | 125 +- src/databases.jl | 96 +- src/graph.jl | 124 ++ src/main.jl | 86 +- ...merge_and_filter_taxa.jl => merge_taxa.jl} | 14 - src/types.jl | 24 +- 24 files changed, 1762 insertions(+), 1696 deletions(-) delete mode 100644 config/filters/bacteria_archaea.silva.yml delete mode 100644 config/filters/environmental_protozoa.silva.yml delete mode 100644 config/filters/fungi.silva.yml delete mode 100644 config/filters/metazoa.silva.yml delete mode 100644 config/filters/parasitic_protozoa.silva.yml delete mode 100644 config/filters/plants.silva.yml delete mode 100644 config/filters/protist.silva.yml create mode 100644 src/analysis/composition.jl create mode 100644 src/analysis/group.jl create mode 100644 src/analysis/helpers.jl create mode 100644 src/analysis/report.jl create mode 100644 src/analysis/run.jl create mode 100644 src/analysis/stats.jl create mode 100644 src/analysis/study.jl create mode 100644 src/graph.jl rename src/{merge_and_filter_taxa.jl => merge_taxa.jl} (96%) diff --git a/config/defaults/databases.yml b/config/defaults/databases.yml index e17a5f4..8541824 100644 --- a/config/defaults/databases.yml +++ b/config/defaults/databases.yml @@ -34,21 +34,3 @@ databases: Metamonada: Metamonada Telonemia: Telonemia Ancyromonadida: Ancyromonadida - silva: - dada2: - uri: "https://zenodo.org/records/4587955/files/silva_nr99_v138.1_train_set.fa.gz" - local: ~ - remote_path: ~ # pre-existing path on remote host (avoids transferring the file) - vsearch: - uri: "https://www.arb-silva.de/fileadmin/silva_databases/release_138_1/Exports/SILVA_138.1_SSURef_NR99_tax_silva.fasta.gz" - local: ~ - reformat: silva_vsearch # embed taxonomy in sequence ID so vsearch returns it in target field - levels: - - Kingdom - - Phylum - - Class - - Order - - Family - - Genus - vsearch_format: silva # >Acc Kingdom;Phylum;...;Genus - corrections: [] diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 1aa29b1..977d02b 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -1,8 +1,21 @@ +fastqc: + threads: 20 # number of parallel FastQC threads + optional_args: "" # additional flags passed verbatim to fastqc + +multiqc: + optional_args: "" # additional flags passed verbatim to multiqc + cutadapt: # Primer pair names to apply. Must match keys in the Pairs section of config/primers.yml. primer_pairs: - PrimerPair1 - optional_args: "-m 200 --discard-untrimmed" + min_length: 200 # discard reads shorter than this after trimming (-m) + discard_untrimmed: true # drop reads where no adapter was found (--discard-untrimmed) + cores: 0 # parallel cores; 0 = auto-detect (-j) + quality_cutoff: ~ # 3' quality trimming cutoff, null to disable (-q) + error_rate: ~ # max adapter mismatch rate, null = cutadapt default (-e) + overlap: ~ # min adapter overlap length, null = cutadapt default (-O) + optional_args: "" # additional flags passed verbatim to cutadapt dada2: file_patterns: @@ -56,10 +69,17 @@ dada2: verbose: true vsearch: - optional_args: "--id 0.75 --query_cov 0.8" + identity: 0.75 # minimum identity threshold (--id) + query_cov: 0.8 # minimum fraction of query covered (--query_cov) + maxaccepts: ~ # stop after this many hits per query, null = vsearch default (--maxaccepts) + maxrejects: ~ # max rejected candidates per query, null = vsearch default (--maxrejects) + strand: ~ # "plus" or "both"; null = vsearch default (--strand) + optional_args: "" # additional flags passed verbatim to vsearch cdhit: - optional_args: "-c 0.9" + identity: 0.97 # sequence identity threshold (-c); use 1.0 to deduplicate only + threads: 0 # worker threads; 0 = all available (-T) + optional_args: "" # additional flags passed verbatim to cd-hit-est merge_taxa: # List of filter configs to apply, each relative to config/filters/. @@ -68,19 +88,12 @@ merge_taxa: # Set to [] or omit entries to produce merged.csv only. filters: - "parasitic_protozoa.pr2.yml" - - "parasitic_protozoa.silva.yml" - "environmental_protozoa.pr2.yml" - - "environmental_protozoa.silva.yml" - "protist.pr2.yml" - - "protist.silva.yml" - "helminths.pr2.yml" - "bacteria_archaea.pr2.yml" - - "bacteria_archaea.silva.yml" - "fungi.pr2.yml" - - "fungi.silva.yml" - - "metazoa.silva.yml" - "plants_invertebrates.pr2.yml" - - "plants.silva.yml" - "vertebrates.pr2.yml" analysis: diff --git a/config/filters/bacteria_archaea.silva.yml b/config/filters/bacteria_archaea.silva.yml deleted file mode 100644 index b5bc15e..0000000 --- a/config/filters/bacteria_archaea.silva.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Bacteria + Archaea filter — retains all prokaryotic ASVs. -# Covers Firmicutes, Bacteroidota, Cyanobacteria, Proteobacteria, -# Spirochaetota, and all Archaea (Euryarchaeota, Crenarchaeota, etc.). -databases: [silva] - -filters: - - column: Kingdom - pattern: "Bacteria|Archaea" - regex: true - action: keep - -remove_empty: - - Kingdom diff --git a/config/filters/environmental_protozoa.silva.yml b/config/filters/environmental_protozoa.silva.yml deleted file mode 100644 index 28870e4..0000000 --- a/config/filters/environmental_protozoa.silva.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Environmental (free-living) protozoa. -# -# SILVA equivalents of PR2 groups: -# Cercozoa -> Order: Cercozoa (Cercomonas, Heteromita; under Class Rhizaria) -# Ciliophora -> Order: Ciliophora (Litostomatea, Colpodea; under Class Alveolata) -# Gyrista -> Order: Ochrophyta (Chrysophyceae, Xanthophyceae; under Class Stramenopiles) -# -# Note: Chrompodellids (Colpodellida) cannot be separated in SILVA's -# 6-rank hierarchy; they fall under Alveolata without a distinct Order. -databases: [silva] - -filters: - - column: Order - pattern: "Cercozoa|Ciliophora|Ochrophyta" - regex: true - action: keep - -remove_empty: - - Order diff --git a/config/filters/fungi.silva.yml b/config/filters/fungi.silva.yml deleted file mode 100644 index 72f682e..0000000 --- a/config/filters/fungi.silva.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Fungi filter — retains all fungal ASVs. -# SILVA taxonomy: Amorphea > Obazoa > Opisthokonta > Nucletmycea. -# Nucletmycea is the Family-level clade containing all Fungi in SILVA's -# 6-rank eukaryotic hierarchy. -databases: [silva] - -filters: - - column: Family - pattern: Nucletmycea - action: keep - -remove_empty: - - Family diff --git a/config/filters/metazoa.silva.yml b/config/filters/metazoa.silva.yml deleted file mode 100644 index d25b478..0000000 --- a/config/filters/metazoa.silva.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Host contamination filter — retains metazoan sequences (SILVA limitation). -# -# SILVA's 6-rank eukaryotic hierarchy resolves all metazoa (vertebrates, -# nematodes, annelids) to Family=Holozoa. This filter cannot distinguish -# vertebrates from invertebrates. Use PR2 for vertebrate-specific filtering. -databases: [silva] - -filters: - - column: Family - pattern: Holozoa - action: keep - -remove_empty: - - Family diff --git a/config/filters/parasitic_protozoa.silva.yml b/config/filters/parasitic_protozoa.silva.yml deleted file mode 100644 index 05f5872..0000000 --- a/config/filters/parasitic_protozoa.silva.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Parasitic protozoa — host-associated protists. -# -# SILVA equivalents of PR2 groups (matched at Family level): -# Apicomplexa -> Family: Conoidasida (Coccidia, Gregarinasina) -# Parabasalia -> Family: Trichomonadea (Hexamastix, Simplicimonas) -# Bigyra -> Family: Blastocystis + Proteromonadea -# -# Note: Fornicata (diplomonads like Iotanema, Hexamita) do not appear in -# this SILVA dataset at a distinguishable Family. If present, they would -# be under Class=Metamonada but lack a unique Family-level label here. -databases: [silva] - -filters: - - column: Family - pattern: "Conoidasida|Trichomonadea|Blastocystis|Proteromonadea" - regex: true - action: keep - -remove_empty: - - Family diff --git a/config/filters/plants.silva.yml b/config/filters/plants.silva.yml deleted file mode 100644 index 67df81e..0000000 --- a/config/filters/plants.silva.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Plants + invertebrates (dietary / environmental contamination). -# Strategy: exclude all other major groups. -# -# Note: SILVA cannot distinguish vertebrates from invertebrates — all -# metazoa resolve to Family=Holozoa. This filter retains all metazoa -# alongside plants. Use PR2 for vertebrate-specific exclusion. -databases: [silva] - -filters: - - column: Kingdom - pattern: "Bacteria|Archaea" - regex: true - - column: Family - pattern: Nucletmycea - - column: Family - pattern: "Conoidasida|Trichomonadea|Blastocystis|Proteromonadea" - regex: true - - column: Order - pattern: "Cercozoa|Ciliophora|Ochrophyta" - regex: true - -remove_empty: - - Kingdom diff --git a/config/filters/protist.silva.yml b/config/filters/protist.silva.yml deleted file mode 100644 index 551b541..0000000 --- a/config/filters/protist.silva.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Protist filter — retains free-living and parasitic protists. -# Only applied when the active database is SILVA. -# -# Excludes: Bacteria, Archaea, plants (Chloroplastida), fungi -# (Nucletmycea), and metazoa (Holozoa). -databases: [silva] - -filters: - - column: Kingdom - pattern: "Bacteria|Archaea" - regex: true - - column: Class - pattern: Chloroplastida - - column: Family - pattern: "Nucletmycea|Holozoa" - regex: true - -remove_empty: - - Kingdom diff --git a/install.jl b/install.jl index 1ea9e62..962f126 100644 --- a/install.jl +++ b/install.jl @@ -302,18 +302,31 @@ function install_python_tool(name::String)::String name end -function install_r_packages(packages::Vector{String}) - pkgs_r = join(["\"$p\"" for p in packages], ", ") +function install_r_packages(packages::Vector{String}; force_reinstall::Bool=false) + pkgs_r = join(["\"$p\"" for p in packages], ", ") + force_r = force_reinstall ? "TRUE" : "FALSE" snippet = """ if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager", repos = "https://cloud.r-project.org") + pkgs <- c($pkgs_r) - missing <- pkgs[!sapply(pkgs, requireNamespace, quietly = TRUE)] - if (length(missing) > 0) { - message("Installing: ", paste(missing, collapse = ", ")) - BiocManager::install(missing, ask = FALSE) + + if ($force_r) { + message("Reinstalling all packages from source (--update mode)...") + BiocManager::install(pkgs, ask = FALSE, force = TRUE, type = "source") } else { - message("All R packages already installed.") + # requireNamespace only checks presence, not load-time linkage (e.g. Rcpp ABI). + # Use tryCatch(library(...)) to detect packages that are present but broken. + broken <- pkgs[!sapply(pkgs, function(p) { + tryCatch({ library(p, character.only = TRUE); TRUE }, + error = function(e) FALSE) + })] + if (length(broken) > 0) { + message("Installing/repairing: ", paste(broken, collapse = ", ")) + BiocManager::install(broken, ask = FALSE, type = "source") + } else { + message("All R packages already installed and loadable.") + } } """ try @@ -440,8 +453,8 @@ function main() println() println(" --- R packages -----------------------------------------------------") r_packages = ["dada2", "tidyverse", "vegan"] - if prompt_yn(" Install/check R packages (dada2, tidyverse)?") - install_r_packages(r_packages) + if prompt_yn(" Install/check R packages (dada2, tidyverse, vegan)?") + install_r_packages(r_packages; force_reinstall=UPDATE_MODE) end # Write config diff --git a/src/analysis.jl b/src/analysis.jl index 77ff9fb..fce14f0 100644 --- a/src/analysis.jl +++ b/src/analysis.jl @@ -1,1382 +1,35 @@ module Analysis -# Orchestration module for post-pipeline analysis. +# Post-pipeline analysis: per-run, per-group, and study-level outputs. # -# Three analysis levels: -# analyse_run - per-run: pipeline summary CSV, taxa bar chart, -# filter composition, alpha diversity (PDFs) -# analyse_group - per-group: multi-run comparison charts, NMDS -# analyse_study - study-wide: cross-group NMDS, alpha comparison, -# PERMANOVA (when metadata.csv is present) +# Entry points (called from main.jl): +# analyse_run(project, merged, asvs, db_meta; plot_lock) +# analyse_study(projects, merged_results, db_metas; plot_lock) +# load_metadata(start_dir, study_dir) # -# analyse_run is called inside the @threads loop (no R calls). -# analyse_study is called once after the loop and dispatches to -# analyse_group internally (may call R for NMDS/PERMANOVA). +# Internal structure (src/analysis/): +# helpers.jl — column helpers, taxonomy view, rank/source key utilities +# stats.jl — alpha diversity, count matrices, NMDS/PERMANOVA (R), metadata loading +# composition.jl — pipeline summary CSV, priority filter composition +# report.jl — chart generation, text report writing, table formatters +# run.jl — analyse_run (level 1: per-run charts + reports) +# group.jl — _analyse_group (level 2: multi-run comparison) +# study.jl — _analyse_study_level + analyse_study (level 3: cross-group) # # © 2026 Joshua Benjamin Jewell. All rights reserved. -# -# This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). +# Licensed under the GNU Affero General Public License version 3 (AGPLv3). export analyse_run, analyse_study, load_metadata using CSV, DataFrames, Dates, Logging, Statistics, YAML, RCall using ..PipelineTypes, ..PipelineLog, ..Config, ..DiversityMetrics, ..PipelinePlots - ## Column identification - """ - _sample_cols(df, db_meta) -> Vector{String} - - Return column names that hold per-sample ASV counts by excluding known - taxonomy / metadata columns and any column ending with `_dada2` or - `_boot`. - """ - function _sample_cols(df::DataFrame, db_meta::DatabaseMeta) - filter(names(df)) do col - col ∉ db_meta.noncounts && - !endswith(col, "_dada2") && - !endswith(col, "_boot") && - !endswith(col, "_vsearch") && - !isempty(col) - end - end - - ## CSV cache - # Read a CSV once and reuse. Stores (DataFrame, sample_cols). - const _CSVCache = Dict{String, Tuple{DataFrame, Vector{String}}} - - function _cached_read(cache::_CSVCache, path::String, db_meta::DatabaseMeta) - haskey(cache, path) && return cache[path] - df = CSV.read(path, DataFrame) - scols = _sample_cols(df, db_meta) - # Ensure sample columns are numeric (DADA2 tax_counts.csv may write them as strings). - for col in scols - if eltype(df[!, col]) <: AbstractString || eltype(df[!, col]) == Union{Missing, String} - df[!, col] = [ismissing(v) ? missing : parse(Float64, v) for v in df[!, col]] - elseif !(eltype(df[!, col]) <: Union{Missing, Number}) - df[!, col] = passmissing(x -> Float64(x)).(df[!, col]) - end - end - cache[path] = (df, scols) - return df, scols - end - - # Taxonomy methods: vsearch uses standard column names, dada2 uses _dada2 suffix. - const _TAX_METHODS = ["vsearch", "dada2"] - - """ - _dada2_taxonomy_view(df, levels) -> DataFrame - - Return a copy where `_dada2` taxonomy columns are renamed to standard names - (vsearch columns get `_vsearch` suffix). Allows plotting/reporting code to - work unchanged on DADA2 taxonomy. - """ - function _dada2_taxonomy_view(df::DataFrame, levels::Vector{String}) - vdf = copy(df) - for rank in levels - dada2_col = rank * "_dada2" - dada2_col in names(vdf) || continue - if rank in names(vdf) - rename!(vdf, rank => rank * "_vsearch") - end - rename!(vdf, dada2_col => rank) - end - return vdf - end - - # Check whether a DataFrame has any DADA2 taxonomy columns. - _has_dada2(df::DataFrame, levels::Vector{String}) = any(c -> endswith(c, "_dada2") && - any(r -> startswith(c, r), levels), names(df)) - - # For a given method, return the appropriate source keys from a MergedTables. - # vsearch: keys without _dada2 suffix (existing behaviour) - # dada2: for each filter stem, use "{stem}_dada2" key; for "merged" use "merged" (same CSV, both taxonomies) - function _method_source_keys(merged::MergedTables, method::String) - if method == "dada2" - keys_out = String[] - for k in sort(collect(keys(merged.tables))) - k == "merged" && continue - endswith(k, "_dada2") && push!(keys_out, k) - end - push!(keys_out, "merged") - return keys_out - else - return sort([k for k in keys(merged.tables) - if k != "merged" && !endswith(k, "_dada2")]) |> - ks -> vcat(ks, ["merged"]) - end - end - - # Get the CSV path and apply taxonomy view for a method. - function _method_df(raw_df::DataFrame, method::String, levels::Vector{String}) - method == "dada2" ? _dada2_taxonomy_view(raw_df, levels) : raw_df - end - - # Source dirname for method-qualified keys (strip _dada2 suffix for directory names). - _method_source_dirname(key::String) = _source_dirname( - endswith(key, "_dada2") ? key[1:end-6] : key) - - # Lowest assigned taxonomic rank for a row. - function _lowest_rank_label(row, levels::Vector{String}) - label = "Unclassified" - for rank in levels - if hasproperty(row, Symbol(rank)) - val = row[Symbol(rank)] - if !ismissing(val) && !isempty(strip(string(val))) - label = string(val) - end - end - end - return label - end - - ## Stem-to-column matching - # Match pipeline_stats sample stems to full FASTQ column names. - # Returns a Dict mapping each stem to the matching column name - # (or the stem itself if no match is found). - function _stem_to_colname(stems::AbstractVector, col_names::Vector{String}) - mapping = Dict{String, String}() - for stem in stems - s = String(stem) - matched = false - for col in col_names - if startswith(col, s * "_") || col == s - mapping[s] = col - matched = true - break - end - end - matched || (mapping[s] = s) - end - return mapping - end - - ## Source label helper - _source_label(key::String) = key == "merged" ? "unfiltered" : key - - ## Taxa rank iteration - # Derive taxa chart ranks from database levels and optional config override. - function _taxa_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) - configured = get(get(analysis_cfg, "taxa_bar", Dict()), "ranks", nothing) - if !isnothing(configured) && configured isa Vector - return Tuple{String, Union{String,Nothing}}[ - (lowercase(string(r)), string(r) == "asv" ? nothing : string(r)) for r in configured - ] - end - n = length(levels) - ranks = Tuple{String, Union{String,Nothing}}[("asv", nothing)] - n >= 1 && push!(ranks, (lowercase(levels[end]), levels[end])) - n >= 2 && push!(ranks, (lowercase(levels[end-1]), levels[end-1])) - return ranks - end - - # Derive report ranks from database levels and optional config override. - function _report_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) - configured = get(get(analysis_cfg, "taxa_bar", Dict()), "report_ranks", nothing) - if !isnothing(configured) && configured isa Vector - return [(string(r), string(r)) for r in configured] - end - n = length(levels) - ranks = Tuple{String,String}[] - n >= 3 && push!(ranks, (levels[end-2], levels[end-2])) - n >= 2 && push!(ranks, (levels[end-1], levels[end-1])) - n >= 1 && push!(ranks, (levels[end], levels[end])) - return ranks - end - - # Source key -> directory name for figures. - _source_dirname(key::String) = key == "merged" ? "unfiltered" : key - - function _generate_taxa_charts(df::DataFrame, scols::Vector{String}, - figures_dir::String, top_n::Int, - subtitle::Union{Nothing,String}, - source_key::String; - ranks, rank_order::Vector{String}) - src_dir = _source_dirname(source_key) - for (rankdir, rank) in ranks - dir = joinpath(figures_dir, src_dir, rankdir) - mkpath(dir) - taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar.pdf"); - top_n, rank, relative=true, subtitle, rank_order) - taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar_absolute.pdf"); - top_n, rank, relative=false, subtitle, rank_order) - end - end - - ## Top-taxa report section - function _top_taxa_section(df::DataFrame, scols::Vector{String}, - rank_name::String, rank_col::String; n::Int=20) - sym = Symbol(rank_col) - hasproperty(df, sym) || return "" - - # Label each row, sum counts across all samples. - labels = String[] - totals = Float64[] - for row in eachrow(df) - val = row[sym] - label = (ismissing(val) || isempty(strip(string(val)))) ? "Unclassified" : string(val) - push!(labels, label) - push!(totals, sum(col -> begin - v = row[Symbol(col)] - ismissing(v) ? 0.0 : Float64(v) - end, scols)) - end - - # Aggregate by label. - agg = Dict{String, Float64}() - for (l, t) in zip(labels, totals) - agg[l] = get(agg, l, 0.0) + t - end - - sorted = sort(collect(agg); by=last, rev=true) - grand_total = sum(last, sorted; init=0.0) - - buf = IOBuffer() - print(buf, rpad("Rank", 4), rpad(rank_name, 30), rpad("Reads", 14), "Percent\n") - for (i, (label, count)) in enumerate(sorted) - i > n && break - pct = grand_total > 0 ? round(100.0 * count / grand_total; digits=1) : 0.0 - print(buf, rpad(string(i), 4), rpad(label, 30), - rpad(string(Int(count)), 14), "$(pct)%\n") - end - return String(take!(buf)) - end - - ## Report generator (dual: filtered + merged) - function _generate_report(df::DataFrame, scols::Vector{String}, - stats_df, merged_df::DataFrame, - src_key::String, report_path::String, - run_name::String; - stats_key::String=src_key, - report_ranks::Vector{Tuple{String,String}}=[("Family","Family"),("Genus","Genus"),("Species","Species")]) - src_label = _source_label(src_key) - report_sections = Pair{String, String}[] - - # Pipeline statistics table. - if !isnothing(stats_df) - buf = IOBuffer() - _print_stats_table(buf, stats_df) - if "input" in names(stats_df) - total_input = sum(stats_df.input) - # Use the reads column matching this report's stats key. - target_col = "reads_" * stats_key - if target_col in names(stats_df) - total_final = sum(stats_df[!, target_col]) - pct = total_input > 0 ? round(100.0 * total_final / total_input; digits=1) : 0.0 - println(buf, "\nOverall retention (input -> $(src_label)): $(pct)%") - end - end - push!(report_sections, "Pipeline Statistics" => String(take!(buf))) - end - - # ASV summary. - merged_asvs = nrow(merged_df) - filter_asvs = nrow(df) - buf = IOBuffer() - println(buf, "Total ASVs (merged): $merged_asvs") - if src_key != "merged" - println(buf, "ASVs after filter: $filter_asvs") - end - push!(report_sections, "ASV Summary" => String(take!(buf))) - - # Alpha diversity table. - alpha = _compute_alpha(df, scols) - buf = IOBuffer() - _print_alpha_table(buf, alpha) - push!(report_sections, "Alpha Diversity" => String(take!(buf))) - - # Top-20 taxa tables. - for (rank_name, rank_col) in report_ranks - section = _top_taxa_section(df, scols, rank_name, rank_col; n=20) - !isempty(section) && push!(report_sections, "Top 20 $(rank_name) ($(src_label))" => section) - end - - _write_report(report_path, - "Analysis Report: $run_name\n Source: $src_label", - report_sections) - end - - ## Text report writer - function _write_report(path::String, title::String, - sections::Vector{Pair{String, String}}) - open(path, "w") do io - sep = "=" ^ 64 - println(io, sep) - println(io, " ", title) - println(io, " Generated: ", Dates.format(now(), "yyyy-mm-dd HH:MM:SS")) - println(io, sep) - for (heading, body) in sections - println(io) - println(io, "--- ", heading, " ---") - println(io, body) - end - end - @info "Written: $path" - end - - ## Source table selection - # Pick the primary analysis source from a MergedTables: the first - # filter CSV (alphabetically) if any filters exist, else "merged". - function _source_key(merged::MergedTables) - "merged" - end - - ## Priority-based filter composition - # Assign each ASV to the first matching filter (by pipeline.yml order). - # Returns (filter_totals, sample_cols) or (nothing, nothing). - function _priority_filter_composition(merged::MergedTables, cache::_CSVCache, - db_meta::DatabaseMeta) - merged_csv = merged.tables["merged"] - isfile(merged_csv) || return nothing, nothing - merged_df, merged_scols = _cached_read(cache, merged_csv, db_meta) - isempty(merged_scols) && return nothing, nothing - - # Use filter_order for priority; fall back to sorted keys. - filter_keys = !isempty(merged.filter_order) ? merged.filter_order : - sort([k for k in keys(merged.tables) if k != "merged"]) - isempty(filter_keys) && return nothing, nothing - - # Per-sample totals from merged (unfiltered). - merged_totals = Float64[ - sum(v -> ismissing(v) ? 0.0 : Float64(v), merged_df[!, s]) - for s in merged_scols - ] - - # Load filter DataFrames. - filter_dfs = Dict{String, DataFrame}() - for fk in filter_keys - haskey(merged.tables, fk) || continue - fpath = merged.tables[fk] - (isfile(fpath) && filesize(fpath) > 0) || continue - filter_dfs[fk] = first(_cached_read(cache, fpath, db_meta)) - end - - # Track which SeqNames have been claimed. - claimed = Set{String}() - filter_totals = Dict{String, Vector{Float64}}() - - for fk in filter_keys - haskey(filter_dfs, fk) || continue - fdf = filter_dfs[fk] - totals = zeros(Float64, length(merged_scols)) - for row in eachrow(fdf) - seq = string(row.SeqName) - seq in claimed && continue - push!(claimed, seq) - for (i, s) in enumerate(merged_scols) - v = hasproperty(row, Symbol(s)) ? row[Symbol(s)] : missing - totals[i] += ismissing(v) ? 0.0 : Float64(v) - end - end - filter_totals[fk] = totals - end - - # Unclassified = merged totals minus all attributed. - attributed = zeros(Float64, length(merged_scols)) - for vals in values(filter_totals) - attributed .+= vals - end - unclassified = max.(merged_totals .- attributed, 0.0) - if any(>(0), unclassified) - filter_totals["Unclassified"] = unclassified - end - - return filter_totals, merged_scols - end - - ## Count-matrix builders - # Extract a samples x features count matrix from a single DataFrame. - # Rows = samples (one per element of `scols`), columns = ASV rows. - function _counts_matrix(df::DataFrame, scols::Vector{String}) - n_samples = length(scols) - n_features = nrow(df) - mat = zeros(Float64, n_samples, n_features) - for (j, row) in enumerate(eachrow(df)) - for (i, col) in enumerate(scols) - v = row[Symbol(col)] - mat[i, j] = ismissing(v) ? 0.0 : Float64(v) - end - end - return mat - end - - # Build a taxonomy-aggregated count matrix across multiple runs. - # - # ASV identifiers (seq1, seq2, ...) are local to each run and cannot - # be compared directly. This function aggregates counts to the - # lowest assigned taxonomic rank, producing a shared feature space - # suitable for between-run Bray-Curtis / NMDS. - # - # Returns (matrix, all_sample_names, taxon_labels). - function _build_combined_counts( - dfs::Vector{DataFrame}, - scols_per_df::Vector{Vector{String}}, - levels::Vector{String}, - ) - # taxon -> sample -> accumulated count - taxa_counts = Dict{String, Dict{String, Float64}}() - all_samples = String[] - - for (df, scols) in zip(dfs, scols_per_df) - append!(all_samples, scols) - for row in eachrow(df) - label = _lowest_rank_label(row, levels) - td = get!(taxa_counts, label, Dict{String, Float64}()) - for col in scols - v = row[Symbol(col)] - td[col] = get(td, col, 0.0) + (ismissing(v) ? 0.0 : Float64(v isa AbstractString ? parse(Float64, v) : v)) - end - end - end - - taxa_labels = sort(collect(keys(taxa_counts))) - n_samples = length(all_samples) - n_taxa = length(taxa_labels) - mat = zeros(Float64, n_samples, n_taxa) - - for (j, taxon) in enumerate(taxa_labels) - td = taxa_counts[taxon] - for (i, sample) in enumerate(all_samples) - mat[i, j] = get(td, sample, 0.0) - end - end - - return mat, all_samples, taxa_labels - end - - ## Alpha diversity helper - # Compute per-sample alpha diversity from a merged/filtered CSV. - function _compute_alpha(df::DataFrame, scols::Vector{String}) - out = DataFrame(sample=String[], richness=Int[], - shannon=Float64[], simpson=Float64[]) - for col in scols - counts = [ismissing(v) ? 0 : Int(v isa AbstractString ? parse(Int, v) : round(Int, Float64(v))) for v in df[!, col]] - push!(out, (col, richness(counts), shannon(counts), simpson(counts))) - end - return out - end - - # Metadata - """ - load_metadata(start_dir, study_dir) -> Union{DataFrame, Nothing} - - Walk from `start_dir` upward to `study_dir` (inclusive), returning the - first `metadata.csv` found as a DataFrame. Returns `nothing` when no - metadata file exists at any level. - """ - function load_metadata(start_dir::String, study_dir::String) - dir = abspath(start_dir) - stop = abspath(study_dir) - while true - csv = joinpath(dir, "metadata.csv") - isfile(csv) && return CSV.read(csv, DataFrame) - dir == stop && break - parent = dirname(dir) - parent == dir && break # filesystem root - dir = parent - end - return nothing - end - - # R: NMDS + PERMANOVA - # NMDS via vegan::metaMDS. - # `mat` is samples x features (community matrix). - # Returns (coords::Matrix{Float64}[nx2], stress::Float64). - # On failure returns a NaN-filled matrix and NaN stress. - function _run_nmds(mat::Matrix{Float64}, r_lock::ReentrantLock) - lock(r_lock) do - @rput mat - R""" - suppressPackageStartupMessages(library(vegan)) - set.seed(42) - nmds_res <- tryCatch( - metaMDS(mat, distance = "bray", k = 2, trymax = 200, - autotransform = FALSE, trace = 0), - error = function(e) NULL - ) - if (!is.null(nmds_res)) { - nmds_coords <- nmds_res$points - nmds_stress <- nmds_res$stress - } else { - nmds_coords <- matrix(NA_real_, nrow = nrow(mat), ncol = 2) - nmds_stress <- NA_real_ - } - """ - coords = rcopy(R"nmds_coords")::Matrix{Float64} - stress = rcopy(R"nmds_stress")::Float64 - return coords, stress - end - end - - # PERMANOVA via vegan::adonis2. - # Returns the captured text output, or `nothing` on failure. - function _run_permanova(mat::Matrix{Float64}, metadata::DataFrame, - r_lock::ReentrantLock) - covariates = [c for c in names(metadata) if lowercase(c) != "sample"] - isempty(covariates) && return nothing - formula_rhs = join(covariates, " + ") - - lock(r_lock) do - meta_r = copy(metadata) - @rput mat meta_r formula_rhs - R""" - suppressPackageStartupMessages(library(vegan)) - set.seed(42) - dist_mat <- vegdist(mat, method = "bray") - form <- as.formula(paste("dist_mat ~", formula_rhs)) - perm_res <- tryCatch( - adonis2(form, data = meta_r, permutations = 999), - error = function(e) NULL - ) - if (!is.null(perm_res)) { - perm_text <- paste(capture.output(print(perm_res)), collapse = "\n") - } else { - perm_text <- NA_character_ - } - """ - txt = rcopy(R"perm_text") - return (ismissing(txt) || txt == "NA") ? nothing : txt - end - end - - ## Pipeline summary CSV - function _pipeline_summary(project::ProjectCtx, merged::MergedTables, - db_meta::DatabaseMeta) - analysis_dir = joinpath(project.dir, "analysis") - summary_path = joinpath(analysis_dir, "pipeline_summary.csv") - - # Read DADA2's pipeline_stats.csv. - # R's write.csv writes row names as the first (unnamed) column. - stats_csv = joinpath(project.dir, "dada2", "Tables", "pipeline_stats.csv") - if !isfile(stats_csv) - @warn "pipeline_stats.csv not found at $stats_csv - skipping pipeline_summary" - return nothing - end - - stats = CSV.read(stats_csv, DataFrame) - first_col = names(stats)[1] - if first_col != "sample" - rename!(stats, first_col => "sample") - end - - # Add per-sample read totals from each merged table. - # Stems in pipeline_stats (e.g. "JIN-Nu-ves55") must be matched to - # full FASTQ column names (e.g. "JIN-Nu-ves55_R1_filt.fastq.gz"). - for key in sort(collect(keys(merged.tables))) - csv_path = merged.tables[key] - isfile(csv_path) || continue - df = CSV.read(csv_path, DataFrame) - scols = _sample_cols(df, db_meta) - - totals = Dict{String, Int}() - for col in scols - totals[col] = sum(v -> ismissing(v) ? 0 : Int(v), df[!, col]; init=0) - end - - # Match stems to full column names. - stem_map = _stem_to_colname(stats.sample, scols) - - # Prefix with "reads_" to avoid colliding with DADA2's own columns. - col_name = "reads_" * key - stats[!, col_name] = [get(totals, get(stem_map, String(s), ""), 0) - for s in stats.sample] - end - - mkpath(analysis_dir) - CSV.write(summary_path, stats) - @info "Written: $summary_path" - log_written(project, summary_path) - return stats - end - - ## Level 1 - Per-run analysis - """ - analyse_run(project, merged, asvs, db_meta; plot_lock=nothing) - - Produce per-run analysis outputs under `{project.dir}/analysis/`. - When `plot_lock` is provided, CairoMakie calls are serialized behind - it so multiple runs can prepare data in parallel. - """ - function analyse_run(project::ProjectCtx, merged::MergedTables, - asvs::ASVResult, db_meta::DatabaseMeta; - plot_lock::Union{Nothing,ReentrantLock}=nothing) - analysis_dir = joinpath(project.dir, "analysis") - figures_dir = joinpath(analysis_dir, "Figures") - merged_csv = merged.tables["merged"] - - filter_keys = sort([k for k in keys(merged.tables) if k != "merged"]) - has_filters = !isempty(filter_keys) - all_source_keys = vcat(filter_keys, ["merged"]) - - # Output paths. - summary_path = joinpath(analysis_dir, "pipeline_summary.csv") - stages_pdf = joinpath(figures_dir, "pipeline_stages.pdf") - alpha_pdf = joinpath(figures_dir, "alpha_diversity.pdf") - filter_pdf = joinpath(figures_dir, "filter_composition.pdf") - - # Read analysis config from the cascade (needed for skip guard and charts). - config_path = write_run_config(project) - analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) - taxa_cfg = get(analysis_cfg, "taxa_bar", Dict()) - top_n = get(taxa_cfg, "top_n", 15) - taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) - report_ranks = _report_ranks(db_meta.levels, analysis_cfg) - - required_outputs = String[summary_path, stages_pdf, alpha_pdf] - has_filters && push!(required_outputs, filter_pdf) - for method in _TAX_METHODS - msrc_keys = _method_source_keys(merged, method) - for src in msrc_keys - sd = _method_source_dirname(src) - for (rankdir, _) in taxa_ranks - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) - end - end - for src in msrc_keys - push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) - end - end - - # skip guard - merged_mtime = isfile(merged_csv) ? mtime(merged_csv) : time() - if all(isfile, required_outputs) && - all(f -> mtime(f) > merged_mtime, required_outputs) - @info "Skipping analyse_run: outputs up to date in $analysis_dir" - return - end - - mkpath(figures_dir) - - # Helper: serialize CairoMakie calls if plot_lock is provided. - _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) - - # CSV cache - cache = _CSVCache() - - src_key = _source_key(merged) - src_label = _source_label(src_key) - subtitle = "Source: $src_label" - - # pipeline summary - stats_df = _pipeline_summary(project, merged, db_meta) - - # pipeline stages plot - if !isnothing(stats_df) - _plot() do - pipeline_stats_plot(stats_df, stages_pdf; subtitle) - end - @info "Written: $stages_pdf" - log_written(project, stages_pdf) - end - - # taxa bar charts for all sources at multiple ranks (dual method) - for method in _TAX_METHODS - msrc_keys = _method_source_keys(merged, method) - for src in msrc_keys - # For dada2 method with "merged" key, use merged.csv; - # for _dada2 suffixed keys, use the corresponding CSV. - csv_key = src - src_csv_path = get(merged.tables, csv_key, "") - if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 - # For "merged" key under dada2 method, fall back to merged.csv - src == "merged" || continue - src_csv_path = merged.tables["merged"] - (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue - end - raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) - isempty(src_scols) && continue - # Skip dada2 method if no dada2 taxonomy columns - method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue - view_df = _method_df(raw_df, method, db_meta.levels) - sd = _method_source_dirname(src) - src_sub = "Source: $sd ($method)" - method_fig_dir = joinpath(figures_dir, method) - _plot() do - _generate_taxa_charts(view_df, src_scols, method_fig_dir, - top_n, src_sub, src == "merged" ? "merged" : - (endswith(src, "_dada2") ? src[1:end-6] : src); - ranks=taxa_ranks, rank_order=db_meta.levels) - end - end - end - - # filter composition (priority-based) - if has_filters - filter_totals, comp_scols = _priority_filter_composition(merged, cache, db_meta) - if !isnothing(filter_totals) && !isempty(filter_totals) - _plot() do - filter_composition_plot(filter_totals, comp_scols, filter_pdf; - subtitle, colour_overrides=merged.filter_colours) - end - @info "Written: $filter_pdf" - log_written(project, filter_pdf) - end - end - - # alpha diversity (from primary source) - primary_csv = merged.tables[src_key] - if isfile(primary_csv) && filesize(primary_csv) > 0 - prim_df, prim_scols = _cached_read(cache, primary_csv, db_meta) - if !isempty(prim_scols) - alpha = _compute_alpha(prim_df, prim_scols) - _plot() do - alpha_diversity_plot(alpha, alpha_pdf; subtitle) - end - @info "Written: $alpha_pdf" - log_written(project, alpha_pdf) - end - end - - # analysis reports (dual method, one per source per method) - run_name = basename(project.dir) - merged_df_full, _ = _cached_read(cache, merged_csv, db_meta) - for method in _TAX_METHODS - msrc_keys = _method_source_keys(merged, method) - for src in msrc_keys - csv_key = src - src_csv_path = get(merged.tables, csv_key, "") - if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 - src == "merged" || continue - src_csv_path = merged.tables["merged"] - (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue - end - raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) - isempty(src_scols) && continue - method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue - view_df = _method_df(raw_df, method, db_meta.levels) - sd = _method_source_dirname(src) - report_path = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") - clean_src = src == "merged" ? "merged" : (endswith(src, "_dada2") ? src[1:end-6] : src) - _generate_report(view_df, src_scols, stats_df, merged_df_full, - clean_src, report_path, run_name; - stats_key=src, report_ranks=report_ranks) - log_written(project, report_path) - end - end - end - - # Format a pipeline stats DataFrame as a text table. - function _print_stats_table(io::IO, stats_df::DataFrame) - cols = names(stats_df) - # Header. - print(io, rpad("Sample", 20)) - for col in cols - col == "sample" && continue - print(io, rpad(col, 16)) - end - println(io) - # Rows. - for row in eachrow(stats_df) - print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) - for col in cols - col == "sample" && continue - v = row[Symbol(col)] - print(io, rpad(ismissing(v) ? "0" : string(Int(v)), 16)) - end - println(io) - end - end - - # Format an alpha diversity DataFrame as a text table. - function _print_alpha_table(io::IO, alpha_df::DataFrame) - print(io, rpad("Sample", 20)) - println(io, rpad("Richness", 12), rpad("Shannon", 12), "Simpson") - for row in eachrow(alpha_df) - print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) - print(io, rpad(string(row.richness), 12)) - print(io, rpad(string(round(row.shannon; digits=3)), 12)) - println(io, round(row.simpson; digits=3)) - end - end - - ## Level 2 - Per-group analysis - function _analyse_group(group_dir::String, - members::Vector{Tuple{ProjectCtx, MergedTables}}, - r_lock::ReentrantLock, - db_meta::DatabaseMeta; - plot_lock::Union{Nothing,ReentrantLock}=nothing) - analysis_dir = joinpath(group_dir, "analysis") - figures_dir = joinpath(analysis_dir, "Figures") - - alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") - nmds_pdf = joinpath(figures_dir, "nmds.pdf") - filter_pdf = joinpath(figures_dir, "filter_composition.pdf") - - # Helper: serialize CairoMakie calls if plot_lock is provided. - _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) - - # Determine source keys. - src_key = _source_key(members[1][2]) - src_label = _source_label(src_key) - subtitle = "Source: $src_label" - - # Derive taxa/report ranks from db_meta. - analysis_cfg = Dict() - try - config_path = write_run_config(members[1][1]) - analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) - catch; end - taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) - report_ranks = _report_ranks(db_meta.levels, analysis_cfg) - - filter_keys = sort([k for k in keys(members[1][2].tables) if k != "merged"]) - has_any_filters = any(pair -> length(pair[2].tables) > 1, members) - all_source_keys = vcat(filter_keys, ["merged"]) - - # skip guard - newest_merged = maximum( - mtime(m.tables["merged"]) - for (_, m) in members if isfile(m.tables["merged"]); - init=0.0 - ) - required_outputs = String[alpha_pdf, nmds_pdf] - has_any_filters && push!(required_outputs, filter_pdf) - for method in _TAX_METHODS - msrc_keys = _method_source_keys(members[1][2], method) - for src in msrc_keys - sd = _method_source_dirname(src) - for (rankdir, _) in taxa_ranks - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison.pdf")) - push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison_absolute.pdf")) - end - end - for src in msrc_keys - push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) - end - end - - if all(isfile, required_outputs) && - all(f -> mtime(f) > newest_merged, required_outputs) - @info "Skipping analyse_group: outputs up to date in $figures_dir" - return - end - - reset_log(group_dir) - mkpath(figures_dir) - - # CSV cache - cache = _CSVCache() - - # helper: collect per-run data for a given source key - function _collect_source_data(source::String) - dfs = DataFrame[] - scols = Vector{String}[] - names_ = String[] - labels = String[] - all_s = String[] - for (proj, merged) in members - haskey(merged.tables, source) || continue - csv = merged.tables[source] - isfile(csv) || continue - df, sc = _cached_read(cache, csv, db_meta) - isempty(sc) && continue - rn = basename(proj.dir) - push!(dfs, df); push!(scols, sc); push!(names_, rn) - append!(all_s, sc); append!(labels, fill(rn, length(sc))) - end - return dfs, scols, names_, labels, all_s - end - - # Primary source data (for NMDS, alpha, filter composition). - run_dfs, run_scols, run_names, run_labels, all_scols = - _collect_source_data(src_key) - isempty(run_dfs) && return - - # taxa bar & group comparison for ALL sources (dual method) - for method in _TAX_METHODS - msrc_keys = _method_source_keys(members[1][2], method) - for src in msrc_keys - s_dfs, s_scols, s_names, _, s_all = _collect_source_data( - endswith(src, "_dada2") ? src : src) - # For "merged" under dada2, use the same "merged" key - if isempty(s_dfs) && method == "dada2" && src == "merged" - s_dfs, s_scols, s_names, _, s_all = _collect_source_data("merged") - end - isempty(s_dfs) && continue - # Skip dada2 if no dada2 columns - method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue - view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] - combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) - sd = _method_source_dirname(src) - src_sub = "Source: $sd ($method)" - method_fig_dir = joinpath(figures_dir, method) - # Use cleaned source key for directory naming - clean_src = src == "merged" ? "merged" : - (endswith(src, "_dada2") ? src[1:end-6] : src) - _plot() do - _generate_taxa_charts(combined, s_all, method_fig_dir, 15, src_sub, clean_src; - ranks=taxa_ranks, rank_order=db_meta.levels) - for (rankdir, rank) in taxa_ranks - dir = joinpath(method_fig_dir, _source_dirname(clean_src), rankdir) - mkpath(dir) - group_comparison_chart(view_dfs, s_scols, s_names, - joinpath(dir, "group_comparison.pdf"); - top_n=15, rank, relative=true, - subtitle=src_sub, - rank_order=db_meta.levels) - group_comparison_chart(view_dfs, s_scols, s_names, - joinpath(dir, "group_comparison_absolute.pdf"); - top_n=15, rank, relative=false, - subtitle=src_sub, - rank_order=db_meta.levels) - end - end - end - end - - # group alpha diversity (boxplot by run) - all_alpha = DataFrame[] - alpha_labels = String[] - for (df, scols, rname) in zip(run_dfs, run_scols, run_names) - alpha = _compute_alpha(df, scols) - push!(all_alpha, alpha) - push!(alpha_labels, rname) - end - if !isempty(all_alpha) - _plot() do - alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) - end - @info "Written: $alpha_pdf" - log_written(group_dir, alpha_pdf) - end - - # group NMDS - mat, _, _ = _build_combined_counts(run_dfs, run_scols, db_meta.levels) - nmds_stress = NaN - if size(mat, 1) >= 3 - coords, stress = _run_nmds(mat, r_lock) - if !any(isnan, coords) - nmds_cfg = Dict() - try - config_path = write_run_config(members[1][1]) - nmds_cfg = get(get(YAML.load_file(config_path), "analysis", Dict()), - "nmds", Dict()) - catch; end - max_stress = get(nmds_cfg, "max_stress", 0.2) - if !isnan(stress) && stress > max_stress - @warn "NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress for group $(basename(group_dir))" - pipeline_log(group_dir, "WARN: NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress") - end - nmds_stress = stress - _plot() do - nmds_plot(coords, all_scols, nmds_pdf; - colour_by=run_labels, stress=stress, subtitle) - end - @info "Written: $nmds_pdf" - log_written(group_dir, nmds_pdf) - else - @warn "NMDS failed for group $(basename(group_dir))" - pipeline_log(group_dir, "WARN: NMDS failed for group $(basename(group_dir))") - end - else - @warn "Too few samples ($(size(mat, 1))) for NMDS in group $(basename(group_dir))" - end - - # group filter composition - if has_any_filters - group_filter_totals = Dict{String, Vector{Float64}}() - group_sample_names = String[] - group_filter_colours = Dict{String,String}() - for (proj, merged) in members - ft, scols_m = _priority_filter_composition(merged, cache, db_meta) - isnothing(ft) && continue - append!(group_sample_names, scols_m) - merge!(group_filter_colours, merged.filter_colours) - for (fk, vals) in ft - existing = get(group_filter_totals, fk, Float64[]) - group_filter_totals[fk] = vcat(existing, vals) - end - end - - if haskey(group_filter_totals, "Unclassified") && - !any(>(0), group_filter_totals["Unclassified"]) - delete!(group_filter_totals, "Unclassified") - end - - if !isempty(group_filter_totals) && !isempty(group_sample_names) - _plot() do - filter_composition_plot(group_filter_totals, group_sample_names, filter_pdf; - subtitle, colour_overrides=group_filter_colours) - end - @info "Written: $filter_pdf" - log_written(group_dir, filter_pdf) - end - end - - # analysis reports (dual method, per source, each with top-20 tables) - # Shared sections: Per-Run Summary + NMDS. - shared_sections = Pair{String, String}[] - - buf = IOBuffer() - print(buf, rpad("Run", 20)) - println(buf, rpad("Samples", 10), rpad("Mean richness", 16), - rpad("Mean Shannon", 16), "Mean Simpson") - for (adf, rname) in zip(all_alpha, alpha_labels) - n = nrow(adf) - mr = round(mean(adf.richness); digits=0) - ms = round(mean(adf.shannon); digits=3) - mp = round(mean(adf.simpson); digits=3) - if n > 1 - sr = round(std(adf.richness); digits=0) - ss = round(std(adf.shannon); digits=2) - sp = round(std(adf.simpson); digits=2) - print(buf, rpad(rname, 20)) - println(buf, rpad(string(n), 10), - rpad("$(Int(mr)) +/- $(Int(sr))", 16), - rpad("$ms +/- $ss", 16), - "$mp +/- $sp") - else - print(buf, rpad(rname, 20)) - println(buf, rpad(string(n), 10), - rpad(string(Int(mr)), 16), - rpad(string(ms), 16), - string(mp)) - end - end - push!(shared_sections, "Per-Run Summary" => String(take!(buf))) - - if !isnan(nmds_stress) - quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" - push!(shared_sections, "NMDS" => - "Stress: $(round(nmds_stress; digits=3)) ($quality)") - end - - # Write one report per source per method. - for method in _TAX_METHODS - msrc_keys = _method_source_keys(members[1][2], method) - for src in msrc_keys - s_dfs, s_scols, _, _, s_all = _collect_source_data( - endswith(src, "_dada2") ? src : src) - if isempty(s_dfs) && method == "dada2" && src == "merged" - s_dfs, s_scols, _, _, s_all = _collect_source_data("merged") - end - isempty(s_dfs) && continue - method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue - view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] - combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) - sd = _method_source_dirname(src) - sl = sd == "unfiltered" ? "unfiltered" : sd - - sections = copy(shared_sections) - for (rank_name, rank_col) in report_ranks - section = _top_taxa_section(combined, s_all, rank_name, rank_col; n=20) - !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) - end - - rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") - _write_report(rpath, - "Comparison Report\n Runs: $(join(run_names, ", "))\n Source: $sl ($method)", - sections) - log_written(group_dir, rpath) - end - end - end - - ## Level 3 - Study-level analysis - function _analyse_study_level(study_dir::String, - valid::Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}, - r_lock::ReentrantLock, - default_db_meta::DatabaseMeta; - plot_lock::Union{Nothing,ReentrantLock}=nothing) - analysis_dir = joinpath(study_dir, "analysis") - figures_dir = joinpath(analysis_dir, "Figures") - nmds_pdf = joinpath(figures_dir, "nmds.pdf") - alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") - perm_txt = joinpath(analysis_dir, "permanova.txt") - - src_key = _source_key(valid[1][2]) - filter_keys = sort([k for k in keys(valid[1][2].tables) if k != "merged"]) - - # Derive taxa/report ranks from the first project's db_meta. - analysis_cfg = Dict() - try - config_path = write_run_config(valid[1][1]) - analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) - catch; end - taxa_ranks = _taxa_ranks(default_db_meta.levels, analysis_cfg) - report_ranks = _report_ranks(default_db_meta.levels, analysis_cfg) - - # skip guard - newest_merged = maximum( - mtime(m.tables["merged"]) - for (_, m) in valid if isfile(m.tables["merged"]); - init=0.0 - ) - required_outputs = [nmds_pdf, alpha_pdf] - for method in _TAX_METHODS - msrc_keys = _method_source_keys(valid[1][2], method) - for src in msrc_keys - push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) - end - end - if all(isfile, required_outputs) && - all(f -> mtime(f) > newest_merged, required_outputs) - @info "Skipping study-level analysis: outputs up to date" - return - end - - reset_log(study_dir) - mkpath(figures_dir) - - # Helper: serialize CairoMakie calls if plot_lock is provided. - _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) - - cache = _CSVCache() - - src_label = _source_label(src_key) - subtitle = "Source: $src_label" - - # group projects by parent dir - groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() - for (proj, merged, dm) in valid - gdir = dirname(proj.dir) - push!(get!(groups, gdir, []), (proj, merged, dm)) - end - - # collect data across all groups - run_dfs = DataFrame[] - run_scols = Vector{String}[] - all_scols = String[] - group_labels = String[] # group name per sample - run_labels = String[] # run name per sample (for shape) - group_names = String[] - - for (gdir, members) in groups - gname = basename(gdir) - push!(group_names, gname) - for (proj, merged, dm) in members - src_csv = merged.tables[_source_key(merged)] - isfile(src_csv) || continue - df, scols = _cached_read(cache, src_csv, dm) - isempty(scols) && continue - rname = basename(proj.dir) - push!(run_dfs, df) - push!(run_scols, scols) - append!(all_scols, scols) - append!(group_labels, fill(gname, length(scols))) - append!(run_labels, fill(rname, length(scols))) - end - end - - isempty(run_dfs) && return - - # study NMDS - mat, _, _ = _build_combined_counts(run_dfs, run_scols, default_db_meta.levels) - nmds_stress = NaN - if size(mat, 1) >= 3 - coords, stress = _run_nmds(mat, r_lock) - if !any(isnan, coords) - nmds_stress = stress - _plot() do - nmds_plot(coords, all_scols, nmds_pdf; - colour_by=group_labels, shape_by=run_labels, - colour_label="Group", shape_label="Run", - stress=stress, subtitle) - end - @info "Written: $nmds_pdf" - log_written(study_dir, nmds_pdf) - else - @warn "Study-level NMDS failed" - pipeline_log(study_dir, "WARN: Study-level NMDS failed") - end - else - @warn "Too few samples ($(size(mat, 1))) for study-level NMDS" - end - - # study alpha boxplot by group - all_alpha = DataFrame[] - alpha_labels = String[] - for (gdir, members) in groups - gname = basename(gdir) - combined_alpha = DataFrame(sample=String[], richness=Int[], - shannon=Float64[], simpson=Float64[]) - for (_, merged, dm) in members - src_csv = merged.tables[_source_key(merged)] - isfile(src_csv) || continue - df, scols = _cached_read(cache, src_csv, dm) - isempty(scols) && continue - append!(combined_alpha, _compute_alpha(df, scols)) - end - if nrow(combined_alpha) > 0 - push!(all_alpha, combined_alpha) - push!(alpha_labels, gname) - end - end - - if !isempty(all_alpha) - _plot() do - alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) - end - @info "Written: $alpha_pdf" - log_written(study_dir, alpha_pdf) - end - - # PERMANOVA (only when metadata.csv exists) - metadata = load_metadata(study_dir, study_dir) - if !isnothing(metadata) && size(mat, 1) >= 3 - sample_col = "sample" in names(metadata) ? "sample" : - "Sample" in names(metadata) ? "Sample" : nothing - if !isnothing(sample_col) - meta_idx = indexin(all_scols, String.(metadata[!, sample_col])) - if all(!isnothing, meta_idx) - meta_matched = metadata[collect(meta_idx), :] - perm_result = _run_permanova(mat, meta_matched, r_lock) - if !isnothing(perm_result) - mkpath(dirname(perm_txt)) - write(perm_txt, perm_result) - @info "Written: $perm_txt" - log_written(study_dir, perm_txt) - end - else - @warn "Not all samples found in metadata.csv - skipping PERMANOVA" - end - else - @warn "metadata.csv has no 'sample' column - skipping PERMANOVA" - end - end - - # dual analysis reports - report_sections = Pair{String, String}[] - - # Group overview. - buf = IOBuffer() - println(buf, "Groups: ", join(group_names, ", ")) - println(buf, "Total samples: ", length(all_scols)) - push!(report_sections, "Overview" => String(take!(buf))) - - # NMDS info. - if !isnan(nmds_stress) - quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" - push!(report_sections, "NMDS" => - "Stress: $(round(nmds_stress; digits=3)) ($quality)") - end - - # Alpha diversity by group. - if !isempty(all_alpha) - buf = IOBuffer() - print(buf, rpad("Group", 20)) - println(buf, rpad("Samples", 10), rpad("Mean richness", 16), - rpad("Mean Shannon", 16), "Mean Simpson") - for (adf, gname) in zip(all_alpha, alpha_labels) - n = nrow(adf) - mr = round(mean(adf.richness); digits=0) - ms = round(mean(adf.shannon); digits=3) - mp = round(mean(adf.simpson); digits=3) - print(buf, rpad(gname, 20)) - if n > 1 - sr = round(std(adf.richness); digits=0) - ss = round(std(adf.shannon); digits=2) - sp = round(std(adf.simpson); digits=2) - println(buf, rpad(string(n), 10), - rpad("$(Int(mr)) +/- $(Int(sr))", 16), - rpad("$ms +/- $ss", 16), - "$mp +/- $sp") - else - println(buf, rpad(string(n), 10), - rpad(string(Int(mr)), 16), - rpad(string(ms), 16), - string(mp)) - end - end - push!(report_sections, "Alpha Diversity by Group" => String(take!(buf))) - end - - # Write one report per source per method. - for method in _TAX_METHODS - msrc_keys = _method_source_keys(valid[1][2], method) - for src in msrc_keys - src_run_dfs = DataFrame[] - src_all_scols = String[] - # Determine the table key to look up - table_key = src - for (gdir_inner, members_inner) in groups - for (proj, merged, dm) in members_inner - csv = get(merged.tables, table_key, "") - # For "merged" under dada2, fall back to "merged" - if isempty(csv) && method == "dada2" && src == "merged" - csv = get(merged.tables, "merged", "") - end - (isempty(csv) || !isfile(csv)) && continue - df, sc = _cached_read(cache, csv, dm) - isempty(sc) && continue - push!(src_run_dfs, df) - append!(src_all_scols, sc) - end - end - isempty(src_run_dfs) && continue - method == "dada2" && !_has_dada2(src_run_dfs[1], default_db_meta.levels) && continue - - view_dfs = [_method_df(df, method, default_db_meta.levels) for df in src_run_dfs] - combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) - sd = _method_source_dirname(src) - sl = sd == "unfiltered" ? "unfiltered" : sd - sections = copy(report_sections) - for (rank_name, rank_col) in report_ranks - section = _top_taxa_section(combined, src_all_scols, rank_name, rank_col; n=20) - !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) - end - - rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") - _write_report(rpath, - "Study Report\n Source: $sl ($method)", - sections) - log_written(study_dir, rpath) - end - end - end - - # Public: analyse_study (entry point from main.jl) - """ - analyse_study(projects, merged_results) - - Run group-level and study-level analysis after the per-run `@threads` - loop has completed. - - Groups projects by `dirname(project.dir)` (= group directory). - Calls `_analyse_group` for each group with ≥2 runs, then - `_analyse_study_level` when there are ≥2 groups. - """ - function analyse_study(projects::Vector{ProjectCtx}, - merged_results::Vector{<:Union{MergedTables, Nothing}}, - db_metas::Vector{DatabaseMeta}; - plot_lock::Union{Nothing,ReentrantLock}=nothing) - isempty(projects) && return - - r_lock = ReentrantLock() - study_dir = projects[1].study_dir - - # Pair projects with results and db_metas, filtering out failures. - valid = Tuple{ProjectCtx, MergedTables, DatabaseMeta}[ - (projects[i], merged_results[i], db_metas[i]) - for i in eachindex(projects) - if i <= length(merged_results) && !isnothing(merged_results[i]) - ] - isempty(valid) && return - - # Group by parent directory. - groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() - for (proj, merged, dm) in valid - gdir = dirname(proj.dir) - push!(get!(groups, gdir, []), (proj, merged, dm)) - end - - # Per-group analysis (skipped for single-run groups). - # Each group uses the db_meta of its first member (all runs in a group share a DB). - for (gdir, members) in groups - if length(members) > 1 - group_db_meta = members[1][3] - group_members = [(p, m) for (p, m, _) in members] - _analyse_group(gdir, group_members, r_lock, group_db_meta; plot_lock) - end - end - - # Study-level analysis (only meaningful with ≥2 genuine groups). - if length(groups) >= 2 - study_db_meta = valid[1][3] - _analyse_study_level(study_dir, valid, r_lock, study_db_meta; plot_lock) - end - end + include("analysis/helpers.jl") + include("analysis/stats.jl") + include("analysis/composition.jl") + include("analysis/report.jl") + include("analysis/run.jl") + include("analysis/group.jl") + include("analysis/study.jl") end diff --git a/src/analysis/composition.jl b/src/analysis/composition.jl new file mode 100644 index 0000000..3a82aca --- /dev/null +++ b/src/analysis/composition.jl @@ -0,0 +1,119 @@ + ## Source table selection + # Pick the primary analysis source from a MergedTables. + # Currently always "merged" (unfiltered); filter selection can be added here. + function _source_key(merged::MergedTables) + "merged" + end + + ## Priority-based filter composition + # Assign each ASV to the first matching filter (by pipeline.yml order). + # Returns (filter_totals, sample_cols) or (nothing, nothing). + function _priority_filter_composition(merged::MergedTables, cache::_CSVCache, + db_meta::DatabaseMeta) + merged_csv = merged.tables["merged"] + isfile(merged_csv) || return nothing, nothing + merged_df, merged_scols = _cached_read(cache, merged_csv, db_meta) + isempty(merged_scols) && return nothing, nothing + + # Use filter_order for priority; fall back to sorted keys. + filter_keys = !isempty(merged.filter_order) ? merged.filter_order : + sort([k for k in keys(merged.tables) if k != "merged"]) + isempty(filter_keys) && return nothing, nothing + + # Per-sample totals from merged (unfiltered). + merged_totals = Float64[ + sum(v -> ismissing(v) ? 0.0 : Float64(v), merged_df[!, s]) + for s in merged_scols + ] + + # Load filter DataFrames. + filter_dfs = Dict{String, DataFrame}() + for fk in filter_keys + haskey(merged.tables, fk) || continue + fpath = merged.tables[fk] + (isfile(fpath) && filesize(fpath) > 0) || continue + filter_dfs[fk] = first(_cached_read(cache, fpath, db_meta)) + end + + # Track which SeqNames have been claimed. + claimed = Set{String}() + filter_totals = Dict{String, Vector{Float64}}() + + for fk in filter_keys + haskey(filter_dfs, fk) || continue + fdf = filter_dfs[fk] + totals = zeros(Float64, length(merged_scols)) + for row in eachrow(fdf) + seq = string(row.SeqName) + seq in claimed && continue + push!(claimed, seq) + for (i, s) in enumerate(merged_scols) + v = hasproperty(row, Symbol(s)) ? row[Symbol(s)] : missing + totals[i] += ismissing(v) ? 0.0 : Float64(v) + end + end + filter_totals[fk] = totals + end + + # Unclassified = merged totals minus all attributed. + attributed = zeros(Float64, length(merged_scols)) + for vals in values(filter_totals) + attributed .+= vals + end + unclassified = max.(merged_totals .- attributed, 0.0) + if any(>(0), unclassified) + filter_totals["Unclassified"] = unclassified + end + + return filter_totals, merged_scols + end + + ## Pipeline summary CSV + function _pipeline_summary(project::ProjectCtx, merged::MergedTables, + db_meta::DatabaseMeta) + analysis_dir = joinpath(project.dir, "analysis") + summary_path = joinpath(analysis_dir, "pipeline_summary.csv") + + # Read DADA2's pipeline_stats.csv. + # R's write.csv writes row names as the first (unnamed) column. + stats_csv = joinpath(project.dir, "dada2", "Tables", "pipeline_stats.csv") + if !isfile(stats_csv) + @warn "pipeline_stats.csv not found at $stats_csv - skipping pipeline_summary" + return nothing + end + + stats = CSV.read(stats_csv, DataFrame) + first_col = names(stats)[1] + if first_col != "sample" + rename!(stats, first_col => "sample") + end + + # Add per-sample read totals from each merged table. + # Stems in pipeline_stats (e.g. "JIN-Nu-ves55") must be matched to + # full FASTQ column names (e.g. "JIN-Nu-ves55_R1_filt.fastq.gz"). + for key in sort(collect(keys(merged.tables))) + csv_path = merged.tables[key] + isfile(csv_path) || continue + df = CSV.read(csv_path, DataFrame) + scols = _sample_cols(df, db_meta) + + totals = Dict{String, Int}() + for col in scols + totals[col] = sum(v -> ismissing(v) ? 0 : Int(v), df[!, col]; init=0) + end + + # Match stems to full column names. + stem_map = _stem_to_colname(stats.sample, scols) + + # Prefix with "reads_" to avoid colliding with DADA2's own columns. + col_name = "reads_" * key + stats[!, col_name] = [get(totals, get(stem_map, String(s), ""), 0) + for s in stats.sample] + end + + mkpath(analysis_dir) + CSV.write(summary_path, stats) + @info "Written: $summary_path" + log_written(project, summary_path) + return stats + end diff --git a/src/analysis/group.jl b/src/analysis/group.jl new file mode 100644 index 0000000..c94f7d5 --- /dev/null +++ b/src/analysis/group.jl @@ -0,0 +1,285 @@ + ## Level 2 - Per-group analysis + function _analyse_group(group_dir::String, + members::Vector{Tuple{ProjectCtx, MergedTables}}, + r_lock::ReentrantLock, + db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(group_dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + + alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") + nmds_pdf = joinpath(figures_dir, "nmds.pdf") + filter_pdf = joinpath(figures_dir, "filter_composition.pdf") + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + # Determine source keys. + src_key = _source_key(members[1][2]) + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # Derive taxa/report ranks from db_meta. + analysis_cfg = Dict() + try + config_path = write_run_config(members[1][1]) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + catch; end + taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(db_meta.levels, analysis_cfg) + + filter_keys = sort([k for k in keys(members[1][2].tables) if k != "merged"]) + has_any_filters = any(pair -> length(pair[2].tables) > 1, members) + all_source_keys = vcat(filter_keys, ["merged"]) + + # skip guard + newest_merged = maximum( + mtime(m.tables["merged"]) + for (_, m) in members if isfile(m.tables["merged"]); + init=0.0 + ) + required_outputs = String[alpha_pdf, nmds_pdf] + has_any_filters && push!(required_outputs, filter_pdf) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + sd = _method_source_dirname(src) + for (rankdir, _) in taxa_ranks + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison_absolute.pdf")) + end + end + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + + if all(isfile, required_outputs) && + all(f -> mtime(f) > newest_merged, required_outputs) + @info "Skipping group analysis: outputs up to date in $group_dir" + return + end + + reset_log(group_dir) + mkpath(figures_dir) + + cache = _CSVCache() + + # helper: collect per-run data for a given source key + function _collect_source_data(source::String) + dfs = DataFrame[] + scols = Vector{String}[] + names_ = String[] + labels = String[] + all_s = String[] + for (proj, merged) in members + haskey(merged.tables, source) || continue + csv = merged.tables[source] + isfile(csv) || continue + df, sc = _cached_read(cache, csv, db_meta) + isempty(sc) && continue + rn = basename(proj.dir) + push!(dfs, df); push!(scols, sc); push!(names_, rn) + append!(all_s, sc); append!(labels, fill(rn, length(sc))) + end + return dfs, scols, names_, labels, all_s + end + + # Primary source data (for NMDS, alpha, filter composition). + run_dfs, run_scols, run_names, run_labels, all_scols = + _collect_source_data(src_key) + isempty(run_dfs) && return + + # taxa bar & group comparison for ALL sources (dual method) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + s_dfs, s_scols, s_names, _, s_all = _collect_source_data( + endswith(src, "_dada2") ? src : src) + # For "merged" under dada2, use the same "merged" key + if isempty(s_dfs) && method == "dada2" && src == "merged" + s_dfs, s_scols, s_names, _, s_all = _collect_source_data("merged") + end + isempty(s_dfs) && continue + # Skip dada2 if no dada2 columns + method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue + view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + _total_seqs(combined, s_all) == 0 && continue + sd = _method_source_dirname(src) + src_sub = "Source: $sd ($method)" + method_fig_dir = joinpath(figures_dir, method) + # Use cleaned source key for directory naming + clean_src = src == "merged" ? "merged" : + (endswith(src, "_dada2") ? src[1:end-6] : src) + _plot() do + _generate_taxa_charts(combined, s_all, method_fig_dir, 15, src_sub, clean_src; + ranks=taxa_ranks, rank_order=db_meta.levels) + for (rankdir, rank) in taxa_ranks + dir = joinpath(method_fig_dir, _source_dirname(clean_src), rankdir) + mkpath(dir) + group_comparison_chart(view_dfs, s_scols, s_names, + joinpath(dir, "group_comparison.pdf"); + top_n=15, rank, relative=true, + subtitle=src_sub, + rank_order=db_meta.levels) + group_comparison_chart(view_dfs, s_scols, s_names, + joinpath(dir, "group_comparison_absolute.pdf"); + top_n=15, rank, relative=false, + subtitle=src_sub, + rank_order=db_meta.levels) + end + end + end + end + + # group alpha diversity (boxplot by run) + all_alpha = DataFrame[] + alpha_labels = String[] + for (df, scols, rname) in zip(run_dfs, run_scols, run_names) + alpha = _compute_alpha(df, scols) + push!(all_alpha, alpha) + push!(alpha_labels, rname) + end + if !isempty(all_alpha) + _plot() do + alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(group_dir, alpha_pdf) + end + + # group NMDS + mat, _, _ = _build_combined_counts(run_dfs, run_scols, db_meta.levels) + nmds_stress = NaN + if size(mat, 1) >= 3 + coords, stress = _run_nmds(mat, r_lock) + if !any(isnan, coords) + nmds_cfg = Dict() + try + config_path = write_run_config(members[1][1]) + nmds_cfg = get(get(YAML.load_file(config_path), "analysis", Dict()), + "nmds", Dict()) + catch; end + max_stress = get(nmds_cfg, "max_stress", 0.2) + if !isnan(stress) && stress > max_stress + @warn "NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress for group $(basename(group_dir))" + pipeline_log(group_dir, "WARN: NMDS stress $(round(stress; digits=3)) exceeds threshold $max_stress") + end + nmds_stress = stress + _plot() do + nmds_plot(coords, all_scols, nmds_pdf; + colour_by=run_labels, stress=stress, subtitle) + end + @info "Written: $nmds_pdf" + log_written(group_dir, nmds_pdf) + else + @warn "NMDS failed for group $(basename(group_dir))" + pipeline_log(group_dir, "WARN: NMDS failed for group $(basename(group_dir))") + end + else + @warn "Too few samples ($(size(mat, 1))) for NMDS in group $(basename(group_dir))" + end + + # group filter composition + if has_any_filters + group_filter_totals = Dict{String, Vector{Float64}}() + group_sample_names = String[] + group_filter_colours = Dict{String,String}() + for (proj, merged) in members + ft, scols_m = _priority_filter_composition(merged, cache, db_meta) + isnothing(ft) && continue + append!(group_sample_names, scols_m) + merge!(group_filter_colours, merged.filter_colours) + for (fk, vals) in ft + existing = get(group_filter_totals, fk, Float64[]) + group_filter_totals[fk] = vcat(existing, vals) + end + end + + if haskey(group_filter_totals, "Unclassified") && + !any(>(0), group_filter_totals["Unclassified"]) + delete!(group_filter_totals, "Unclassified") + end + + if !isempty(group_filter_totals) && !isempty(group_sample_names) + _plot() do + filter_composition_plot(group_filter_totals, group_sample_names, filter_pdf; + subtitle, colour_overrides=group_filter_colours) + end + @info "Written: $filter_pdf" + log_written(group_dir, filter_pdf) + end + end + + # analysis reports (dual method, per source, each with top-20 tables) + # Shared sections: Per-Run Summary + NMDS. + shared_sections = Pair{String, String}[] + + buf = IOBuffer() + print(buf, rpad("Run", 20)) + println(buf, rpad("Samples", 10), rpad("Mean richness", 16), + rpad("Mean Shannon", 16), "Mean Simpson") + for (adf, rname) in zip(all_alpha, alpha_labels) + n = nrow(adf) + mr = round(mean(adf.richness); digits=0) + ms = round(mean(adf.shannon); digits=3) + mp = round(mean(adf.simpson); digits=3) + if n > 1 + sr = round(std(adf.richness); digits=0) + ss = round(std(adf.shannon); digits=2) + sp = round(std(adf.simpson); digits=2) + print(buf, rpad(rname, 20)) + println(buf, rpad(string(n), 10), + rpad("$(Int(mr)) +/- $(Int(sr))", 16), + rpad("$ms +/- $ss", 16), + "$mp +/- $sp") + else + print(buf, rpad(rname, 20)) + println(buf, rpad(string(n), 10), + rpad(string(Int(mr)), 16), + rpad(string(ms), 16), + string(mp)) + end + end + push!(shared_sections, "Per-Run Summary" => String(take!(buf))) + + if !isnan(nmds_stress) + quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" + push!(shared_sections, "NMDS" => + "Stress: $(round(nmds_stress; digits=3)) ($quality)") + end + + # Write one report per source per method. + for method in _TAX_METHODS + msrc_keys = _method_source_keys(members[1][2], method) + for src in msrc_keys + s_dfs, s_scols, _, _, s_all = _collect_source_data( + endswith(src, "_dada2") ? src : src) + if isempty(s_dfs) && method == "dada2" && src == "merged" + s_dfs, s_scols, _, _, s_all = _collect_source_data("merged") + end + isempty(s_dfs) && continue + method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue + view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + _total_seqs(combined, s_all) == 0 && continue + sd = _method_source_dirname(src) + sl = sd == "unfiltered" ? "unfiltered" : sd + + sections = copy(shared_sections) + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(combined, s_all, rank_name, rank_col; n=20) + !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) + end + + rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + _write_report(rpath, + "Comparison Report\n Runs: $(join(run_names, ", "))\n Source: $sl ($method)", + sections) + log_written(group_dir, rpath) + end + end + end diff --git a/src/analysis/helpers.jl b/src/analysis/helpers.jl new file mode 100644 index 0000000..549293b --- /dev/null +++ b/src/analysis/helpers.jl @@ -0,0 +1,186 @@ + ## Column identification + """ + _sample_cols(df, db_meta) -> Vector{String} + + Return column names that hold per-sample ASV counts by excluding known + taxonomy / metadata columns and any column ending with `_dada2` or + `_boot`. + """ + function _sample_cols(df::DataFrame, db_meta::DatabaseMeta) + filter(names(df)) do col + col ∉ db_meta.noncounts && + !endswith(col, "_dada2") && + !endswith(col, "_boot") && + !endswith(col, "_vsearch") && + !isempty(col) + end + end + + ## CSV cache + # Read a CSV once and reuse. Stores (DataFrame, sample_cols). + const _CSVCache = Dict{String, Tuple{DataFrame, Vector{String}}} + + function _cached_read(cache::_CSVCache, path::String, db_meta::DatabaseMeta) + haskey(cache, path) && return cache[path] + df = CSV.read(path, DataFrame) + scols = _sample_cols(df, db_meta) + # Ensure sample columns are numeric (DADA2 tax_counts.csv may write them as strings). + for col in scols + if eltype(df[!, col]) <: AbstractString || eltype(df[!, col]) == Union{Missing, String} + df[!, col] = [ismissing(v) ? missing : parse(Float64, v) for v in df[!, col]] + elseif !(eltype(df[!, col]) <: Union{Missing, Number}) + df[!, col] = passmissing(x -> Float64(x)).(df[!, col]) + end + end + cache[path] = (df, scols) + return df, scols + end + + # Taxonomy methods: vsearch uses standard column names, dada2 uses _dada2 suffix. + const _TAX_METHODS = ["vsearch", "dada2"] + + """ + _dada2_taxonomy_view(df, levels) -> DataFrame + + Return a copy where `_dada2` taxonomy columns are renamed to standard names + (vsearch columns get `_vsearch` suffix). Allows plotting/reporting code to + work unchanged on DADA2 taxonomy. + """ + function _dada2_taxonomy_view(df::DataFrame, levels::Vector{String}) + vdf = copy(df) + for rank in levels + dada2_col = rank * "_dada2" + dada2_col in names(vdf) || continue + if rank in names(vdf) + rename!(vdf, rank => rank * "_vsearch") + end + rename!(vdf, dada2_col => rank) + end + return vdf + end + + # Check whether a DataFrame has any DADA2 taxonomy columns. + _has_dada2(df::DataFrame, levels::Vector{String}) = any(c -> endswith(c, "_dada2") && + any(r -> startswith(c, r), levels), names(df)) + + # Union source keys across multiple MergedTables (for study/group levels where + # different runs may carry different filter sets, e.g. .pr2 vs .silva filters). + function _all_method_source_keys(mergeds::Vector{MergedTables}, method::String) + seen = Set{String}() + out = String[] + for m in mergeds + for k in _method_source_keys(m, method) + if k ∉ seen + push!(seen, k) + push!(out, k) + end + end + end + # Ensure "merged" is always last. + filter!(!=("merged"), out) + push!(out, "merged") + return out + end + + # For a given method, return the appropriate source keys from a MergedTables. + # vsearch: keys without _dada2 suffix (existing behaviour) + # dada2: for each filter stem, use "{stem}_dada2" key; for "merged" use "merged" (same CSV, both taxonomies) + function _method_source_keys(merged::MergedTables, method::String) + if method == "dada2" + keys_out = String[] + for k in sort(collect(keys(merged.tables))) + k == "merged" && continue + endswith(k, "_dada2") && push!(keys_out, k) + end + push!(keys_out, "merged") + return keys_out + else + return sort([k for k in keys(merged.tables) + if k != "merged" && !endswith(k, "_dada2")]) |> + ks -> vcat(ks, ["merged"]) + end + end + + # Get the CSV path and apply taxonomy view for a method. + function _method_df(raw_df::DataFrame, method::String, levels::Vector{String}) + method == "dada2" ? _dada2_taxonomy_view(raw_df, levels) : raw_df + end + + # Source dirname for method-qualified keys (strip _dada2 suffix for directory names). + _method_source_dirname(key::String) = _source_dirname( + endswith(key, "_dada2") ? key[1:end-6] : key) + + # Lowest assigned taxonomic rank for a row. + function _lowest_rank_label(row, levels::Vector{String}) + label = "Unclassified" + for rank in levels + if hasproperty(row, Symbol(rank)) + val = row[Symbol(rank)] + if !ismissing(val) && !isempty(strip(string(val))) + label = string(val) + end + end + end + return label + end + + ## Stem-to-column matching + # Match pipeline_stats sample stems to full FASTQ column names. + # Returns a Dict mapping each stem to the matching column name + # (or the stem itself if no match is found). + function _stem_to_colname(stems::AbstractVector, col_names::Vector{String}) + mapping = Dict{String, String}() + for stem in stems + s = String(stem) + matched = false + for col in col_names + if startswith(col, s * "_") || col == s + mapping[s] = col + matched = true + break + end + end + matched || (mapping[s] = s) + end + return mapping + end + + ## Source label helper + _source_label(key::String) = key == "merged" ? "unfiltered" : key + + ## Taxa rank iteration + # Derive taxa chart ranks from database levels and optional config override. + function _taxa_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) + configured = get(get(analysis_cfg, "taxa_bar", Dict()), "ranks", nothing) + if !isnothing(configured) && configured isa Vector + return Tuple{String, Union{String,Nothing}}[ + (lowercase(string(r)), string(r) == "asv" ? nothing : string(r)) for r in configured + ] + end + n = length(levels) + ranks = Tuple{String, Union{String,Nothing}}[("asv", nothing)] + n >= 1 && push!(ranks, (lowercase(levels[end]), levels[end])) + n >= 2 && push!(ranks, (lowercase(levels[end-1]), levels[end-1])) + return ranks + end + + # Derive report ranks from database levels and optional config override. + function _report_ranks(levels::Vector{String}, analysis_cfg::Dict=Dict()) + configured = get(get(analysis_cfg, "taxa_bar", Dict()), "report_ranks", nothing) + if !isnothing(configured) && configured isa Vector + return [(string(r), string(r)) for r in configured] + end + n = length(levels) + ranks = Tuple{String,String}[] + n >= 3 && push!(ranks, (levels[end-2], levels[end-2])) + n >= 2 && push!(ranks, (levels[end-1], levels[end-1])) + n >= 1 && push!(ranks, (levels[end], levels[end])) + return ranks + end + + # Source key -> directory name for figures. + _source_dirname(key::String) = key == "merged" ? "unfiltered" : key + + # Total sequence count across all sample columns. Used to skip empty filtered tables. + _total_seqs(df::DataFrame, scols::Vector{String}) = + sum(col -> sum(skipmissing(df[!, col]); init=0), scols; init=0) diff --git a/src/analysis/report.jl b/src/analysis/report.jl new file mode 100644 index 0000000..d962636 --- /dev/null +++ b/src/analysis/report.jl @@ -0,0 +1,160 @@ + function _generate_taxa_charts(df::DataFrame, scols::Vector{String}, + figures_dir::String, top_n::Int, + subtitle::Union{Nothing,String}, + source_key::String; + ranks, rank_order::Vector{String}) + src_dir = _source_dirname(source_key) + for (rankdir, rank) in ranks + dir = joinpath(figures_dir, src_dir, rankdir) + mkpath(dir) + taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar.pdf"); + top_n, rank, relative=true, subtitle, rank_order) + taxa_bar_chart(df, scols, joinpath(dir, "taxa_bar_absolute.pdf"); + top_n, rank, relative=false, subtitle, rank_order) + end + end + + ## Top-taxa report section + function _top_taxa_section(df::DataFrame, scols::Vector{String}, + rank_name::String, rank_col::String; n::Int=20) + sym = Symbol(rank_col) + hasproperty(df, sym) || return "" + + # Label each row, sum counts across all samples. + labels = String[] + totals = Float64[] + for row in eachrow(df) + val = row[sym] + label = (ismissing(val) || isempty(strip(string(val)))) ? "Unclassified" : string(val) + push!(labels, label) + push!(totals, sum(col -> begin + v = row[Symbol(col)] + ismissing(v) ? 0.0 : Float64(v) + end, scols)) + end + + # Aggregate by label. + agg = Dict{String, Float64}() + for (l, t) in zip(labels, totals) + agg[l] = get(agg, l, 0.0) + t + end + + sorted = sort(collect(agg); by=last, rev=true) + grand_total = sum(last, sorted; init=0.0) + + buf = IOBuffer() + print(buf, rpad("Rank", 4), rpad(rank_name, 30), rpad("Reads", 14), "Percent\n") + for (i, (label, count)) in enumerate(sorted) + i > n && break + pct = grand_total > 0 ? round(100.0 * count / grand_total; digits=1) : 0.0 + print(buf, rpad(string(i), 4), rpad(label, 30), + rpad(string(Int(count)), 14), "$(pct)%\n") + end + return String(take!(buf)) + end + + ## Report generator (dual: filtered + merged) + function _generate_report(df::DataFrame, scols::Vector{String}, + stats_df, merged_df::DataFrame, + src_key::String, report_path::String, + run_name::String; + stats_key::String=src_key, + report_ranks::Vector{Tuple{String,String}}=[("Family","Family"),("Genus","Genus"),("Species","Species")]) + src_label = _source_label(src_key) + report_sections = Pair{String, String}[] + + # Pipeline statistics table. + if !isnothing(stats_df) + buf = IOBuffer() + _print_stats_table(buf, stats_df) + if "input" in names(stats_df) + total_input = sum(stats_df.input) + # Use the reads column matching this report's stats key. + target_col = "reads_" * stats_key + if target_col in names(stats_df) + total_final = sum(stats_df[!, target_col]) + pct = total_input > 0 ? round(100.0 * total_final / total_input; digits=1) : 0.0 + println(buf, "\nOverall retention (input -> $(src_label)): $(pct)%") + end + end + push!(report_sections, "Pipeline Statistics" => String(take!(buf))) + end + + # ASV summary. + merged_asvs = nrow(merged_df) + filter_asvs = nrow(df) + buf = IOBuffer() + println(buf, "Total ASVs (merged): $merged_asvs") + if src_key != "merged" + println(buf, "ASVs after filter: $filter_asvs") + end + push!(report_sections, "ASV Summary" => String(take!(buf))) + + # Alpha diversity table. + alpha = _compute_alpha(df, scols) + buf = IOBuffer() + _print_alpha_table(buf, alpha) + push!(report_sections, "Alpha Diversity" => String(take!(buf))) + + # Top-20 taxa tables. + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(df, scols, rank_name, rank_col; n=20) + !isempty(section) && push!(report_sections, "Top 20 $(rank_name) ($(src_label))" => section) + end + + _write_report(report_path, + "Analysis Report: $run_name\n Source: $src_label", + report_sections) + end + + ## Text report writer + function _write_report(path::String, title::String, + sections::Vector{Pair{String, String}}) + open(path, "w") do io + sep = "=" ^ 64 + println(io, sep) + println(io, " ", title) + println(io, " Generated: ", Dates.format(now(), "yyyy-mm-dd HH:MM:SS")) + println(io, sep) + for (heading, body) in sections + println(io) + println(io, "--- ", heading, " ---") + println(io, body) + end + end + @info "Written: $path" + end + + # Format a pipeline stats DataFrame as a text table. + function _print_stats_table(io::IO, stats_df::DataFrame) + cols = names(stats_df) + # Header. + print(io, rpad("Sample", 20)) + for col in cols + col == "sample" && continue + print(io, rpad(col, 16)) + end + println(io) + # Rows. + for row in eachrow(stats_df) + print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) + for col in cols + col == "sample" && continue + v = row[Symbol(col)] + print(io, rpad(ismissing(v) ? "0" : string(Int(v)), 16)) + end + println(io) + end + end + + # Format an alpha diversity DataFrame as a text table. + function _print_alpha_table(io::IO, alpha_df::DataFrame) + print(io, rpad("Sample", 20)) + println(io, rpad("Richness", 12), rpad("Shannon", 12), "Simpson") + for row in eachrow(alpha_df) + print(io, rpad(PipelinePlots._display_name(String(row.sample)), 20)) + print(io, rpad(string(row.richness), 12)) + print(io, rpad(string(round(row.shannon; digits=3)), 12)) + println(io, round(row.simpson; digits=3)) + end + end diff --git a/src/analysis/run.jl b/src/analysis/run.jl new file mode 100644 index 0000000..3302899 --- /dev/null +++ b/src/analysis/run.jl @@ -0,0 +1,168 @@ + ## Level 1 - Per-run analysis + """ + analyse_run(project, merged, asvs, db_meta; plot_lock=nothing) + + Produce per-run analysis outputs under `{project.dir}/analysis/`. + When `plot_lock` is provided, CairoMakie calls are serialized behind + it so multiple runs can prepare data in parallel. + """ + function analyse_run(project::ProjectCtx, merged::MergedTables, + asvs::ASVResult, db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(project.dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + merged_csv = merged.tables["merged"] + + filter_keys = sort([k for k in keys(merged.tables) if k != "merged"]) + has_filters = !isempty(filter_keys) + all_source_keys = vcat(filter_keys, ["merged"]) + + # Output paths. + summary_path = joinpath(analysis_dir, "pipeline_summary.csv") + stages_pdf = joinpath(figures_dir, "pipeline_stages.pdf") + alpha_pdf = joinpath(figures_dir, "alpha_diversity.pdf") + filter_pdf = joinpath(figures_dir, "filter_composition.pdf") + + # Read analysis config from the cascade (needed for skip guard and charts). + config_path = write_run_config(project) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + taxa_cfg = get(analysis_cfg, "taxa_bar", Dict()) + top_n = get(taxa_cfg, "top_n", 15) + taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(db_meta.levels, analysis_cfg) + + required_outputs = String[summary_path, stages_pdf, alpha_pdf] + has_filters && push!(required_outputs, filter_pdf) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + sd = _method_source_dirname(src) + for (rankdir, _) in taxa_ranks + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) + push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) + end + end + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + + # skip guard + merged_mtime = isfile(merged_csv) ? mtime(merged_csv) : time() + if all(isfile, required_outputs) && + all(f -> mtime(f) > merged_mtime, required_outputs) + @info "Skipping analyse_run: outputs up to date in $analysis_dir" + return + end + + mkpath(figures_dir) + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + # CSV cache + cache = _CSVCache() + + src_key = _source_key(merged) + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # pipeline summary + stats_df = _pipeline_summary(project, merged, db_meta) + + # pipeline stages plot + if !isnothing(stats_df) + _plot() do + pipeline_stats_plot(stats_df, stages_pdf; subtitle) + end + @info "Written: $stages_pdf" + log_written(project, stages_pdf) + end + + # taxa bar charts for all sources at multiple ranks (dual method) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + # For dada2 method with "merged" key, use merged.csv; + # for _dada2 suffixed keys, use the corresponding CSV. + csv_key = src + src_csv_path = get(merged.tables, csv_key, "") + if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 + # For "merged" key under dada2 method, fall back to merged.csv + src == "merged" || continue + src_csv_path = merged.tables["merged"] + (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue + end + raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) + isempty(src_scols) && continue + # Skip dada2 method if no dada2 taxonomy columns + method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue + view_df = _method_df(raw_df, method, db_meta.levels) + _total_seqs(view_df, src_scols) == 0 && continue + sd = _method_source_dirname(src) + src_sub = "Source: $sd ($method)" + method_fig_dir = joinpath(figures_dir, method) + _plot() do + _generate_taxa_charts(view_df, src_scols, method_fig_dir, + top_n, src_sub, src == "merged" ? "merged" : + (endswith(src, "_dada2") ? src[1:end-6] : src); + ranks=taxa_ranks, rank_order=db_meta.levels) + end + end + end + + # filter composition (priority-based) + if has_filters + filter_totals, comp_scols = _priority_filter_composition(merged, cache, db_meta) + if !isnothing(filter_totals) && !isempty(filter_totals) + _plot() do + filter_composition_plot(filter_totals, comp_scols, filter_pdf; + subtitle, colour_overrides=merged.filter_colours) + end + @info "Written: $filter_pdf" + log_written(project, filter_pdf) + end + end + + # alpha diversity (from primary source) + primary_csv = merged.tables[src_key] + if isfile(primary_csv) && filesize(primary_csv) > 0 + prim_df, prim_scols = _cached_read(cache, primary_csv, db_meta) + if !isempty(prim_scols) + alpha = _compute_alpha(prim_df, prim_scols) + _plot() do + alpha_diversity_plot(alpha, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(project, alpha_pdf) + end + end + + # analysis reports (dual method, one per source per method) + run_name = basename(project.dir) + merged_df_full, _ = _cached_read(cache, merged_csv, db_meta) + for method in _TAX_METHODS + msrc_keys = _method_source_keys(merged, method) + for src in msrc_keys + csv_key = src + src_csv_path = get(merged.tables, csv_key, "") + if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 + src == "merged" || continue + src_csv_path = merged.tables["merged"] + (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue + end + raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) + isempty(src_scols) && continue + method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue + view_df = _method_df(raw_df, method, db_meta.levels) + _total_seqs(view_df, src_scols) == 0 && continue + sd = _method_source_dirname(src) + report_path = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + clean_src = src == "merged" ? "merged" : (endswith(src, "_dada2") ? src[1:end-6] : src) + _generate_report(view_df, src_scols, stats_df, merged_df_full, + clean_src, report_path, run_name; + stats_key=src, report_ranks=report_ranks) + log_written(project, report_path) + end + end + end diff --git a/src/analysis/stats.jl b/src/analysis/stats.jl new file mode 100644 index 0000000..87a9426 --- /dev/null +++ b/src/analysis/stats.jl @@ -0,0 +1,154 @@ + ## Count-matrix builders + # Extract a samples x features count matrix from a single DataFrame. + # Rows = samples (one per element of `scols`), columns = ASV rows. + function _counts_matrix(df::DataFrame, scols::Vector{String}) + n_samples = length(scols) + n_features = nrow(df) + mat = zeros(Float64, n_samples, n_features) + for (j, row) in enumerate(eachrow(df)) + for (i, col) in enumerate(scols) + v = row[Symbol(col)] + mat[i, j] = ismissing(v) ? 0.0 : Float64(v) + end + end + return mat + end + + # Build a taxonomy-aggregated count matrix across multiple runs. + # + # ASV identifiers (seq1, seq2, ...) are local to each run and cannot + # be compared directly. This function aggregates counts to the + # lowest assigned taxonomic rank, producing a shared feature space + # suitable for between-run Bray-Curtis / NMDS. + # + # Returns (matrix, all_sample_names, taxon_labels). + function _build_combined_counts( + dfs::Vector{DataFrame}, + scols_per_df::Vector{Vector{String}}, + levels::Vector{String}, + ) + # taxon -> sample -> accumulated count + taxa_counts = Dict{String, Dict{String, Float64}}() + all_samples = String[] + + for (df, scols) in zip(dfs, scols_per_df) + append!(all_samples, scols) + for row in eachrow(df) + label = _lowest_rank_label(row, levels) + td = get!(taxa_counts, label, Dict{String, Float64}()) + for col in scols + v = row[Symbol(col)] + td[col] = get(td, col, 0.0) + (ismissing(v) ? 0.0 : Float64(v isa AbstractString ? parse(Float64, v) : v)) + end + end + end + + taxa_labels = sort(collect(keys(taxa_counts))) + n_samples = length(all_samples) + n_taxa = length(taxa_labels) + mat = zeros(Float64, n_samples, n_taxa) + + for (j, taxon) in enumerate(taxa_labels) + td = taxa_counts[taxon] + for (i, sample) in enumerate(all_samples) + mat[i, j] = get(td, sample, 0.0) + end + end + + return mat, all_samples, taxa_labels + end + + ## Alpha diversity helper + # Compute per-sample alpha diversity from a merged/filtered CSV. + function _compute_alpha(df::DataFrame, scols::Vector{String}) + out = DataFrame(sample=String[], richness=Int[], + shannon=Float64[], simpson=Float64[]) + for col in scols + counts = [ismissing(v) ? 0 : Int(v isa AbstractString ? parse(Int, v) : round(Int, Float64(v))) for v in df[!, col]] + push!(out, (col, richness(counts), shannon(counts), simpson(counts))) + end + return out + end + + # Metadata + """ + load_metadata(start_dir, study_dir) -> Union{DataFrame, Nothing} + + Walk from `start_dir` upward to `study_dir` (inclusive), returning the + first `metadata.csv` found as a DataFrame. Returns `nothing` when no + metadata file exists at any level. + """ + function load_metadata(start_dir::String, study_dir::String) + dir = abspath(start_dir) + stop = abspath(study_dir) + while true + csv = joinpath(dir, "metadata.csv") + isfile(csv) && return CSV.read(csv, DataFrame) + dir == stop && break + parent = dirname(dir) + parent == dir && break # filesystem root + dir = parent + end + return nothing + end + + # R: NMDS + PERMANOVA + # NMDS via vegan::metaMDS. + # `mat` is samples x features (community matrix). + # Returns (coords::Matrix{Float64}[nx2], stress::Float64). + # On failure returns a NaN-filled matrix and NaN stress. + function _run_nmds(mat::Matrix{Float64}, r_lock::ReentrantLock) + lock(r_lock) do + @rput mat + R""" + suppressPackageStartupMessages(library(vegan)) + set.seed(42) + nmds_res <- tryCatch( + metaMDS(mat, distance = "bray", k = 2, trymax = 200, + autotransform = FALSE, trace = 0), + error = function(e) NULL + ) + if (!is.null(nmds_res)) { + nmds_coords <- nmds_res$points + nmds_stress <- nmds_res$stress + } else { + nmds_coords <- matrix(NA_real_, nrow = nrow(mat), ncol = 2) + nmds_stress <- NA_real_ + } + """ + coords = rcopy(R"nmds_coords")::Matrix{Float64} + stress = rcopy(R"nmds_stress")::Float64 + return coords, stress + end + end + + # PERMANOVA via vegan::adonis2. + # Returns the captured text output, or `nothing` on failure. + function _run_permanova(mat::Matrix{Float64}, metadata::DataFrame, + r_lock::ReentrantLock) + covariates = [c for c in names(metadata) if lowercase(c) != "sample"] + isempty(covariates) && return nothing + formula_rhs = join(covariates, " + ") + + lock(r_lock) do + meta_r = copy(metadata) + @rput mat meta_r formula_rhs + R""" + suppressPackageStartupMessages(library(vegan)) + set.seed(42) + dist_mat <- vegdist(mat, method = "bray") + form <- as.formula(paste("dist_mat ~", formula_rhs)) + perm_res <- tryCatch( + adonis2(form, data = meta_r, permutations = 999), + error = function(e) NULL + ) + if (!is.null(perm_res)) { + perm_text <- paste(capture.output(print(perm_res)), collapse = "\n") + } else { + perm_text <- NA_character_ + } + """ + txt = rcopy(R"perm_text") + return (ismissing(txt) || txt == "NA") ? nothing : txt + end + end diff --git a/src/analysis/study.jl b/src/analysis/study.jl new file mode 100644 index 0000000..35a1e55 --- /dev/null +++ b/src/analysis/study.jl @@ -0,0 +1,325 @@ + ## Level 3 - Study-level analysis + function _analyse_study_level(study_dir::String, + valid::Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}, + r_lock::ReentrantLock, + default_db_meta::DatabaseMeta; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + analysis_dir = joinpath(study_dir, "analysis") + figures_dir = joinpath(analysis_dir, "Figures") + nmds_pdf = joinpath(figures_dir, "nmds.pdf") + alpha_pdf = joinpath(figures_dir, "alpha_comparison.pdf") + perm_txt = joinpath(analysis_dir, "permanova.txt") + + src_key = _source_key(valid[1][2]) + filter_keys = sort([k for k in keys(valid[1][2].tables) if k != "merged"]) + + # Derive taxa/report ranks from the first project's db_meta. + analysis_cfg = Dict() + try + config_path = write_run_config(valid[1][1]) + analysis_cfg = get(YAML.load_file(config_path), "analysis", Dict()) + catch; end + taxa_ranks = _taxa_ranks(default_db_meta.levels, analysis_cfg) + report_ranks = _report_ranks(default_db_meta.levels, analysis_cfg) + + # skip guard + newest_merged = maximum( + mtime(m.tables["merged"]) + for (_, m) in valid if isfile(m.tables["merged"]); + init=0.0 + ) + all_mergeds = [m for (_, m, _) in valid] + required_outputs = [nmds_pdf, alpha_pdf] + for method in _TAX_METHODS + msrc_keys = _all_method_source_keys(all_mergeds, method) + for src in msrc_keys + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + end + end + if all(isfile, required_outputs) && + all(f -> mtime(f) > newest_merged, required_outputs) + @info "Skipping study-level analysis: outputs up to date" + return + end + + reset_log(study_dir) + mkpath(figures_dir) + + # Helper: serialize CairoMakie calls if plot_lock is provided. + _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) + + cache = _CSVCache() + + src_label = _source_label(src_key) + subtitle = "Source: $src_label" + + # group projects by parent dir + groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() + for (proj, merged, dm) in valid + gdir = dirname(proj.dir) + push!(get!(groups, gdir, []), (proj, merged, dm)) + end + + # collect data across all groups + run_dfs = DataFrame[] + run_scols = Vector{String}[] + all_scols = String[] + group_labels = String[] # group name per sample + run_labels = String[] # run name per sample (for shape) + group_names = String[] + + for (gdir, members) in groups + gname = basename(gdir) + push!(group_names, gname) + for (proj, merged, dm) in members + src_csv = merged.tables[_source_key(merged)] + isfile(src_csv) || continue + df, scols = _cached_read(cache, src_csv, dm) + isempty(scols) && continue + rname = basename(proj.dir) + push!(run_dfs, df) + push!(run_scols, scols) + append!(all_scols, scols) + append!(group_labels, fill(gname, length(scols))) + append!(run_labels, fill(rname, length(scols))) + end + end + + isempty(run_dfs) && return + + # study NMDS + mat, _, _ = _build_combined_counts(run_dfs, run_scols, default_db_meta.levels) + nmds_stress = NaN + if size(mat, 1) >= 3 + coords, stress = _run_nmds(mat, r_lock) + if !any(isnan, coords) + nmds_stress = stress + _plot() do + nmds_plot(coords, all_scols, nmds_pdf; + colour_by=group_labels, shape_by=run_labels, + colour_label="Group", shape_label="Run", + stress=stress, subtitle) + end + @info "Written: $nmds_pdf" + log_written(study_dir, nmds_pdf) + else + @warn "Study-level NMDS failed" + pipeline_log(study_dir, "WARN: Study-level NMDS failed") + end + else + @warn "Too few samples ($(size(mat, 1))) for study-level NMDS" + end + + # study alpha boxplot by group + all_alpha = DataFrame[] + alpha_labels = String[] + for (gdir, members) in groups + gname = basename(gdir) + combined_alpha = DataFrame(sample=String[], richness=Int[], + shannon=Float64[], simpson=Float64[]) + for (_, merged, dm) in members + src_csv = merged.tables[_source_key(merged)] + isfile(src_csv) || continue + df, scols = _cached_read(cache, src_csv, dm) + isempty(scols) && continue + append!(combined_alpha, _compute_alpha(df, scols)) + end + if nrow(combined_alpha) > 0 + push!(all_alpha, combined_alpha) + push!(alpha_labels, gname) + end + end + + if !isempty(all_alpha) + _plot() do + alpha_boxplot(all_alpha, alpha_labels, alpha_pdf; subtitle) + end + @info "Written: $alpha_pdf" + log_written(study_dir, alpha_pdf) + end + + # PERMANOVA — always runs using group and run as covariates (derived from the + # project structure). Optionally, place a metadata.csv in the study directory + # with a 'sample' column and extra covariate columns; those columns are merged + # in when every sample name in all_scols is unique (i.e. no cross-database + # duplication of the same biological sample name). + if size(mat, 1) >= 3 + # Base metadata: one row per entry in all_scols (may contain duplicates). + base_meta = DataFrame(sample=all_scols, group=group_labels, run=run_labels) + + # Optionally merge user-supplied extra covariates. + user_meta = load_metadata(study_dir, study_dir) + if !isnothing(user_meta) + sample_col = "sample" in names(user_meta) ? "sample" : + "Sample" in names(user_meta) ? "Sample" : nothing + if isnothing(sample_col) + @warn "metadata.csv has no 'sample' column — using group/run only for PERMANOVA" + elseif length(unique(all_scols)) < length(all_scols) + @warn "Duplicate sample names across runs — extra metadata.csv columns ignored; using group/run only" + else + extra_cols = [c for c in names(user_meta) if c != sample_col] + if !isempty(extra_cols) + lookup = Dict(String(r[sample_col]) => r for r in eachrow(user_meta)) + matched = [get(lookup, s, nothing) for s in all_scols] + if all(!isnothing, matched) + for col in extra_cols + base_meta[!, col] = [m[col] for m in matched] + end + else + @warn "Not all samples found in metadata.csv — extra columns ignored" + end + end + end + end + + perm_result = _run_permanova(mat, base_meta, r_lock) + if !isnothing(perm_result) + mkpath(dirname(perm_txt)) + write(perm_txt, perm_result) + @info "Written: $perm_txt" + log_written(study_dir, perm_txt) + end + end + + # dual analysis reports + report_sections = Pair{String, String}[] + + # Group overview. + buf = IOBuffer() + println(buf, "Groups: ", join(group_names, ", ")) + println(buf, "Total samples: ", length(all_scols)) + push!(report_sections, "Overview" => String(take!(buf))) + + # NMDS info. + if !isnan(nmds_stress) + quality = nmds_stress < 0.2 ? "Good: below 0.2 threshold" : "Poor: above 0.2 threshold" + push!(report_sections, "NMDS" => + "Stress: $(round(nmds_stress; digits=3)) ($quality)") + end + + # Alpha diversity by group. + if !isempty(all_alpha) + buf = IOBuffer() + print(buf, rpad("Group", 20)) + println(buf, rpad("Samples", 10), rpad("Mean richness", 16), + rpad("Mean Shannon", 16), "Mean Simpson") + for (adf, gname) in zip(all_alpha, alpha_labels) + n = nrow(adf) + mr = round(mean(adf.richness); digits=0) + ms = round(mean(adf.shannon); digits=3) + mp = round(mean(adf.simpson); digits=3) + print(buf, rpad(gname, 20)) + if n > 1 + sr = round(std(adf.richness); digits=0) + ss = round(std(adf.shannon); digits=2) + sp = round(std(adf.simpson); digits=2) + println(buf, rpad(string(n), 10), + rpad("$(Int(mr)) +/- $(Int(sr))", 16), + rpad("$ms +/- $ss", 16), + "$mp +/- $sp") + else + println(buf, rpad(string(n), 10), + rpad(string(Int(mr)), 16), + rpad(string(ms), 16), + string(mp)) + end + end + push!(report_sections, "Alpha Diversity by Group" => String(take!(buf))) + end + + # Write one report per source per method. + for method in _TAX_METHODS + msrc_keys = _all_method_source_keys(all_mergeds, method) + for src in msrc_keys + src_run_dfs = DataFrame[] + src_all_scols = String[] + # Determine the table key to look up + table_key = src + for (gdir_inner, members_inner) in groups + for (proj, merged, dm) in members_inner + csv = get(merged.tables, table_key, "") + # For "merged" under dada2, fall back to "merged" + if isempty(csv) && method == "dada2" && src == "merged" + csv = get(merged.tables, "merged", "") + end + (isempty(csv) || !isfile(csv)) && continue + df, sc = _cached_read(cache, csv, dm) + isempty(sc) && continue + push!(src_run_dfs, df) + append!(src_all_scols, sc) + end + end + isempty(src_run_dfs) && continue + method == "dada2" && !_has_dada2(src_run_dfs[1], default_db_meta.levels) && continue + + view_dfs = [_method_df(df, method, default_db_meta.levels) for df in src_run_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + _total_seqs(combined, src_all_scols) == 0 && continue + sd = _method_source_dirname(src) + sl = sd == "unfiltered" ? "unfiltered" : sd + sections = copy(report_sections) + for (rank_name, rank_col) in report_ranks + section = _top_taxa_section(combined, src_all_scols, rank_name, rank_col; n=20) + !isempty(section) && push!(sections, "Top 20 $(rank_name) ($(sl))" => section) + end + + rpath = joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt") + _write_report(rpath, + "Study Report\n Source: $sl ($method)", + sections) + log_written(study_dir, rpath) + end + end + end + + # Public: analyse_study (entry point from main.jl) + """ + analyse_study(projects, merged_results) + + Run group-level and study-level analysis after the per-run `@threads` + loop has completed. + + Groups projects by `dirname(project.dir)` (= group directory). + Calls `_analyse_group` for each group with ≥2 runs, then + `_analyse_study_level` when there are ≥2 groups. + """ + function analyse_study(projects::Vector{ProjectCtx}, + merged_results::Vector{<:Union{MergedTables, Nothing}}, + db_metas::Vector{DatabaseMeta}; + plot_lock::Union{Nothing,ReentrantLock}=nothing) + isempty(projects) && return + + r_lock = ReentrantLock() + study_dir = projects[1].study_dir + + # Pair projects with results and db_metas, filtering out failures. + valid = Tuple{ProjectCtx, MergedTables, DatabaseMeta}[ + (projects[i], merged_results[i], db_metas[i]) + for i in eachindex(projects) + if i <= length(merged_results) && !isnothing(merged_results[i]) + ] + isempty(valid) && return + + # Group by parent directory. + groups = Dict{String, Vector{Tuple{ProjectCtx, MergedTables, DatabaseMeta}}}() + for (proj, merged, dm) in valid + gdir = dirname(proj.dir) + push!(get!(groups, gdir, []), (proj, merged, dm)) + end + + # Per-group analysis (skipped for single-run groups). + # Each group uses the db_meta of its first member (all runs in a group share a DB). + for (gdir, members) in groups + if length(members) > 1 + group_db_meta = members[1][3] + group_members = [(p, m) for (p, m, _) in members] + _analyse_group(gdir, group_members, r_lock, group_db_meta; plot_lock) + end + end + + # Study-level analysis (only meaningful with ≥2 genuine groups). + if length(groups) >= 2 + study_db_meta = valid[1][3] + _analyse_study_level(study_dir, valid, r_lock, study_db_meta; plot_lock) + end + end diff --git a/src/call_tools.jl b/src/call_tools.jl index 581843d..bd484e4 100644 --- a/src/call_tools.jl +++ b/src/call_tools.jl @@ -49,6 +49,55 @@ export cutadapt, vsearch, multiqc, cdhit const _tools = load_tools() + ## Argument builders + # Build the optional-args string for cutadapt from named config keys + any + # user-supplied optional_args. Named keys cover the options most commonly tuned + # for metabarcoding; optional_args accepts anything else cutadapt supports. + function _cutadapt_optional_args(cfg::Dict)::String + parts = String[] + min_len = get(cfg, "min_length", 200) + push!(parts, "-m $min_len") + get(cfg, "discard_untrimmed", true) && push!(parts, "--discard-untrimmed") + cores = get(cfg, "cores", 0) + cores != 1 && push!(parts, "-j $cores") # -j 1 is default; 0 = auto + quality_cutoff = get(cfg, "quality_cutoff", nothing) + isnothing(quality_cutoff) || push!(parts, "-q $quality_cutoff") + error_rate = get(cfg, "error_rate", nothing) + isnothing(error_rate) || push!(parts, "-e $error_rate") + overlap = get(cfg, "overlap", nothing) + isnothing(overlap) || push!(parts, "-O $overlap") + extra = strip(get(cfg, "optional_args", "")) + isempty(extra) || push!(parts, extra) + join(parts, " ") + end + + # Build the --usearch_global args for vsearch from named config keys + optional_args. + function _vsearch_args(cfg::Dict)::String + parts = String[] + push!(parts, "--id $(get(cfg, "identity", 0.75))") + push!(parts, "--query_cov $(get(cfg, "query_cov", 0.8))") + maxaccepts = get(cfg, "maxaccepts", nothing) + isnothing(maxaccepts) || push!(parts, "--maxaccepts $maxaccepts") + maxrejects = get(cfg, "maxrejects", nothing) + isnothing(maxrejects) || push!(parts, "--maxrejects $maxrejects") + strand = get(cfg, "strand", nothing) + isnothing(strand) || push!(parts, "--strand $strand") + extra = strip(get(cfg, "optional_args", "")) + isempty(extra) || push!(parts, extra) + join(parts, " ") + end + + # Build the cd-hit-est args from named config keys + optional_args. + function _cdhit_args(cfg::Dict)::String + parts = String[] + push!(parts, "-c $(get(cfg, "identity", 0.97))") + threads = get(cfg, "threads", 0) + push!(parts, "-T $threads") + extra = strip(get(cfg, "optional_args", "")) + isempty(extra) || push!(parts, extra) + join(parts, " ") + end + ## cutadapt function get_primers(input, primers_path, seen_fwd::Vector{String}, seen_rev::Vector{String}) data = YAML.load_file(primers_path) @@ -208,28 +257,33 @@ export cutadapt, vsearch, multiqc, cdhit end function cutadapt(project::ProjectCtx; - optional_args::Union{String,Nothing} = nothing, cutadapt_bin = tool_bin("cutadapt")) - config_path = write_run_config(project) - primers_path = joinpath(project.config_dir, "primers.yml") - cfg = get(YAML.load_file(config_path), "cutadapt", Dict()) - primer_pairs = cfg["primer_pairs"] - optional_args = isnothing(optional_args) ? - get(cfg, "optional_args", "-m 200 --discard-untrimmed") : optional_args - cutadapt_dir = joinpath(project.dir, "cutadapt") - hash_file = joinpath(cutadapt_dir, "config.hash") + config_path = write_run_config(project) + primers_path = joinpath(project.config_dir, "primers.yml") + cfg = get(YAML.load_file(config_path), "cutadapt", Dict()) + primer_pairs = cfg["primer_pairs"] + built_args = _cutadapt_optional_args(cfg) + cutadapt_dir = joinpath(project.dir, "cutadapt") + hash_file = joinpath(cutadapt_dir, "config.hash") + + # Collect mtimes of all inputs: primers config + raw FASTQs. + raw_fastqs = filter(f -> endswith(f, ".fastq.gz"), readdir(project.data_dir)) + raw_mtimes = [mtime(joinpath(project.data_dir, f)) for f in raw_fastqs] + primers_mtime = mtime(primers_path) + input_mtime = isempty(raw_mtimes) ? primers_mtime : max(primers_mtime, maximum(raw_mtimes)) + if isdir(cutadapt_dir) trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) if !isempty(trimmed) && !_section_stale(config_path, "cutadapt", hash_file) && - all(f -> mtime(joinpath(cutadapt_dir, f)) > mtime(primers_path), trimmed) && - all(f -> filesize(joinpath(cutadapt_dir, f)) > 20, trimmed) # skip empty gzips + all(f -> mtime(joinpath(cutadapt_dir, f)) > input_mtime, trimmed) && + all(f -> filesize(joinpath(cutadapt_dir, f)) > 20, trimmed) @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" return TrimmedReads(cutadapt_dir) end end mkpath(cutadapt_dir) - run_cutadapt(get_primer_args(primer_pairs, primers_path), optional_args, + run_cutadapt(get_primer_args(primer_pairs, primers_path), built_args, project.data_dir, cutadapt_dir, cutadapt_bin) _write_section_hash(config_path, "cutadapt", hash_file) pipeline_log(project, "cutadapt complete") @@ -287,6 +341,21 @@ export cutadapt, vsearch, multiqc, cdhit @info "MultiQC complete. Output: $qc_dir Log: $multiqc_log" end + function multiqc(project::ProjectCtx; + fastqc_bin = tool_bin("fastqc"), + multiqc_bin = tool_bin("multiqc")) + config_path = write_run_config(project) + fqc_cfg = get(YAML.load_file(config_path), "fastqc", Dict()) + mqc_cfg = get(YAML.load_file(config_path), "multiqc", Dict()) + threads = get(fqc_cfg, "threads", 20) + fastqc_args = "-t $threads --extract --delete" + extra_fqc = strip(get(fqc_cfg, "optional_args", "")) + isempty(extra_fqc) || (fastqc_args *= " $extra_fqc") + multiqc_args = strip(get(mqc_cfg, "optional_args", "")) + multiqc(project.data_dir, joinpath(project.dir, "QC"); + fastqc_args, multiqc_args, fastqc_bin, multiqc_bin) + end + ## vsearch """ vsearch(fasta_in_dir, reference_database; optional_args = "--id 0.75 --query_cov 0.8", vsearch_bin = "vsearch") @@ -331,22 +400,20 @@ export cutadapt, vsearch, multiqc, cdhit end function vsearch(project::ProjectCtx, input::HasFasta, reference_database::String; - optional_args::Union{String,Nothing} = nothing, vsearch_bin = tool_bin("vsearch")) - config_path = write_run_config(project) - cfg = get(YAML.load_file(config_path), "vsearch", Dict()) - optional_args = isnothing(optional_args) ? - get(cfg, "optional_args", "--id 0.75 --query_cov 0.8") : optional_args - vsearch_dir = joinpath(project.dir, "vsearch") - tsv = joinpath(vsearch_dir, "taxonomy.tsv") - hash_file = joinpath(vsearch_dir, "config.hash") + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "vsearch", Dict()) + built_args = _vsearch_args(cfg) + vsearch_dir = joinpath(project.dir, "vsearch") + tsv = joinpath(vsearch_dir, "taxonomy.tsv") + hash_file = joinpath(vsearch_dir, "config.hash") if isfile(tsv) && !_section_stale(config_path, "vsearch", hash_file) && mtime(tsv) > mtime(input.fasta) @info "Skipping vsearch: $tsv up to date" return TaxonomyHits(tsv) end - vsearch(input.fasta, reference_database, vsearch_dir; optional_args, vsearch_bin) + vsearch(input.fasta, reference_database, vsearch_dir; optional_args=built_args, vsearch_bin) _write_section_hash(config_path, "vsearch", hash_file) pipeline_log(project, "VSEARCH: $(input.fasta) against $(basename(reference_database))") log_written(project, tsv) @@ -394,22 +461,20 @@ export cutadapt, vsearch, multiqc, cdhit end function cdhit(project::ProjectCtx, input::ASVResult; - optional_args::Union{String,Nothing} = nothing, cdhit_bin = tool_bin("cd_hit_est")) - config_path = write_run_config(project) - cfg = get(YAML.load_file(config_path), "cdhit", Dict()) - optional_args = isnothing(optional_args) ? - get(cfg, "optional_args", "-c 0.9") : optional_args - cdhit_dir = joinpath(project.dir, "cdhit") - new_fasta = joinpath(cdhit_dir, basename(input.fasta)) - hash_file = joinpath(cdhit_dir, "config.hash") + config_path = write_run_config(project) + cfg = get(YAML.load_file(config_path), "cdhit", Dict()) + built_args = _cdhit_args(cfg) + cdhit_dir = joinpath(project.dir, "cdhit") + new_fasta = joinpath(cdhit_dir, basename(input.fasta)) + hash_file = joinpath(cdhit_dir, "config.hash") if isfile(new_fasta) && !_section_stale(config_path, "cdhit", hash_file) && mtime(new_fasta) > mtime(input.fasta) @info "Skipping cdhit: $new_fasta up to date" return ASVResult(new_fasta, input.count_table, input.taxonomy) end - new_fasta = cdhit(input.fasta, cdhit_dir; optional_args, cdhit_bin) + new_fasta = cdhit(input.fasta, cdhit_dir; optional_args=built_args, cdhit_bin) _write_section_hash(config_path, "cdhit", hash_file) pipeline_log(project, "cd-hit-est complete") log_written(project, new_fasta) diff --git a/src/databases.jl b/src/databases.jl index 6351451..d556ce3 100644 --- a/src/databases.jl +++ b/src/databases.jl @@ -17,7 +17,7 @@ module Databases # This module is licensed under the GNU Affero General Public License version 3 (AGPLv3). import Downloads -using YAML, Logging, CodecZlib +using YAML, Logging using ..PipelineTypes export ensure_databases, resolve_db, make_db_meta @@ -166,101 +166,7 @@ export ensure_databases, resolve_db, make_db_meta log("[$key] Saved to: $cached") end - # Post-processing: reformat database if requested. - reformat = get(fmt_info, "reformat", nothing) - if !isnothing(reformat) && !isempty(string(reformat)) - cached = _apply_reformat(key, string(reformat), cached, db_dir; log) - end - return cached end - """ - Apply a named reformat step to a downloaded database file. - Returns the path to the (possibly new) reformatted file. - """ - function _apply_reformat(key, reformat, src_path, db_dir; log=msg->@info(msg)) - if reformat == "silva_vsearch" - return _reformat_silva_vsearch(key, src_path, db_dir; log) - else - @warn "[$key] Unknown reformat '$reformat' - skipping" - return src_path - end - end - - """ - Reformat a SILVA vsearch FASTA so taxonomy is embedded in the sequence ID. - - SILVA headers are `>Accession Kingdom;Phylum;...;Genus` (taxonomy after a space). - vsearch only captures the sequence ID (before the first space), so taxonomy is lost. - - This function rewrites headers to `>Accession;Kingdom;Phylum;...;Genus` with spaces - within taxon names replaced by underscores, so vsearch returns the full taxonomy - string in the `target` field. - - The reformatted file is cached alongside the original; the original is kept intact. - """ - function _reformat_silva_vsearch(key, src_path, db_dir; log=msg->@info(msg)) - # Derive output filename from source - src_base = basename(src_path) - # Strip .gz if present to insert _reformatted before the extension - if endswith(src_base, ".fasta.gz") - out_base = src_base[1:end-9] * "_reformatted.fasta.gz" - elseif endswith(src_base, ".fa.gz") - out_base = src_base[1:end-6] * "_reformatted.fa.gz" - elseif endswith(src_base, ".fasta") - out_base = src_base[1:end-6] * "_reformatted.fasta" - else - out_base = src_base * "_reformatted" - end - out_path = joinpath(db_dir, out_base) - - if isfile(out_path) - log("[$key] Using cached reformatted SILVA: $out_path") - return out_path - end - - log("[$key] Reformatting SILVA FASTA for vsearch compatibility: $src_path -> $out_path") - - # Determine if gzipped - is_gz = endswith(src_path, ".gz") - is_out_gz = endswith(out_path, ".gz") - - open_in = is_gz ? () -> GzipDecompressorStream(open(src_path, "r")) : () -> open(src_path, "r") - open_out = is_out_gz ? () -> GzipCompressorStream(open(out_path, "w")) : () -> open(out_path, "w") - - in_io = open_in() - out_io = open_out() - n_seq = 0 - try - for line in eachline(in_io) - if startswith(line, '>') - # Header: ">Accession.start.end Kingdom;Phylum;...;Genus" - rest = line[2:end] - sp = findfirst(' ', rest) - if isnothing(sp) - # No space - already no taxonomy; pass through - write(out_io, line, '\n') - else - acc = rest[1:sp-1] - tax = rest[sp+1:end] - # Replace spaces within taxonomy with underscores, - # then join accession + taxonomy with semicolon. - tax_clean = replace(tax, ' ' => '_') - write(out_io, '>', acc, ';', tax_clean, '\n') - end - n_seq += 1 - else - write(out_io, line, '\n') - end - end - finally - close(out_io) - close(in_io) - end - - log("[$key] Reformatted $n_seq sequences -> $out_path") - return out_path - end - end \ No newline at end of file diff --git a/src/graph.jl b/src/graph.jl new file mode 100644 index 0000000..a9cdd41 --- /dev/null +++ b/src/graph.jl @@ -0,0 +1,124 @@ +module PipelineGraph + +# Declarative pipeline graph. +# +# PIPELINE_STAGES lists every stage in dependency order. Each StageNode records: +# - what wire types it consumes (inputs) and produces (output) +# - which pipeline.yml config sections affect it (for cache invalidation and WebUI) +# +# Stages that consume ProjectCtx do so implicitly — it is omitted from `inputs` +# because every high-level overload takes it. The graph is linear for Illumina +# paired-end runs; branching (e.g. dada2 vs swarm, cdhit optional) is expressed +# by optional/alternative nodes that share the same output type. +# +# This module does not drive execution; main.jl still calls stages explicitly. +# The declaration exists so tools (print_pipeline, the WebUI, future schedulers) +# can inspect the graph without parsing source code. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# Licensed under AGPLv3. + +export PIPELINE_STAGES, print_pipeline, stage_by_name + + using ..PipelineTypes + + const PIPELINE_STAGES = StageNode[ + StageNode( + :fastqc_multiqc, + "Quality Control", + DataType[], # inputs: raw FASTQs are found via ProjectCtx.data_dir + Nothing, + ["fastqc", "multiqc"] + ), + StageNode( + :cutadapt, + "Primer Trimming", + DataType[], # inputs: raw FASTQs via ProjectCtx.data_dir + TrimmedReads, + ["cutadapt"] + ), + StageNode( + :dada2, + "Denoising & ASV Table", + DataType[TrimmedReads], + ASVResult, + ["dada2"] + ), + StageNode( + :cdhit, + "ASV Dereplication (optional)", + DataType[ASVResult], + ASVResult, + ["cdhit"] + ), + StageNode( + :vsearch, + "Taxonomy Search", + DataType[ASVResult], # dispatches on HasFasta — also works for OTUResult + TaxonomyHits, + ["vsearch"] + ), + StageNode( + :merge_taxa, + "Taxonomy Table Merge & Filter", + DataType[ASVResult, TaxonomyHits], + MergedTables, + ["merge_taxa"] + ), + StageNode( + :analyse_run, + "Per-Run Analysis", + DataType[MergedTables, ASVResult], + Nothing, + ["analysis"] + ), + StageNode( + :analyse_group, + "Per-Group Analysis", + DataType[MergedTables], # vector of MergedTables across runs in a group + Nothing, + ["analysis"] + ), + StageNode( + :analyse_study, + "Study-Wide Analysis", + DataType[MergedTables], # vector across all groups + Nothing, + ["analysis"] + ), + ] + + """ + stage_by_name(name::Symbol) -> StageNode + + Look up a stage by its machine name. Throws if not found. + """ + function stage_by_name(name::Symbol) + idx = findfirst(s -> s.name == name, PIPELINE_STAGES) + isnothing(idx) && error("No pipeline stage named :$name") + PIPELINE_STAGES[idx] + end + + """ + print_pipeline([io]) + + Print a human-readable summary of all declared pipeline stages. + """ + function print_pipeline(io::IO = stdout) + println(io, "Declared pipeline stages:") + println(io, "─"^60) + for (i, s) in enumerate(PIPELINE_STAGES) + ins = isempty(s.inputs) ? "raw FASTQs (via ProjectCtx)" : + join(string.(s.inputs), ", ") + out = s.output === Nothing ? "— (side effects only)" : string(s.output) + cfgs = join(s.config_sections, ", ") + println(io, " $i. $(s.label) [:$(s.name)]") + println(io, " inputs: $ins") + println(io, " output: $out") + println(io, " config: $cfgs") + i < length(PIPELINE_STAGES) && println(io) + end + println(io, "─"^60) + end + +end diff --git a/src/main.jl b/src/main.jl index 4e0c34e..dd4eb5d 100755 --- a/src/main.jl +++ b/src/main.jl @@ -4,22 +4,36 @@ import Pkg Pkg.activate(joinpath(@__DIR__, ".."); io=devnull) +import Logging, Dates +struct _TimestampLogger <: Logging.AbstractLogger + inner::Logging.AbstractLogger +end +Logging.min_enabled_level(l::_TimestampLogger) = Logging.min_enabled_level(l.inner) +Logging.shouldlog(l::_TimestampLogger, args...) = Logging.shouldlog(l.inner, args...) +Logging.catch_exceptions(l::_TimestampLogger) = Logging.catch_exceptions(l.inner) +function Logging.handle_message(l::_TimestampLogger, level, message, _module, group, id, file, line; kwargs...) + ts = Dates.format(Dates.now(), "HH:MM:SS") + Logging.handle_message(l.inner, level, "[$ts] $message", _module, group, id, file, line; kwargs...) +end +Logging.global_logger(_TimestampLogger(Logging.global_logger())) + include("types.jl") include("log.jl") include("config.jl") include("databases.jl") include("call_tools.jl") include("dada2.jl") -include("merge_and_filter_taxa.jl") +include("merge_taxa.jl") include("project.jl") include("diversity.jl") include("plots.jl") include("analysis.jl") +include("graph.jl") using CSV using YAML using .PipelineTypes, .PipelineLog, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup -using .DiversityMetrics, .PipelinePlots, .Analysis +using .DiversityMetrics, .PipelinePlots, .Analysis, .PipelineGraph ## Instantiate parameters config_dir = "./config" @@ -30,40 +44,46 @@ dbs = ensure_databases(databases_config) ## Main const r_lock = ReentrantLock() -projects = new_project("PR2_v_SILVA") +projects = new_project("Multi_v_Vespa") merged_results = Vector{Union{MergedTables, Nothing}}(undef, length(projects)) asvs_results = Vector{Union{ASVResult, Nothing}}(undef, length(projects)) db_metas = Vector{DatabaseMeta}(undef, length(projects)) Threads.@threads for (i, project) in collect(enumerate(projects)) - reset_log(project) - multiqc(project.data_dir, joinpath(project.dir, "QC")) - - # Resolve per-project database from its config cascade. - run_cfg_path = write_run_config(project) - db_name = string(get(get(get(YAML.load_file(run_cfg_path), "dada2", Dict()), "taxonomy", Dict()), "database", "pr2")) - db_metas[i] = make_db_meta(databases_config, db_name) - - trimmed = cutadapt(project) - - asvs = lock(r_lock) do - dada2(project, trimmed, taxonomy_db = dbs["$(db_name)_dada2"]) + try + reset_log(project) + multiqc(project) + + # Resolve per-project database from its config cascade. + run_cfg_path = write_run_config(project) + db_name = string(get(get(get(YAML.load_file(run_cfg_path), "dada2", Dict()), "taxonomy", Dict()), "database", "pr2")) + db_metas[i] = make_db_meta(databases_config, db_name) + + trimmed = cutadapt(project) + + asvs = lock(r_lock) do + dada2(project, trimmed, taxonomy_db = dbs["$(db_name)_dada2"]) + end + asvs = lock(r_lock) do; cdhit(project, asvs); end + + if haskey(dbs, "$(db_name)_vsearch") + tax = vsearch(project, asvs, dbs["$(db_name)_vsearch"]) + merged = merge_taxa(project, asvs, tax, db_metas[i]) + else + @warn "No vsearch database for '$db_name' - using DADA2 tax_counts as merged for $(basename(project.dir))" + tax_counts = joinpath(project.dir, "dada2", "Tables", "tax_counts.csv") + merged = isfile(tax_counts) && filesize(tax_counts) > 0 ? + MergedTables(Dict("merged" => tax_counts), String[], Dict{String,String}()) : + nothing + end + merged_results[i] = merged + asvs_results[i] = asvs + catch e + @error "Project $(basename(project.dir)) failed" exception=(e, catch_backtrace()) + merged_results[i] = nothing + asvs_results[i] = nothing end - asvs = lock(r_lock) do; cdhit(project, asvs; optional_args = "-c 1"); end - - if haskey(dbs, "$(db_name)_vsearch") - tax = vsearch(project, asvs, dbs["$(db_name)_vsearch"]) - merged = merge_taxa(project, asvs, tax, db_metas[i]) - else - @warn "No vsearch database for '$db_name' - using DADA2 tax_counts as merged for $(basename(project.dir))" - tax_counts = joinpath(project.dir, "dada2", "Tables", "tax_counts.csv") - merged = isfile(tax_counts) && filesize(tax_counts) > 0 ? - MergedTables(Dict("merged" => tax_counts), String[], Dict{String,String}()) : - nothing - end - merged_results[i] = merged - asvs_results[i] = asvs end # Analysis: data prep runs in parallel, CairoMakie calls serialized via plot_lock. @@ -73,7 +93,12 @@ Threads.@threads for (i, project) in collect(enumerate(projects)) merged = merged_results[i] asvs = asvs_results[i] (isnothing(merged) || isnothing(asvs)) && continue - analyse_run(project, merged, asvs, db_metas[i]; plot_lock) + isassigned(db_metas, i) || continue + try + analyse_run(project, merged, asvs, db_metas[i]; plot_lock) + catch e + @error "analyse_run failed for $(basename(project.dir))" exception=(e, catch_backtrace()) + end end if any(!isnothing, merged_results) @@ -81,3 +106,4 @@ if any(!isnothing, merged_results) end write_combined_log(projects) +print_pipeline() diff --git a/src/merge_and_filter_taxa.jl b/src/merge_taxa.jl similarity index 96% rename from src/merge_and_filter_taxa.jl rename to src/merge_taxa.jl index 8890f47..cc4d4b0 100644 --- a/src/merge_and_filter_taxa.jl +++ b/src/merge_taxa.jl @@ -44,9 +44,6 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa meta_cols = ["SeqName", "Pident", "Accession", "rRNA", "Organellum", "specimen"] header = vcat(meta_cols, levels) n_meta = length(meta_cols) - 2 # fields parsed from sseqtax (excl SeqName, Pident) - elseif db_meta.vsearch_format == "silva" - header = vcat(["SeqName", "Pident"], levels) - n_meta = 0 else # Generic: semicolon-separated, mapped positionally to levels header = vcat(["SeqName", "Pident"], levels) @@ -74,17 +71,6 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa for j in 1:min(length(parts), length(levels)) data[tax_start + j - 1] = parts[j] end - elseif db_meta.vsearch_format == "silva" - # SILVA (reformatted): vsearch target field is "Accession;Kingdom;Phylum;...;Genus" - # (spaces within taxon names replaced by underscores during DB reformatting). - # Split on ';' and skip the first field (accession). - all_parts = split(tax_str, ';') - # Drop leading accession field - parts = filter(!isempty, length(all_parts) > 1 ? all_parts[2:end] : all_parts) - tax_start = 3 # after SeqName, Pident - for j in 1:min(length(parts), length(levels)) - data[tax_start + j - 1] = replace(String(strip(parts[j])), '_' => ' ') - end else # Generic: semicolon-separated, mapped positionally to levels parts = filter(!isempty, split(tax_str, ';')) diff --git a/src/types.jl b/src/types.jl index 4e6cf92..0ef722f 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,6 +1,7 @@ module PipelineTypes - export HasFasta, ProjectCtx, TrimmedReads, ASVResult, DenoisedASVs, TaxonomyHits, MergedTables, DatabaseMeta + export HasFasta, ProjectCtx, TrimmedReads, ASVResult, DenoisedASVs, TaxonomyHits, MergedTables, DatabaseMeta, + StageNode abstract type HasFasta end @@ -46,4 +47,25 @@ module PipelineTypes filter_colours::Dict{String,String} # filter stem => hex colour override (from YAML "colour" key) end + """ + StageNode(name, label, inputs, output, config_sections) + + Declarative description of one pipeline stage. + + - `name`: machine identifier (Symbol), e.g. `:cutadapt` + - `label`: human-readable stage name + - `inputs`: wire types this stage requires (excluding `ProjectCtx`, which every + ProjectCtx-aware stage implicitly receives) + - `output`: wire type produced; `Nothing` for side-effect-only stages + - `config_sections`: pipeline.yml section keys whose content controls this stage + (used by skip guards and the WebUI to know what to re-hash) + """ + struct StageNode + name::Symbol + label::String + inputs::Vector{DataType} + output::DataType + config_sections::Vector{String} + end + end From abc79edd06043b9837897056c2c4cd92a797cc9b Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 27 Feb 2026 12:13:35 +0100 Subject: [PATCH 031/175] Much needed src/ restructure. --- README.md | 49 ++---- config/defaults/primers.yml | 4 +- config/filters/bacteria_archaea.pr2.yml | 2 +- config/filters/fungi.pr2.yml | 2 +- config/filters/helminths.pr2.yml | 2 +- config/filters/parasitic_protozoa.pr2.yml | 2 +- config/filters/protist.pr2.yml | 2 +- config/filters/vertebrates.pr2.yml | 2 +- src/analysis.jl | 35 ---- src/analysis/analysis.jl | 35 ++++ src/{ => analysis}/diversity.jl | 0 src/analysis/group.jl | 70 ++++---- src/analysis/helpers.jl | 2 +- src/{ => analysis}/plots.jl | 0 src/analysis/run.jl | 22 ++- src/analysis/study.jl | 29 +++- src/{ => core}/config.jl | 0 src/{ => core}/databases.jl | 0 src/core/graph.jl | 183 +++++++++++++++++++++ src/{ => core}/log.jl | 0 src/{ => core}/project.jl | 0 src/{ => core}/types.jl | 0 src/graph.jl | 124 -------------- src/main.jl | 29 ++-- src/{ => pipeline}/dada2.jl | 1 + src/{ => pipeline}/dada2/chimera.jl | 4 +- src/{ => pipeline}/dada2/context.jl | 2 +- src/{ => pipeline}/dada2/dada2_functions.r | 0 src/{ => pipeline}/dada2/denoise.jl | 12 +- src/{ => pipeline}/dada2/qc.jl | 4 +- src/{ => pipeline}/dada2/taxonomy.jl | 6 +- src/{ => pipeline}/dada2/taxonomy_remote.r | 0 src/{ => pipeline}/merge_taxa.jl | 15 +- src/{call_tools.jl => pipeline/tools.jl} | 16 +- 34 files changed, 367 insertions(+), 287 deletions(-) delete mode 100644 src/analysis.jl create mode 100644 src/analysis/analysis.jl rename src/{ => analysis}/diversity.jl (100%) rename src/{ => analysis}/plots.jl (100%) rename src/{ => core}/config.jl (100%) rename src/{ => core}/databases.jl (100%) create mode 100644 src/core/graph.jl rename src/{ => core}/log.jl (100%) rename src/{ => core}/project.jl (100%) rename src/{ => core}/types.jl (100%) delete mode 100644 src/graph.jl rename src/{ => pipeline}/dada2.jl (99%) rename src/{ => pipeline}/dada2/chimera.jl (96%) rename src/{ => pipeline}/dada2/context.jl (98%) rename src/{ => pipeline}/dada2/dada2_functions.r (100%) rename src/{ => pipeline}/dada2/denoise.jl (94%) rename src/{ => pipeline}/dada2/qc.jl (96%) rename src/{ => pipeline}/dada2/taxonomy.jl (97%) rename src/{ => pipeline}/dada2/taxonomy_remote.r (100%) rename src/{ => pipeline}/merge_taxa.jl (97%) rename src/{call_tools.jl => pipeline/tools.jl} (97%) diff --git a/README.md b/README.md index 0db4ce6..048664c 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,20 @@ Each file in `config/filters/` defines one biological group to extract from the #### Database-specific filters -Filter files are named `{category}.{database}.yml` and carry a `databases:` key so that each filter is only applied when the active database matches: +Filter files carry a `databases:` key so that each filter is only applied when the active database matches. The following filters ship in `config/filters/`: + +| Category | PR2 match | +|----------|-----------| +| `bacteria_archaea` | `Domain` = Bacteria\|Archaea | +| `environmental_protozoa` | `Subdivision` = Cercozoa\|Gyrista\|Ciliophora\|Chrompodellids | +| `fungi` | `Subdivision` = Fungi | +| `helminths` | `Class` = Nematoda (excl. *Miculenchus*) | +| `parasitic_protozoa` | `Subdivision` = Apicomplexa\|Parabasalia\|Fornicata\|Bigyra | +| `plants_invertebrates` | Exclusion-based (PR2 ranks) | +| `protist` | Exclusion-based (PR2 ranks) | +| `vertebrates` | `Class` = Craniata | + +Example: ```yaml # fungi.pr2.yml @@ -274,40 +287,6 @@ remove_empty: - Subdivision ``` -```yaml -# fungi.silva.yml -databases: [silva] - -filters: - - column: Family - pattern: Nucletmycea - action: keep - -remove_empty: - - Family -``` - -Both files can be listed in `merge_taxa.filters` simultaneously; the pipeline silently skips whichever does not apply to the active database. - -The following filter pairs ship in `config/filters/`: - -| Category | PR2 match | SILVA match | -|----------|-----------|-------------| -| `bacteria_archaea` | `Domain` = Bacteria\|Archaea | `Kingdom` = Bacteria\|Archaea | -| `environmental_protozoa` | `Subdivision` = Cercozoa\|Gyrista\|Ciliophora\|Chrompodellids | `Order` = Cercozoa\|Ciliophora\|Ochrophyta | -| `fungi` | `Subdivision` = Fungi | `Family` = Nucletmycea | -| `helminths` | `Class` = Nematoda (excl. *Miculenchus*) | `Family` = Holozoa ¹ | -| `parasitic_protozoa` | `Subdivision` = Apicomplexa\|Parabasalia\|Fornicata\|Bigyra | `Family` = Conoidasida\|Trichomonadea\|Blastocystis\|Proteromonadea | -| `plants_invertebrates` | Exclusion-based (PR2 ranks) | Exclusion-based (SILVA ranks) ¹ | -| `protist` | Exclusion-based (PR2 ranks) | Exclusion-based (SILVA ranks) | -| `vertebrates` | `Class` = Craniata | `Family` = Holozoa ¹ | - -> ¹ **SILVA metazoan resolution limit.** SILVA's 18S taxonomy uses only six ranks (Kingdom → Genus). The entire Metazoa lineage compresses into these ranks such that all animals — vertebrates, nematodes, annelids — resolve to `Family = Holozoa`, `Genus = Choanozoa`. There is no rank at which helminths and vertebrates can be distinguished. As a result: -> - `helminths.silva.yml` and `vertebrates.silva.yml` are functionally equivalent: both capture all metazoa. -> - `plants_invertebrates.silva.yml` retains metazoa alongside plants but cannot exclude vertebrates. -> -> Use PR2 (`database: pr2`) when helminth- or vertebrate-specific filtering is required. - #### Filter file format ```yaml diff --git a/config/defaults/primers.yml b/config/defaults/primers.yml index 303a0b4..b3e5a01 100644 --- a/config/defaults/primers.yml +++ b/config/defaults/primers.yml @@ -1,5 +1,5 @@ -# Kandaurova, E. (2025) ‘Metabarcoding of gut protists of Eurasian beaver’. -# Owens, L.A., Friant, S., Martorelli Di Genova, B., Knoll, L.J., Contreras, M., Noya-Alarcon, O., Dominguez-Bello, M.G., and Goldberg, T.L. (2024) ‘VESPA: an optimized protocol for accurate metabarcoding-based characterization of vertebrate eukaryotic endosymbiont and parasite assemblages’, Nature Communications, 15(1), 402, available: https://doi.org/10.1038/s41467-023-44521-3. +# Kandaurova, E. (2025) 'Metabarcoding of gut protists of Eurasian beaver'. +# Owens, L.A., Friant, S., Martorelli Di Genova, B., Knoll, L.J., Contreras, M., Noya-Alarcon, O., Dominguez-Bello, M.G., and Goldberg, T.L. (2024) 'VESPA: an optimized protocol for accurate metabarcoding-based characterization of vertebrate eukaryotic endosymbiont and parasite assemblages', Nature Communications, 15(1), 402, available: https://doi.org/10.1038/s41467-023-44521-3. Forward: TarEukF: "CCAGCASCYGCGGTAATTCC" diff --git a/config/filters/bacteria_archaea.pr2.yml b/config/filters/bacteria_archaea.pr2.yml index 58b7b4c..b4b493c 100644 --- a/config/filters/bacteria_archaea.pr2.yml +++ b/config/filters/bacteria_archaea.pr2.yml @@ -1,4 +1,4 @@ -# Bacteria + Archaea filter — retains all prokaryotic ASVs. +# Bacteria + Archaea filter - retains all prokaryotic ASVs. # Covers Bacteroidetes, Firmicutes, Tenericutes, Cyanobacteria, and # any Archaea if present. databases: [pr2] diff --git a/config/filters/fungi.pr2.yml b/config/filters/fungi.pr2.yml index 324e488..6c00f14 100644 --- a/config/filters/fungi.pr2.yml +++ b/config/filters/fungi.pr2.yml @@ -1,4 +1,4 @@ -# Fungi filter — retains all fungal ASVs. +# Fungi filter - retains all fungal ASVs. # PR2 taxonomy: Opisthokonta > Fungi > (Basidiomycota, Ascomycota, # Mucoromycota, Chytridiomycota, Blastocladiomycota, Neocallimastigomycota). databases: [pr2] diff --git a/config/filters/helminths.pr2.yml b/config/filters/helminths.pr2.yml index c38618a..7ffbc3b 100644 --- a/config/filters/helminths.pr2.yml +++ b/config/filters/helminths.pr2.yml @@ -1,4 +1,4 @@ -# Helminth filter — retains parasitic nematodes only. +# Helminth filter - retains parasitic nematodes only. # Keeps Nematoda (Trichuris, Strongyloides) but excludes Miculenchus # (a moss-associated tylenchid, not a medical helminth). databases: [pr2] diff --git a/config/filters/parasitic_protozoa.pr2.yml b/config/filters/parasitic_protozoa.pr2.yml index 319e57f..2bb34c5 100644 --- a/config/filters/parasitic_protozoa.pr2.yml +++ b/config/filters/parasitic_protozoa.pr2.yml @@ -1,4 +1,4 @@ -# Parasitic protozoa — host-associated protists. +# Parasitic protozoa - host-associated protists. # # Apicomplexa: Eimeria, Isospora (gut coccidians); Monocystis, # Amoebogregarina, Leidyana (invertebrate gregarines) diff --git a/config/filters/protist.pr2.yml b/config/filters/protist.pr2.yml index e0aed5b..5d128d3 100644 --- a/config/filters/protist.pr2.yml +++ b/config/filters/protist.pr2.yml @@ -1,4 +1,4 @@ -# Protist filter — retains free-living and parasitic protists. +# Protist filter - retains free-living and parasitic protists. # Only applied when the active database is in the 'databases' list below. databases: [pr2] # Excludes: Bacteria, Archaea, organellar sequences, chromista sensu stricto, diff --git a/config/filters/vertebrates.pr2.yml b/config/filters/vertebrates.pr2.yml index fd5e0c7..905654f 100644 --- a/config/filters/vertebrates.pr2.yml +++ b/config/filters/vertebrates.pr2.yml @@ -1,4 +1,4 @@ -# Host contamination filter — retains vertebrate sequences. +# Host contamination filter - retains vertebrate sequences. # PR2 taxonomy: Subdivision=Metazoa > Class=Craniata. # Craniata covers all vertebrates (mammals, birds, fish, etc.). databases: [pr2] diff --git a/src/analysis.jl b/src/analysis.jl deleted file mode 100644 index fce14f0..0000000 --- a/src/analysis.jl +++ /dev/null @@ -1,35 +0,0 @@ -module Analysis - -# Post-pipeline analysis: per-run, per-group, and study-level outputs. -# -# Entry points (called from main.jl): -# analyse_run(project, merged, asvs, db_meta; plot_lock) -# analyse_study(projects, merged_results, db_metas; plot_lock) -# load_metadata(start_dir, study_dir) -# -# Internal structure (src/analysis/): -# helpers.jl — column helpers, taxonomy view, rank/source key utilities -# stats.jl — alpha diversity, count matrices, NMDS/PERMANOVA (R), metadata loading -# composition.jl — pipeline summary CSV, priority filter composition -# report.jl — chart generation, text report writing, table formatters -# run.jl — analyse_run (level 1: per-run charts + reports) -# group.jl — _analyse_group (level 2: multi-run comparison) -# study.jl — _analyse_study_level + analyse_study (level 3: cross-group) -# -# © 2026 Joshua Benjamin Jewell. All rights reserved. -# Licensed under the GNU Affero General Public License version 3 (AGPLv3). - -export analyse_run, analyse_study, load_metadata - - using CSV, DataFrames, Dates, Logging, Statistics, YAML, RCall - using ..PipelineTypes, ..PipelineLog, ..Config, ..DiversityMetrics, ..PipelinePlots - - include("analysis/helpers.jl") - include("analysis/stats.jl") - include("analysis/composition.jl") - include("analysis/report.jl") - include("analysis/run.jl") - include("analysis/group.jl") - include("analysis/study.jl") - -end diff --git a/src/analysis/analysis.jl b/src/analysis/analysis.jl new file mode 100644 index 0000000..e3a849b --- /dev/null +++ b/src/analysis/analysis.jl @@ -0,0 +1,35 @@ +module Analysis + +# Post-pipeline analysis: per-run, per-group, and study-level outputs. +# +# Entry points (called from main.jl): +# analyse_run(project, merged, asvs, db_meta; plot_lock) +# analyse_study(projects, merged_results, db_metas; plot_lock) +# load_metadata(start_dir, study_dir) +# +# Internal structure (src/analysis/): +# helpers.jl - column helpers, taxonomy view, rank/source key utilities +# stats.jl - alpha diversity, count matrices, NMDS/PERMANOVA (R), metadata loading +# composition.jl - pipeline summary CSV, priority filter composition +# report.jl - chart generation, text report writing, table formatters +# run.jl - analyse_run (level 1: per-run charts + reports) +# group.jl - _analyse_group (level 2: multi-run comparison) +# study.jl - _analyse_study_level + analyse_study (level 3: cross-group) +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# Licensed under the GNU Affero General Public License version 3 (AGPLv3). + +export analyse_run, analyse_study, load_metadata + + using CSV, DataFrames, Dates, Logging, Statistics, YAML, RCall + using ..PipelineTypes, ..PipelineLog, ..Config, ..DiversityMetrics, ..PipelinePlots + + include("helpers.jl") + include("stats.jl") + include("composition.jl") + include("report.jl") + include("run.jl") + include("group.jl") + include("study.jl") + +end diff --git a/src/diversity.jl b/src/analysis/diversity.jl similarity index 100% rename from src/diversity.jl rename to src/analysis/diversity.jl diff --git a/src/analysis/group.jl b/src/analysis/group.jl index c94f7d5..f51f976 100644 --- a/src/analysis/group.jl +++ b/src/analysis/group.jl @@ -38,11 +38,45 @@ for (_, m) in members if isfile(m.tables["merged"]); init=0.0 ) + + # Initialise cache here so it can be shared between the skip guard and execution. + cache = _CSVCache() + + # Helper used in both the skip guard and the execution loop. + function _collect_source_data_sg(source::String) + dfs = DataFrame[] + scols = Vector{String}[] + names_ = String[] + labels = String[] + all_s = String[] + for (proj, merged) in members + haskey(merged.tables, source) || continue + csv = merged.tables[source] + isfile(csv) || continue + df, sc = _cached_read(cache, csv, db_meta) + isempty(sc) && continue + rn = basename(proj.dir) + push!(dfs, df); push!(scols, sc); push!(names_, rn) + append!(all_s, sc); append!(labels, fill(rn, length(sc))) + end + return dfs, scols, names_, labels, all_s + end + required_outputs = String[alpha_pdf, nmds_pdf] has_any_filters && push!(required_outputs, filter_pdf) for method in _TAX_METHODS msrc_keys = _method_source_keys(members[1][2], method) for src in msrc_keys + s_dfs, _, _, _, s_all = _collect_source_data_sg( + endswith(src, "_dada2") ? src : src) + if isempty(s_dfs) && method == "dada2" && src == "merged" + s_dfs, _, _, _, s_all = _collect_source_data_sg("merged") + end + isempty(s_dfs) && continue + method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue + view_dfs = [_method_df(df, method, db_meta.levels) for df in s_dfs] + combined = reduce((a, b) -> vcat(a, b; cols=:union), view_dfs) + _total_seqs(combined, s_all) == 0 && continue sd = _method_source_dirname(src) for (rankdir, _) in taxa_ranks push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) @@ -50,9 +84,7 @@ push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison.pdf")) push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "group_comparison_absolute.pdf")) end - end - for src in msrc_keys - push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt")) end end @@ -65,42 +97,20 @@ reset_log(group_dir) mkpath(figures_dir) - cache = _CSVCache() - - # helper: collect per-run data for a given source key - function _collect_source_data(source::String) - dfs = DataFrame[] - scols = Vector{String}[] - names_ = String[] - labels = String[] - all_s = String[] - for (proj, merged) in members - haskey(merged.tables, source) || continue - csv = merged.tables[source] - isfile(csv) || continue - df, sc = _cached_read(cache, csv, db_meta) - isempty(sc) && continue - rn = basename(proj.dir) - push!(dfs, df); push!(scols, sc); push!(names_, rn) - append!(all_s, sc); append!(labels, fill(rn, length(sc))) - end - return dfs, scols, names_, labels, all_s - end - # Primary source data (for NMDS, alpha, filter composition). run_dfs, run_scols, run_names, run_labels, all_scols = - _collect_source_data(src_key) + _collect_source_data_sg(src_key) isempty(run_dfs) && return # taxa bar & group comparison for ALL sources (dual method) for method in _TAX_METHODS msrc_keys = _method_source_keys(members[1][2], method) for src in msrc_keys - s_dfs, s_scols, s_names, _, s_all = _collect_source_data( + s_dfs, s_scols, s_names, _, s_all = _collect_source_data_sg( endswith(src, "_dada2") ? src : src) # For "merged" under dada2, use the same "merged" key if isempty(s_dfs) && method == "dada2" && src == "merged" - s_dfs, s_scols, s_names, _, s_all = _collect_source_data("merged") + s_dfs, s_scols, s_names, _, s_all = _collect_source_data_sg("merged") end isempty(s_dfs) && continue # Skip dada2 if no dada2 columns @@ -256,10 +266,10 @@ for method in _TAX_METHODS msrc_keys = _method_source_keys(members[1][2], method) for src in msrc_keys - s_dfs, s_scols, _, _, s_all = _collect_source_data( + s_dfs, s_scols, _, _, s_all = _collect_source_data_sg( endswith(src, "_dada2") ? src : src) if isempty(s_dfs) && method == "dada2" && src == "merged" - s_dfs, s_scols, _, _, s_all = _collect_source_data("merged") + s_dfs, s_scols, _, _, s_all = _collect_source_data_sg("merged") end isempty(s_dfs) && continue method == "dada2" && !_has_dada2(s_dfs[1], db_meta.levels) && continue diff --git a/src/analysis/helpers.jl b/src/analysis/helpers.jl index 549293b..97e96b8 100644 --- a/src/analysis/helpers.jl +++ b/src/analysis/helpers.jl @@ -64,7 +64,7 @@ any(r -> startswith(c, r), levels), names(df)) # Union source keys across multiple MergedTables (for study/group levels where - # different runs may carry different filter sets, e.g. .pr2 vs .silva filters). + # different runs may carry different filter sets). function _all_method_source_keys(mergeds::Vector{MergedTables}, method::String) seen = Set{String}() out = String[] diff --git a/src/plots.jl b/src/analysis/plots.jl similarity index 100% rename from src/plots.jl rename to src/analysis/plots.jl diff --git a/src/analysis/run.jl b/src/analysis/run.jl index 3302899..490f319 100644 --- a/src/analysis/run.jl +++ b/src/analysis/run.jl @@ -31,19 +31,33 @@ taxa_ranks = _taxa_ranks(db_meta.levels, analysis_cfg) report_ranks = _report_ranks(db_meta.levels, analysis_cfg) - required_outputs = String[summary_path, stages_pdf, alpha_pdf] + # CSV cache - initialised here so the skip guard and execution share one set of reads. + cache = _CSVCache() + + # Build required_outputs using the same guards as the execution loop so that + # filters producing no rows (empty CSVs) don't add paths that will never be written. + required_outputs = String[summary_path, alpha_pdf] has_filters && push!(required_outputs, filter_pdf) for method in _TAX_METHODS msrc_keys = _method_source_keys(merged, method) for src in msrc_keys + src_csv_path = get(merged.tables, src, "") + if isempty(src_csv_path) || !isfile(src_csv_path) || filesize(src_csv_path) == 0 + src == "merged" || continue + src_csv_path = merged.tables["merged"] + (!isfile(src_csv_path) || filesize(src_csv_path) == 0) && continue + end + raw_df, src_scols = _cached_read(cache, src_csv_path, db_meta) + isempty(src_scols) && continue + method == "dada2" && !_has_dada2(raw_df, db_meta.levels) && continue + view_df = _method_df(raw_df, method, db_meta.levels) + _total_seqs(view_df, src_scols) == 0 && continue sd = _method_source_dirname(src) for (rankdir, _) in taxa_ranks push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar.pdf")) push!(required_outputs, joinpath(figures_dir, method, sd, rankdir, "taxa_bar_absolute.pdf")) end - end - for src in msrc_keys - push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) + push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(sd).txt")) end end diff --git a/src/analysis/study.jl b/src/analysis/study.jl index 35a1e55..9bc096a 100644 --- a/src/analysis/study.jl +++ b/src/analysis/study.jl @@ -29,10 +29,29 @@ init=0.0 ) all_mergeds = [m for (_, m, _) in valid] + + # Initialise cache here so it is shared between the skip guard and execution. + cache = _CSVCache() + required_outputs = [nmds_pdf, alpha_pdf] for method in _TAX_METHODS msrc_keys = _all_method_source_keys(all_mergeds, method) for src in msrc_keys + # Mirror the execution loop: only include the report path if at + # least one run has a non-empty CSV for this source key. + has_data = any(valid) do (proj, merged, dm) + csv = get(merged.tables, src, "") + if isempty(csv) && method == "dada2" && src == "merged" + csv = get(merged.tables, "merged", "") + end + (isempty(csv) || !isfile(csv) || filesize(csv) == 0) && return false + df, sc = _cached_read(cache, csv, dm) + isempty(sc) && return false + method == "dada2" && !_has_dada2(df, default_db_meta.levels) && return false + vdf = _method_df(df, method, default_db_meta.levels) + _total_seqs(vdf, sc) > 0 + end + has_data || continue push!(required_outputs, joinpath(analysis_dir, "analysis_report_$(method)_$(_method_source_dirname(src)).txt")) end end @@ -48,8 +67,6 @@ # Helper: serialize CairoMakie calls if plot_lock is provided. _plot(f) = isnothing(plot_lock) ? f() : lock(f, plot_lock) - cache = _CSVCache() - src_label = _source_label(src_key) subtitle = "Source: $src_label" @@ -138,7 +155,7 @@ log_written(study_dir, alpha_pdf) end - # PERMANOVA — always runs using group and run as covariates (derived from the + # PERMANOVA - always runs using group and run as covariates (derived from the # project structure). Optionally, place a metadata.csv in the study directory # with a 'sample' column and extra covariate columns; those columns are merged # in when every sample name in all_scols is unique (i.e. no cross-database @@ -153,9 +170,9 @@ sample_col = "sample" in names(user_meta) ? "sample" : "Sample" in names(user_meta) ? "Sample" : nothing if isnothing(sample_col) - @warn "metadata.csv has no 'sample' column — using group/run only for PERMANOVA" + @warn "metadata.csv has no 'sample' column - using group/run only for PERMANOVA" elseif length(unique(all_scols)) < length(all_scols) - @warn "Duplicate sample names across runs — extra metadata.csv columns ignored; using group/run only" + @warn "Duplicate sample names across runs - extra metadata.csv columns ignored; using group/run only" else extra_cols = [c for c in names(user_meta) if c != sample_col] if !isempty(extra_cols) @@ -166,7 +183,7 @@ base_meta[!, col] = [m[col] for m in matched] end else - @warn "Not all samples found in metadata.csv — extra columns ignored" + @warn "Not all samples found in metadata.csv - extra columns ignored" end end end diff --git a/src/config.jl b/src/core/config.jl similarity index 100% rename from src/config.jl rename to src/core/config.jl diff --git a/src/databases.jl b/src/core/databases.jl similarity index 100% rename from src/databases.jl rename to src/core/databases.jl diff --git a/src/core/graph.jl b/src/core/graph.jl new file mode 100644 index 0000000..f964cca --- /dev/null +++ b/src/core/graph.jl @@ -0,0 +1,183 @@ +module PipelineGraph + +# Declarative pipeline graph - single source of truth for stage identity. +# +# PIPELINE_STAGES lists every stage in dependency order. Each StageNode records: +# - what wire types it consumes (inputs) and produces (output) +# - which pipeline.yml config sections affect it +# +# config_sections is AUTHORITATIVE: skip guards in each stage function call +# stage_sections(:name) rather than hardcoding section strings. Any change to +# which config keys invalidate a stage is made here and nowhere else. +# +# Stages that consume ProjectCtx do so implicitly - it is omitted from `inputs` +# because every high-level overload takes it. +# +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# Licensed under AGPLv3. + +export PIPELINE_STAGES, stage_by_name, stage_sections, print_pipeline + + using ..PipelineTypes + + const PIPELINE_STAGES = StageNode[ + + ## Quality Control + StageNode( + :fastqc_multiqc, + "Quality Control", + DataType[], + Nothing, + ["fastqc", "multiqc"] + ), + + ## Primer Trimming + StageNode( + :cutadapt, + "Primer Trimming", + DataType[], + TrimmedReads, + ["cutadapt"] + ), + + ## DADA2 sub-stages (individually resumable) + StageNode( + :dada2_filter_trim, + "Filter & Trim", + DataType[TrimmedReads], + Nothing, # produces ckpt_filter.RData; wire type is DenoisedASVs at end + ["dada2.filter_trim"] + ), + StageNode( + :dada2_learn_errors, + "Learn Error Rates", + DataType[TrimmedReads], + Nothing, + ["dada2.dada"] + ), + StageNode( + :dada2_denoise, + "Denoise & Merge", + DataType[TrimmedReads], + Nothing, + ["dada2.dada", "dada2.merge"] + ), + StageNode( + :dada2_filter_length, + "ASV Length Filter", + DataType[TrimmedReads], + Nothing, + ["dada2.asv"] + ), + StageNode( + :dada2_chimera_removal, + "Chimera Removal", + DataType[TrimmedReads], + DenoisedASVs, + ["dada2.asv", "dada2.output"] + ), + StageNode( + :dada2_assign_taxonomy, + "Taxonomy Assignment (DADA2)", + DataType[DenoisedASVs], + ASVResult, + ["dada2.taxonomy", "dada2.output"] + ), + + ## Post-DADA2 + StageNode( + :cdhit, + "ASV Dereplication (optional)", + DataType[ASVResult], + ASVResult, + ["cdhit"] + ), + StageNode( + :vsearch, + "Taxonomy Search", + DataType[ASVResult], # dispatches on HasFasta - also works for OTUResult + TaxonomyHits, + ["vsearch"] + ), + StageNode( + :merge_taxa, + "Taxonomy Table Merge & Filter", + DataType[ASVResult, TaxonomyHits], + MergedTables, + ["merge_taxa"] + ), + + ## Analysis + StageNode( + :analyse_run, + "Per-Run Analysis", + DataType[MergedTables, ASVResult], + Nothing, + ["analysis"] + ), + StageNode( + :analyse_group, + "Per-Group Analysis", + DataType[MergedTables], + Nothing, + ["analysis"] + ), + StageNode( + :analyse_study, + "Study-Wide Analysis", + DataType[MergedTables], + Nothing, + ["analysis"] + ), + ] + + """ + stage_by_name(name::Symbol) -> StageNode + + Look up a stage by its machine name. Throws if not found. + """ + function stage_by_name(name::Symbol) + idx = findfirst(s -> s.name == name, PIPELINE_STAGES) + isnothing(idx) && error("No pipeline stage named :$name") + PIPELINE_STAGES[idx] + end + + """ + stage_sections(name::Symbol) -> String + + Return the comma-joined config section string for a stage, in the format + expected by `_section_stale` / `_write_section_hash`. + + This is the single point of truth: skip guards call this rather than + hardcoding section strings, so changes to which config keys invalidate a + stage are made here and propagate automatically. + + _section_stale(config_path, stage_sections(:cutadapt), hash_file) + """ + function stage_sections(name::Symbol)::String + join(stage_by_name(name).config_sections, ",") + end + + """ + print_pipeline([io]) + + Print a human-readable summary of all declared pipeline stages. + """ + function print_pipeline(io::IO = stdout) + println(io, "Declared pipeline stages:") + println(io, "-"^60) + for (i, s) in enumerate(PIPELINE_STAGES) + ins = isempty(s.inputs) ? "raw FASTQs (via ProjectCtx)" : + join(string.(s.inputs), ", ") + out = s.output === Nothing ? "- (side effects only)" : string(s.output) + cfgs = join(s.config_sections, ", ") + println(io, " $i. $(s.label) [:$(s.name)]") + println(io, " inputs: $ins") + println(io, " output: $out") + println(io, " config: $cfgs") + i < length(PIPELINE_STAGES) && println(io) + end + println(io, "-"^60) + end + +end diff --git a/src/log.jl b/src/core/log.jl similarity index 100% rename from src/log.jl rename to src/core/log.jl diff --git a/src/project.jl b/src/core/project.jl similarity index 100% rename from src/project.jl rename to src/core/project.jl diff --git a/src/types.jl b/src/core/types.jl similarity index 100% rename from src/types.jl rename to src/core/types.jl diff --git a/src/graph.jl b/src/graph.jl deleted file mode 100644 index a9cdd41..0000000 --- a/src/graph.jl +++ /dev/null @@ -1,124 +0,0 @@ -module PipelineGraph - -# Declarative pipeline graph. -# -# PIPELINE_STAGES lists every stage in dependency order. Each StageNode records: -# - what wire types it consumes (inputs) and produces (output) -# - which pipeline.yml config sections affect it (for cache invalidation and WebUI) -# -# Stages that consume ProjectCtx do so implicitly — it is omitted from `inputs` -# because every high-level overload takes it. The graph is linear for Illumina -# paired-end runs; branching (e.g. dada2 vs swarm, cdhit optional) is expressed -# by optional/alternative nodes that share the same output type. -# -# This module does not drive execution; main.jl still calls stages explicitly. -# The declaration exists so tools (print_pipeline, the WebUI, future schedulers) -# can inspect the graph without parsing source code. -# -# © 2026 Joshua Benjamin Jewell. All rights reserved. -# Licensed under AGPLv3. - -export PIPELINE_STAGES, print_pipeline, stage_by_name - - using ..PipelineTypes - - const PIPELINE_STAGES = StageNode[ - StageNode( - :fastqc_multiqc, - "Quality Control", - DataType[], # inputs: raw FASTQs are found via ProjectCtx.data_dir - Nothing, - ["fastqc", "multiqc"] - ), - StageNode( - :cutadapt, - "Primer Trimming", - DataType[], # inputs: raw FASTQs via ProjectCtx.data_dir - TrimmedReads, - ["cutadapt"] - ), - StageNode( - :dada2, - "Denoising & ASV Table", - DataType[TrimmedReads], - ASVResult, - ["dada2"] - ), - StageNode( - :cdhit, - "ASV Dereplication (optional)", - DataType[ASVResult], - ASVResult, - ["cdhit"] - ), - StageNode( - :vsearch, - "Taxonomy Search", - DataType[ASVResult], # dispatches on HasFasta — also works for OTUResult - TaxonomyHits, - ["vsearch"] - ), - StageNode( - :merge_taxa, - "Taxonomy Table Merge & Filter", - DataType[ASVResult, TaxonomyHits], - MergedTables, - ["merge_taxa"] - ), - StageNode( - :analyse_run, - "Per-Run Analysis", - DataType[MergedTables, ASVResult], - Nothing, - ["analysis"] - ), - StageNode( - :analyse_group, - "Per-Group Analysis", - DataType[MergedTables], # vector of MergedTables across runs in a group - Nothing, - ["analysis"] - ), - StageNode( - :analyse_study, - "Study-Wide Analysis", - DataType[MergedTables], # vector across all groups - Nothing, - ["analysis"] - ), - ] - - """ - stage_by_name(name::Symbol) -> StageNode - - Look up a stage by its machine name. Throws if not found. - """ - function stage_by_name(name::Symbol) - idx = findfirst(s -> s.name == name, PIPELINE_STAGES) - isnothing(idx) && error("No pipeline stage named :$name") - PIPELINE_STAGES[idx] - end - - """ - print_pipeline([io]) - - Print a human-readable summary of all declared pipeline stages. - """ - function print_pipeline(io::IO = stdout) - println(io, "Declared pipeline stages:") - println(io, "─"^60) - for (i, s) in enumerate(PIPELINE_STAGES) - ins = isempty(s.inputs) ? "raw FASTQs (via ProjectCtx)" : - join(string.(s.inputs), ", ") - out = s.output === Nothing ? "— (side effects only)" : string(s.output) - cfgs = join(s.config_sections, ", ") - println(io, " $i. $(s.label) [:$(s.name)]") - println(io, " inputs: $ins") - println(io, " output: $out") - println(io, " config: $cfgs") - i < length(PIPELINE_STAGES) && println(io) - end - println(io, "─"^60) - end - -end diff --git a/src/main.jl b/src/main.jl index dd4eb5d..dc76ab6 100755 --- a/src/main.jl +++ b/src/main.jl @@ -17,23 +17,23 @@ function Logging.handle_message(l::_TimestampLogger, level, message, _module, gr end Logging.global_logger(_TimestampLogger(Logging.global_logger())) -include("types.jl") -include("log.jl") -include("config.jl") -include("databases.jl") -include("call_tools.jl") -include("dada2.jl") -include("merge_taxa.jl") -include("project.jl") -include("diversity.jl") -include("plots.jl") -include("analysis.jl") -include("graph.jl") +include("core/types.jl") +include("core/graph.jl") +include("core/log.jl") +include("core/config.jl") +include("core/databases.jl") +include("pipeline/tools.jl") +include("pipeline/dada2.jl") +include("pipeline/merge_taxa.jl") +include("core/project.jl") +include("analysis/diversity.jl") +include("analysis/plots.jl") +include("analysis/analysis.jl") using CSV using YAML -using .PipelineTypes, .PipelineLog, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup -using .DiversityMetrics, .PipelinePlots, .Analysis, .PipelineGraph +using .PipelineTypes, .PipelineGraph, .PipelineLog, .Config, .Databases, .Tools, .TaxonomyTableTools, .DADA2, .ProjectSetup +using .DiversityMetrics, .PipelinePlots, .Analysis ## Instantiate parameters config_dir = "./config" @@ -106,4 +106,3 @@ if any(!isnothing, merged_results) end write_combined_log(projects) -print_pipeline() diff --git a/src/dada2.jl b/src/pipeline/dada2.jl similarity index 99% rename from src/dada2.jl rename to src/pipeline/dada2.jl index 0f4dae8..408043d 100644 --- a/src/dada2.jl +++ b/src/pipeline/dada2.jl @@ -32,6 +32,7 @@ export dada2, dada2_denoise, dada2_classify, import Downloads using Logging, RCall, YAML using ..PipelineTypes + using ..PipelineGraph using ..PipelineLog using ..Databases using ..Config diff --git a/src/dada2/chimera.jl b/src/pipeline/dada2/chimera.jl similarity index 96% rename from src/dada2/chimera.jl rename to src/pipeline/dada2/chimera.jl index b021b14..164b1d0 100644 --- a/src/dada2/chimera.jl +++ b/src/pipeline/dada2/chimera.jl @@ -31,7 +31,7 @@ chimera_ckpt = ctx.ckpts["chimera"] hash_file = joinpath(ctx.dirs["Checkpoints"], "chimera_removal.hash") if isfile(chimera_ckpt) && - !_section_stale(config_path, "dada2.asv,dada2.output", hash_file) && + !_section_stale(config_path, stage_sections(:dada2_chimera_removal), hash_file) && mtime(chimera_ckpt) > mtime(filter_ckpt) && mtime(chimera_ckpt) > mtime(length_ckpt) @info "Skipping chimera_removal: checkpoint up to date" @@ -102,7 +102,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2.asv,dada2.output", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_chimera_removal), hash_file) emit("Written: $(joinpath(tables_dir, "pipeline_stats.csv"))") emit("Written: $(joinpath(tables_dir, seq_prefix * ".csv"))") emit("Written: $(joinpath(tables_dir, fasta_prefix * ".fasta"))") diff --git a/src/dada2/context.jl b/src/pipeline/dada2/context.jl similarity index 98% rename from src/dada2/context.jl rename to src/pipeline/dada2/context.jl index 9f8a2a6..f35f3bc 100644 --- a/src/dada2/context.jl +++ b/src/pipeline/dada2/context.jl @@ -13,7 +13,7 @@ # Resolve the DADA2 taxonomy database from config/databases.yml. function _resolve_taxonomy_db(cfg, emit) db_key = string(cfg["taxonomy"]["database"]) - dbs_path = joinpath(@__DIR__, "..", "..", "config", "databases.yml") + dbs_path = joinpath(@__DIR__, "..", "..", "..", "config", "databases.yml") isfile(dbs_path) || error("config/databases.yml not found. Run new_project() first.") return resolve_db(dbs_path, db_key, "dada2"; emit) diff --git a/src/dada2/dada2_functions.r b/src/pipeline/dada2/dada2_functions.r similarity index 100% rename from src/dada2/dada2_functions.r rename to src/pipeline/dada2/dada2_functions.r diff --git a/src/dada2/denoise.jl b/src/pipeline/dada2/denoise.jl similarity index 94% rename from src/dada2/denoise.jl rename to src/pipeline/dada2/denoise.jl index eb7e6c7..72e193e 100644 --- a/src/dada2/denoise.jl +++ b/src/pipeline/dada2/denoise.jl @@ -21,7 +21,7 @@ filter_ckpt = ctx.ckpts["filter"] hash_file = joinpath(ctx.dirs["Checkpoints"], "learn_errors.hash") if isfile(errors_ckpt) && isfile(filter_ckpt) && - !_section_stale(config_path, "dada2.dada", hash_file) && + !_section_stale(config_path, stage_sections(:dada2_learn_errors), hash_file) && mtime(errors_ckpt) > mtime(filter_ckpt) @info "Skipping learn_errors: checkpoint up to date" return nothing @@ -59,7 +59,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2.dada", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_learn_errors), hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "error_rates.pdf"))") emit("Checkpoint: $(ctx.ckpts["errors"])") emit("Log: $log_path") @@ -94,7 +94,7 @@ denoise_ckpt = ctx.ckpts["denoise"] hash_file = joinpath(ctx.dirs["Checkpoints"], "denoise.hash") if isfile(denoise_ckpt) && - !_section_stale(config_path, "dada2.dada,dada2.merge", hash_file) && + !_section_stale(config_path, stage_sections(:dada2_denoise), hash_file) && mtime(denoise_ckpt) > mtime(errors_ckpt) @info "Skipping denoise: checkpoint up to date" return nothing @@ -156,7 +156,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2.dada,dada2.merge", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_denoise), hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "length_distribution.pdf"))") emit("Checkpoint: $(ctx.ckpts["denoise"])") emit("Log: $log_path") @@ -188,7 +188,7 @@ length_ckpt = ctx.ckpts["length"] hash_file = joinpath(ctx.dirs["Checkpoints"], "filter_length.hash") if isfile(length_ckpt) && - !_section_stale(config_path, "dada2.asv", hash_file) && + !_section_stale(config_path, stage_sections(:dada2_filter_length), hash_file) && mtime(length_ckpt) > mtime(denoise_ckpt) @info "Skipping filter_length: checkpoint up to date" return nothing @@ -225,7 +225,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2.asv", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_filter_length), hash_file) emit("Checkpoint: $(ctx.ckpts["length"])") emit("Log: $log_path") nothing diff --git a/src/dada2/qc.jl b/src/pipeline/dada2/qc.jl similarity index 96% rename from src/dada2/qc.jl rename to src/pipeline/dada2/qc.jl index df7d058..a2198f3 100644 --- a/src/dada2/qc.jl +++ b/src/pipeline/dada2/qc.jl @@ -61,7 +61,7 @@ filter_ckpt = ctx.ckpts["filter"] hash_file = joinpath(ctx.dirs["Checkpoints"], "filter_trim.hash") - if isfile(filter_ckpt) && !_section_stale(config_path, "dada2.filter_trim", hash_file) + if isfile(filter_ckpt) && !_section_stale(config_path, stage_sections(:dada2_filter_trim), hash_file) all_inputs = vcat(ctx.fwd_files, ctx.rev_files) if isempty(all_inputs) || all(mtime(filter_ckpt) > mtime(f) for f in all_inputs) @info "Skipping filter_trim: checkpoint up to date" @@ -128,7 +128,7 @@ finally R"tryCatch({ sink(type='message'); sink(); close(con) }, error = function(e) NULL)" end - _write_section_hash(config_path, "dada2.filter_trim", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_filter_trim), hash_file) emit("Written: $(joinpath(ctx.dirs["Figures"], "quality_filtered.pdf"))") emit("Checkpoint: $(ctx.ckpts["filter"])") emit("Log: $log_path") diff --git a/src/dada2/taxonomy.jl b/src/pipeline/dada2/taxonomy.jl similarity index 97% rename from src/dada2/taxonomy.jl rename to src/pipeline/dada2/taxonomy.jl index 359f255..6ece2d0 100644 --- a/src/dada2/taxonomy.jl +++ b/src/pipeline/dada2/taxonomy.jl @@ -132,7 +132,7 @@ checkpoint = joinpath(ctx.dirs["Checkpoints"], "checkpoint.RData") hash_file = joinpath(ctx.dirs["Checkpoints"], "assign_taxonomy.hash") if isfile(checkpoint) && - !_section_stale(config_path, "dada2.taxonomy,dada2.output", hash_file) && + !_section_stale(config_path, stage_sections(:dada2_assign_taxonomy), hash_file) && mtime(checkpoint) > mtime(chimera_ckpt) @info "Skipping assign_taxonomy: checkpoint up to date" return nothing @@ -161,7 +161,7 @@ min_boot = get(ctx.cfg["taxonomy"], "min_boot", 0) # Read levels from databases.yml (single source of truth). db_key = string(ctx.cfg["taxonomy"]["database"]) - dbs_path = joinpath(@__DIR__, "..", "..", "config", "databases.yml") + dbs_path = joinpath(@__DIR__, "..", "..", "..", "config", "databases.yml") dbs_cfg = get(YAML.load_file(dbs_path), "databases", Dict()) tax_levels = String[string(l) for l in get(get(dbs_cfg, db_key, Dict()), "levels", String[])] isempty(tax_levels) && error("No levels defined for database '$db_key' in $dbs_path") @@ -224,7 +224,7 @@ emit("Log: $log_path") end - _write_section_hash(config_path, "dada2.taxonomy,dada2.output", hash_file) + _write_section_hash(config_path, stage_sections(:dada2_assign_taxonomy), hash_file) emit("Checkpoint: $checkpoint") emit("Pipeline complete. Outputs:") diff --git a/src/dada2/taxonomy_remote.r b/src/pipeline/dada2/taxonomy_remote.r similarity index 100% rename from src/dada2/taxonomy_remote.r rename to src/pipeline/dada2/taxonomy_remote.r diff --git a/src/merge_taxa.jl b/src/pipeline/merge_taxa.jl similarity index 97% rename from src/merge_taxa.jl rename to src/pipeline/merge_taxa.jl index cc4d4b0..adebc8a 100644 --- a/src/merge_taxa.jl +++ b/src/pipeline/merge_taxa.jl @@ -6,6 +6,7 @@ module TaxonomyTableTools using CSV, DataFrames, Logging, YAML using ..PipelineTypes +using ..PipelineGraph using ..PipelineLog using ..Config @@ -312,10 +313,10 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa @info "Filtering table using configuration from $filters_yaml_path" - # ---- mappings (auto-detect old vs new format) ---- + # mappings (auto-detect old vs new format) _apply_mappings!(df, get(config, "mappings", Dict())) - # ---- remove_empty (backwards compat with remove_empty_domain) ---- + # remove_empty (backwards compat with remove_empty_domain) remove_cols = if haskey(config, "remove_empty") val = config["remove_empty"] val isa Vector ? string.(val) : [string(val)] @@ -337,7 +338,7 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa end end - # ---- filters (with action + regex support) ---- + # filters (with action + regex support) raw_filters = get(config, "filters", []) @assert raw_filters isa Vector "filters must be a list in YAML" @@ -437,7 +438,7 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa function merge_taxa(project::ProjectCtx, source::ASVResult, tax::TaxonomyHits, db_meta::DatabaseMeta) config_path = write_run_config(project) - cfg = get(YAML.load_file(config_path), "merge_taxa", Dict()) + cfg = get(YAML.load_file(config_path), stage_sections(:merge_taxa), Dict()) filter_list = get(cfg, "filters", [nothing]) merge_dir = joinpath(project.dir, "merged") hash_file = joinpath(merge_dir, "config.hash") @@ -448,7 +449,7 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa data_mtime = max(mtime(tax.tsv), mtime(source.taxonomy), isfile(tax_counts_path) ? mtime(tax_counts_path) : 0.0, isfile(boot_path_) ? mtime(boot_path_) : 0.0) - config_changed = _section_stale(config_path, "merge_taxa", hash_file) + config_changed = _section_stale(config_path, stage_sections(:merge_taxa), hash_file) merged_csv = joinpath(merge_dir, "merged.csv") # Determine what needs to be (re-)computed. @@ -534,7 +535,7 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa log_written(project, output_csv) end - # ---- DADA2 filtered CSVs (same filters, applied to _dada2 columns) ---- + # DADA2 filtered CSVs (same filters, applied to _dada2 columns) # Check whether any _dada2 taxonomy columns exist before attempting. has_dada2 = any(endswith(c, "_dada2") for c in names(df) if any(startswith(c, t) for t in db_meta.levels)) @@ -571,7 +572,7 @@ export merge_taxonomy_counts, filter_table, filter_table_dada2, merge_taxa end end - _write_section_hash(config_path, "merge_taxa", hash_file) + _write_section_hash(config_path, stage_sections(:merge_taxa), hash_file) return MergedTables(tables, filter_stems, filter_colours) end diff --git a/src/call_tools.jl b/src/pipeline/tools.jl similarity index 97% rename from src/call_tools.jl rename to src/pipeline/tools.jl index bd484e4..9f2a7e8 100644 --- a/src/call_tools.jl +++ b/src/pipeline/tools.jl @@ -9,6 +9,7 @@ export cutadapt, vsearch, multiqc, cdhit using YAML using Logging using ..PipelineTypes + using ..PipelineGraph using ..PipelineLog using ..Config @@ -34,7 +35,7 @@ export cutadapt, vsearch, multiqc, cdhit Paths with `@` are SSH remote paths (user@host:/path), the calling module is responsible for routing those calls via SSH. """ - function load_tools(config_path = joinpath(@__DIR__, "..", "config", "tools.yml")) + function load_tools(config_path = joinpath(@__DIR__, "..", "..", "config", "tools.yml")) isfile(config_path) || return Dict{String,String}() data = YAML.load_file(config_path) tools = Dict{String,String}() @@ -275,7 +276,7 @@ export cutadapt, vsearch, multiqc, cdhit if isdir(cutadapt_dir) trimmed = filter(f -> endswith(f, "_trimmed.fastq.gz"), readdir(cutadapt_dir)) if !isempty(trimmed) && - !_section_stale(config_path, "cutadapt", hash_file) && + !_section_stale(config_path, stage_sections(:cutadapt), hash_file) && all(f -> mtime(joinpath(cutadapt_dir, f)) > input_mtime, trimmed) && all(f -> filesize(joinpath(cutadapt_dir, f)) > 20, trimmed) @info "Skipping cutadapt: trimmed reads up to date in $cutadapt_dir" @@ -285,7 +286,7 @@ export cutadapt, vsearch, multiqc, cdhit mkpath(cutadapt_dir) run_cutadapt(get_primer_args(primer_pairs, primers_path), built_args, project.data_dir, cutadapt_dir, cutadapt_bin) - _write_section_hash(config_path, "cutadapt", hash_file) + _write_section_hash(config_path, stage_sections(:cutadapt), hash_file) pipeline_log(project, "cutadapt complete") return TrimmedReads(cutadapt_dir) end @@ -380,7 +381,6 @@ export cutadapt, vsearch, multiqc, cdhit @info "VSEARCH running: $fasta_in_dir against $(basename(reference_database))" # --userout with query+target+id captures the full FASTA header (including # description after the space), unlike --blast6out which truncates at the first space. - # This is required for databases like SILVA where taxonomy is in the description field. cmd = "$vsearch_bin --usearch_global $fasta_in_dir --db $reference_database --userout $outfile --userfields query+target+id $optional_args" _run_logged(cmd, log_path) @info "VSEARCH complete. Output: $outfile Log: $log_path" @@ -408,13 +408,13 @@ export cutadapt, vsearch, multiqc, cdhit tsv = joinpath(vsearch_dir, "taxonomy.tsv") hash_file = joinpath(vsearch_dir, "config.hash") if isfile(tsv) && - !_section_stale(config_path, "vsearch", hash_file) && + !_section_stale(config_path, stage_sections(:vsearch), hash_file) && mtime(tsv) > mtime(input.fasta) @info "Skipping vsearch: $tsv up to date" return TaxonomyHits(tsv) end vsearch(input.fasta, reference_database, vsearch_dir; optional_args=built_args, vsearch_bin) - _write_section_hash(config_path, "vsearch", hash_file) + _write_section_hash(config_path, stage_sections(:vsearch), hash_file) pipeline_log(project, "VSEARCH: $(input.fasta) against $(basename(reference_database))") log_written(project, tsv) return TaxonomyHits(tsv) @@ -469,13 +469,13 @@ export cutadapt, vsearch, multiqc, cdhit new_fasta = joinpath(cdhit_dir, basename(input.fasta)) hash_file = joinpath(cdhit_dir, "config.hash") if isfile(new_fasta) && - !_section_stale(config_path, "cdhit", hash_file) && + !_section_stale(config_path, stage_sections(:cdhit), hash_file) && mtime(new_fasta) > mtime(input.fasta) @info "Skipping cdhit: $new_fasta up to date" return ASVResult(new_fasta, input.count_table, input.taxonomy) end new_fasta = cdhit(input.fasta, cdhit_dir; optional_args=built_args, cdhit_bin) - _write_section_hash(config_path, "cdhit", hash_file) + _write_section_hash(config_path, stage_sections(:cdhit), hash_file) pipeline_log(project, "cd-hit-est complete") log_written(project, new_fasta) return ASVResult(new_fasta, input.count_table, input.taxonomy) From 3a2731805ee2ebc3a6a8d7882a4544c9211bc540 Mon Sep 17 00:00:00 2001 From: JoshuaJewell Date: Fri, 27 Feb 2026 18:33:14 +0100 Subject: [PATCH 032/175] Programmatic testing, yay! Brought back in SWARM pipeline. --- .github/workflows/ci.yml | 62 ++++ .gitignore | 7 +- README.md | 3 +- config/defaults/pipeline.yml | 9 + config/defaults/primers.yml | 32 +- data/MiSeq_SOP/F3D0_S188_L001_R1_001.fastq.gz | Bin 0 -> 825014 bytes data/MiSeq_SOP/F3D0_S188_L001_R2_001.fastq.gz | Bin 0 -> 1087728 bytes data/MiSeq_SOP/F3D1_S189_L001_R1_001.fastq.gz | Bin 0 -> 648456 bytes data/MiSeq_SOP/F3D1_S189_L001_R2_001.fastq.gz | Bin 0 -> 822580 bytes data/MiSeq_SOP/F3D2_S190_L001_R1_001.fastq.gz | Bin 0 -> 1975582 bytes data/MiSeq_SOP/F3D2_S190_L001_R2_001.fastq.gz | Bin 0 -> 2723348 bytes data/MiSeq_SOP/pipeline.yml | 29 ++ install.jl | 44 +++ src/analysis/group.jl | 56 ++- src/analysis/helpers.jl | 6 +- src/analysis/run.jl | 130 +++---- src/analysis/study.jl | 36 +- src/core/config.jl | 17 +- src/core/graph.jl | 9 + src/core/project.jl | 119 +++---- src/core/types.jl | 16 +- src/core/validate.jl | 323 ++++++++++++++++++ src/main.jl | 46 ++- src/pipeline/dada2.jl | 4 +- src/pipeline/dada2/chimera.jl | 4 +- src/pipeline/dada2/denoise.jl | 6 +- src/pipeline/dada2/qc.jl | 4 +- src/pipeline/dada2/taxonomy.jl | 6 +- src/pipeline/merge_taxa.jl | 140 +++++++- src/pipeline/swarm.jl | 230 +++++++++++++ src/pipeline/tools.jl | 44 +-- test/integration/test_pipeline.jl | 151 ++++++++ test/runtests.jl | 55 +++ test/unit/test_config.jl | 37 ++ test/unit/test_diversity.jl | 55 +++ test/unit/test_merge_taxa.jl | 117 +++++++ test/unit/test_validation.jl | 101 ++++++ 37 files changed, 1625 insertions(+), 273 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 data/MiSeq_SOP/F3D0_S188_L001_R1_001.fastq.gz create mode 100644 data/MiSeq_SOP/F3D0_S188_L001_R2_001.fastq.gz create mode 100644 data/MiSeq_SOP/F3D1_S189_L001_R1_001.fastq.gz create mode 100644 data/MiSeq_SOP/F3D1_S189_L001_R2_001.fastq.gz create mode 100644 data/MiSeq_SOP/F3D2_S190_L001_R1_001.fastq.gz create mode 100644 data/MiSeq_SOP/F3D2_S190_L001_R2_001.fastq.gz create mode 100644 data/MiSeq_SOP/pipeline.yml create mode 100644 src/core/validate.jl create mode 100644 src/pipeline/swarm.jl create mode 100644 test/integration/test_pipeline.jl create mode 100644 test/runtests.jl create mode 100644 test/unit/test_config.jl create mode 100644 test/unit/test_diversity.jl create mode 100644 test/unit/test_merge_taxa.jl create mode 100644 test/unit/test_validation.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d5f6ec7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Julia ${{ matrix.julia-version }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + julia-version: ["1.10", "1"] # 1.10 LTS + latest stable + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.julia-version }} + + - name: Cache Julia packages + uses: julia-actions/cache@v2 + + # R + required packages (needed by RCall) + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: "4.4" + + - name: Install R packages + uses: r-lib/actions/setup-r-dependencies@v2 + with: + packages: | + any::dada2 + any::vegan + + - name: Instantiate Julia project + run: julia --project=. -e 'import Pkg; Pkg.instantiate()' + + - name: Run unit tests + run: julia --project=. --code-coverage=user test/runtests.jl + + - name: Process coverage + uses: julia-actions/julia-processcoverage@v1 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + slug: JoshuaJewell/MetaManifold + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index e9612e8..3f38bb5 100644 --- a/.gitignore +++ b/.gitignore @@ -253,7 +253,12 @@ rsconnect/ ### Other # Things with ambiguous intellectual property internal -data/ +data/* +# MiSeq SOP integration test dataset (3-sample subset, public domain) +!data/MiSeq_SOP/ +data/MiSeq_SOP/* +!data/MiSeq_SOP/pipeline.yml +!data/MiSeq_SOP/*.fastq.gz ### Lock files .~lock.* diff --git a/README.md b/README.md index 048664c..3513858 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![License: AGPL-3.0](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](LICENSE) [![Julia ≥ 1.0](https://img.shields.io/badge/Julia-%E2%89%A51.0-9558B2?logo=julia)](https://julialang.org) [![R ≥ 4.0](https://img.shields.io/badge/R-%E2%89%A54.0-276DC3?logo=r)](https://www.r-project.org) +[![CI](https://github.com/JoshuaJewell/MetaManifold/actions/workflows/ci.yml/badge.svg)](https://github.com/JoshuaJewell/MetaManifold/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/JoshuaJewell/MetaManifold/graph/badge.svg)](https://codecov.io/gh/JoshuaJewell/MetaManifold) A Julia pipeline for amplicon metabarcoding from raw paired-end Illumina reads to filtered, taxonomy-annotated ASV tables. @@ -423,7 +425,6 @@ The following colleagues at the **Department of Parasitology, Charles University - `merge_and_filter_taxa.jl` - `dada2.yml` - `protist_filter.yml` -- **Bc. Ekaterina Kandaurova** - designed the primer pairs in `primers.yml`. - **doc. Mgr. Vladimír Hampl** - provided laboratory access and resources. ## License diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 977d02b..bee86f5 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -81,6 +81,15 @@ cdhit: threads: 0 # worker threads; 0 = all available (-T) optional_args: "" # additional flags passed verbatim to cd-hit-est +swarm: + differences: 1 # -d: max differences between sequences in the same cluster + threads: 0 # -t: worker threads; 0 = all available + chimera_check: true # run vsearch --uchime_denovo before clustering + min_abundance: 2 # --minsize: discard singleton dereps before clustering + fastq_minovlen: 20 # --fastq_minovlen: min overlap for paired-end merging + identity: 0.97 # --id: threshold for mapping reads back to OTU seeds + optional_args: "" # additional flags passed verbatim to swarm + merge_taxa: # List of filter configs to apply, each relative to config/filters/. # Each entry produces a separate filtered CSV in the project's merged/ directory. diff --git a/config/defaults/primers.yml b/config/defaults/primers.yml index b3e5a01..b42f08f 100644 --- a/config/defaults/primers.yml +++ b/config/defaults/primers.yml @@ -1,30 +1,12 @@ -# Kandaurova, E. (2025) 'Metabarcoding of gut protists of Eurasian beaver'. -# Owens, L.A., Friant, S., Martorelli Di Genova, B., Knoll, L.J., Contreras, M., Noya-Alarcon, O., Dominguez-Bello, M.G., and Goldberg, T.L. (2024) 'VESPA: an optimized protocol for accurate metabarcoding-based characterization of vertebrate eukaryotic endosymbiont and parasite assemblages', Nature Communications, 15(1), 402, available: https://doi.org/10.1038/s41467-023-44521-3. - Forward: - TarEukF: "CCAGCASCYGCGGTAATTCC" - ArchF: "YGCGGTAAYTCCAGCTC" - VESPAF: "AGCAGCCGCGGTAATTCC" + # Earth Microbiome Project 16S V4 (Parada/Apprill, 2016) + EMP515F: "GTGYCAGCMGCCGCGGTAA" Reverse: - TarEukR: "ACTTTCGTTCTTGATYRA" - Meta1R: "CYCCTACYYTMGYYCTKGA" - Meta2R: "DCTKTCGTYCTTGATYRA" - VESPAR: "TCCGTCAATTYCTTNAASTTTC" + # Earth Microbiome Project 16S V4 (Parada/Apprill, 2016) + EMP806R: "GGACTACNVGGGTWTCTAAT" Pairs: - - TarEuk: - - TarEukF - - TarEukR - - Meta1: - - TarEukF - - Meta1R - - Meta2: - - TarEukF - - Meta2R - - Arch: - - ArchF - - TarEukR - - VESPA: - - VESPAF - - VESPAR \ No newline at end of file + - EMP: + - EMP515F + - EMP806R diff --git a/data/MiSeq_SOP/F3D0_S188_L001_R1_001.fastq.gz b/data/MiSeq_SOP/F3D0_S188_L001_R1_001.fastq.gz new file mode 100644 index 0000000000000000000000000000000000000000..cd31624bc46a64f07294db27adb0abb49ed17b06 GIT binary patch literal 825014 zcmV(^K-Iq=iwFn+LsU@!14c7MFke$KI5=NSFfcJ+QZZjJFflG>VRLkG0PI~^lcPMA zegA$%N6g~{Lf$RNG-xL#V)}{y|6AtXs|B&xCM&ahx~p1Ml?EihX&s+??joxHU(Ut# z)|^ibPpy)t{~3I9XbuMsKkldFtvOtU_}iiRd&A+t{ja}E?Y)G*qu21N{W!`|+3cJfu`oDd(JDv>~*#<%e{9)*VKK-Kk5#rb)-k(eVns!O=DhS8!{a?#hg_E;OUv z(`Mk0!;vtiqit%k7@|x#Y@G}n3VUUTmxtf?ul_w?&ipC>CC6IGB z!<&Wq*5SqxF^C?xAH7|~@f|G0%OWZfd|~fMEn5e z0`agtfH@sc2$=in8w{fZU~o>x4ThoVkAQ*sBPuBw1}`>7ywL4_kzvLc3`2L1c+qrv zVi@{uB)+N0FiI;;?FnGEOp-5qXC^7`&G9iC?$goLnQ=4RJ|CF1xzT)PtmJAVf@j(! zK*jY@pusnz~J{5zpQcv82ZYN6T^&&VLZ)81dJRL!{FRZfT4?On(&MmBP5#v{GZ^$0yX7={1?c!5t+8_)tz0D}l{VFGPQHNiY3k_mHMJDS%pUUu4( z*}+bydInT?||1*O10wtxs&O4on7WP2tJ6kx)d)>S>x zje}pCwW5PDyBJ2CZujPRIlnPUe-Ojy)Sfa--tu!VhAQ3znA`PsZjQIxl3{)YFbIbdF!-e%^-A>OrDlL307by)2{8WKfYBK+ z{t0088Ze+XRSk0xbs(Wyyu=&aR#KJBFu8GJNF@gOjS9fHvDB!E3JOupJK z2)u6HY)_U_S-HuJav<+(x!WY&`CR~D?uQMN)GL#e#AlrFS?^XHX4-ZW05;<=>F}(r z0MXm!p=H~Kl6E12V@&D?*T{A)(2GDIS;hKY3?nYrLvuQue^#ItFIP#}E)D7mTCk%%`NH_HegZtl`BpkJEP>3rD@m=_v`CNgLk z88u9FxF^5_y)x4!1yl%MxlrSqaWVuCSc$*-_kH{;U7;JLF|=;w1JcY6RD|bZ$0ZvOY;IB-PX~n z0zIP)1FT5aVyuSk5xqwM2yafe!;)d117<8z12Vd-9#Ud) z#xOcNQPYHnty+b|PzgYoF`9PtolM9?I^%81P-{0Mn}mrFZu^QU!ccCpH=(DE38Y|- z%K$PGGq*a^N;;Z%T;9?X5Q@x_OQGC2#?S68O~TE>ODd70 z%0p|pyJ|SZlJZSTxY`34ak^a*FXv6XjL$667G5akPZntmqLpSg@_LNT)AR+;Xq=MO z!AqjJ9iAH;t0K}%R%a71u0BjSiXM_nM@+zo1SD~tbKE865HuFZdV-A_ zy=#_AVJxZ@8K_D!_NgQj26IunBn-dY&d`56{>hmESrW{Fcu{tg8E~ELtd*b(xV2Ve z#e{eT2cvpQeSe$=+z{!9!r?JEa zIywQjhNZ{(?PBvX%QRu$wyDHlkRo95=TFAcd1GjuQg*ivjPgv=zR#cUEzzci@ zl&avIn&?D|N{FiM`>KXcsVcF{B3%wgj)2*6l8R(?E{EB2l44q!jFMhxUP6a2oTM>k z2FheLVkkRFy6~!}%)k@PI})Wm)4Vc!)qZl4{2R?nZ~@DHP%?dYZQv5_4q=*oa*_}i zwS`HwzQ*=|{)0O@{|IDsJUBS0w68T?0lvrlfedrNzdR@#oRtqlJObNoU}OL*Y|G$z zc%Jg+y_`Ir4y{T}vkx1cN_MD609gP+AIqu;H8ubRz=OH#O7ws}sQ`d58jeACSM34J z;V8f)o$pHt^A*5k2_p*$ldBuXSluwj>V{Crq-CjL%51gt(V~rh0*rs*RqN-eBpoSC zkK-uk8fGlB)g@q}e4)REF)B?G5=U9sGR5tR8kf0~v^ttSAU_|R`>dPJ42jCg-SW$( zNFz^rE{k*#2LOy%TO|4<_lE6! z2;0RI7EzcPV0;YQ1>nWZKpD2vF&~?2Y`zh;%jHva+7`iVM`40acpA3z+dfj-o^y)_USQ~z&j!H6ZaHW_~Ml_$W@kt5nJp?1T5Y731`O_-E z_0vQWu!~+V8Ay0B1zuu(U1%DssKdk?f9gQ!QiaPYb;tNzt<6Uy4M2@C)quoM4qTa*3hPu2T%rU_ovJ14-NV)3^i-M(>stA67Ca&m zU*>;*#cqrAZ(eRE-CzNI?-}ms^3g>&oi|YFpi6m%a)%DSZQXr4(7XE;5 zrIba#s1~BFWCdEd!Z_X@z#Q)4cm%+lmQk2rNhAfcv`QpJ2{X=pq#}_-VgFn`l?r-) zH1jCGkVry`d6O3ia%GtJY9C|0^X6w~uKQOuX_zYTHm?)cpuixL}rC%tFGy8mVW39JVg{SE1%xXUT&? zg$tk&ADBjUs_I#*R1H)hfEvW36+PCiz&>LY1G-Q(_&FGOLjwdJw^9uJ=HA#!flrP5 zVCG;bV`#pV=_CU=1MH(J=S1Czs(Hl}hK#IaVFbZ>c=)Bq|GYr`j&0X}8Ks!V$Q`e79aYfTkwuY>eden!5N>rgZ zeCRb-4z-Yvh&mqo!lycV6_zn!Y)7Yiy5W$#qyXB`1$Ihl?%{_EiETN_uD+{;s1u*0 z3_>L_U}G(qsSYg_?)823Nq*FxD9rJ2htA9C`V*PQt@=#=v_3P;e9#zEoOeU?j&V{G zUi?hMjHT$+Z|gHjyadtfQ&5c1Hf0`BiQ*4;ep{cZ;sjX&%#^J@0k8Bhou|&99`y&j z0XkfFAD|#KY8@B6<*?mD2NSJC4iLO*n~G7VXw!T) zj6pB*1Dq)o46oYCFl;SvLF-m{mVm#Ew>1a0!P~*lrbp_bFffjlFlHBEZv5C>&gU%+ zv&}GNZA18|b^bn&TeS@oSWC4H#3DX<)l+6*UE3g2SVq67Z5U-lwlnX-c0Tb#DIPY&$XvQ$6(_0JV6WV)dl}QqrFp5Pir_-cq`{N&DT6ez ze&OR)9#X&;<3j}k+GCQg;&ubP+`m~oMH4xWg+-DXFU!nhE}oh!5}`r{7v~>WZ6|j{ zVUk6X2jH0}cmbwI_d2hY4@dyh6CeP}W;ozFGo#DOO+SfNGEy$2CJ*F%8A@j2YBvV= z?mj+@;U=Djw=p3C$)pYIQl#u1=j8c=W26vV%7SC(Z42f`l5dm~N>$xb+Q0&hsLt66 z(*Y!nLBjMOvM0ZKmqp^IJ4l%8x7H*Tr43tNHEEIz7`gJQ0`8D5rFBx4|%l6-|2aPb&(P54N2p4Sn*Dx8uT0~>u4 zY&xuNFnJuq6>VZCUc~JP0_Jx8PSo!ADxG?}NVX`Qn)mvMmYLE(x5k+H@EM7eTKd^~ zfdzVYwwDD|1_5I_qg|@C+f_PsJYk~xa=ZQZc>y19^?rd#xn3Ykm^EPhmO*mPl^~d4 zLbL-i#q>rFW%w?RwXW(KrnT*rmO?vRO_Y3Oq=GG8sUsqaX*UR0B$5h$yV!gmnLZ<-CY)lE4qOzfoG&7HDo~QI?9!JhCKak4o<{Q;L z6F8=p4FU_?@CK~#uEgW%bZ9PjzUd;#FIK`tAAAfC0U{2H8aLr3_~~ObLYZJT3SoXQ z$orxYW`c{J5e%t9HO1@}iWeQ@1;M!3zu{BoQ)%&i20`Ibm+MT1SP+i?v}eBYnl~Ct zs&ubJFCB*GjA$gof-2bN4d@RHqcG8i_`U>H0?rU?rK+*8N=7bHKrAp&+D5ijD_PB& zrZ*sY;bhg>eTm2O6#xUU`MuKuI~FA2``fj=WXLtQ!&YyJl(ov44_wDxil#dA`ALtR zQ>t-bJlv-dp|0PVM)oWkBzgt|tAQ%I7T1b24~?+-q!I?@m=sL#rr`=;W$BqP@QYWK zfWr-wm65DxtgiPx7Uq6AHdnq1!Tg>AX3q*)%p$2+%R5QlAu>V{q_usrBpbsl~Y6XLt1m-mUjhC5% zS;D0hxkX=MoyG7acF_CDMDwlPD6@$6o&dv39yZ;%bc6**|_jQZ(0x)r} z>>Ac|lz*)0D8kII?zYi>2{T38C1Eaipc%gMsRLm2Yrx3gkzsZ}6-E!G>d2Opr5m0vG}A|T zXS?4cvA=CUsc)#Cpnm;Wu;wMd^82^ zbf|23QE8`BT|WF*A&RQ)#C_!f5(rUp}0-v>9s~v*el?+blsc z15UnQ+9?N?5b_)CJ$%}@&P3C;3eUM?JsjivN;9wGz$sb%BRDwzNDO zZ!rL+83SGp|Mc*cucY(3(Y5Cv+NEccVXdZC#p5a4^Rb!@T%BVJxL; z4F>&@gfan6w^g{mh23 zV3<&w`F#yjFw7ne6IMKi|6t$?8pdy{lahrBfNc!P&7~{3weuaG+M38s+QpRuoThhN zm>-wc#6rWzGjzx@p9-I5Mm`e5dWU@)<_4_+;k#AXu#K0p_FLyvArTCWeqQ^nQ|))i zF3DiGidx=RwcozPi=X3Vv-Ue$q^b6Mxw<4-B$@Eyb72Ds!8e+BmJY(#9?guh9=^c6 zHNXgGNB`#p#?x^66F zNS_OmW!MfjdgEukx!?*P`+DAzS$-al)C2C6SUnVY%1pY7@r1W6R?< z`P<#)nEbskF#7i;fAbdrQ%Tv5p_-y$m)68UL`>gCTX3nk;uSS7MIQ#4P+4$Yi{&ib z`iQJTmV=osNPN9*R0}ImC71{u1DfTLn)BJ@)Yo=h=6%H6%qK+{I)}&Z4S46GA`IRc z+xo)*6K^;UyE&I+WzFc4YBv{Q9$zlPtf+Q0lgo?8j)hS&D#8SNvE(Ww*m~h`-|Q*N zbF;IVkEXO z`mNkdO!jm80{nAXyRft`IGa6W#5Zhy>{o{-H$1*3{?4cx4Tt8~UF+sM)P|oG^Ja3V?EOy8E`QBq z1S|lPqR)MTn$(h(>5dm1(BfG7E|YFYJ7pIqGXQhCUQgY%`PRJSZ^>++U!}qvP|7*T zlsUSATRCoTFb1xe9XFu(B9Vw7?Rd0u#Wi=a1j%LX6dUY|pwrB)<-iEWwXHkNXFuYx zYVUE#cjlqR$es*-bWx1dFBK>|Hz7%xxN|xrt!ig#a9JOyoiY1)G%Mz9j->6bhtqeG zB>q|BktcxJYCP&y%&U_s%no2^e&Z4IG3;bnsYVMJcPwSf21>n&g2cd7l~bLn zEBHIdQ`KE6H!;%c#ylC!6Z2QxfHbc|Om(YnLv_0q6naIa22tJ{XnYVzY^uI8*-Afx z#76;i{$eyQR~2f2nA7G_Z_a_-7=eF5aIESHE#MtRPeJZjOhu@kqz|)g_PgcWXTAH_opCLrEOy?mUtYKk!wl(+G1jg7F?~ zdOE)*<~Inn_<%h>mC4GKtEU2`@8b0qc`{{(i?J}rx6pZi2s;yxyfRFBEO8P&+d@$c zyyZLNKzi}Q@_Q~3^uocsld|3`E2};pj!?X7moe{))COuTOe7ZXEsU+YI1wRR((@KZ zr$KL=RqHn=Nl_yy9$?d37KVK~!8W;NHmjI-l_w<;aFl_s@v}Bc_K+!>LW@8wlo0Jc z5>#_81f`Khk+FS|_)wNGOdwTxFN<*$l$B+lXJO2LhXYo7-@=7?{NY@fyp3AZ=AA|^{ZZPe{v z0*o(jtSro~_;+GqJdIL2i{R^a*%oH6RqZVd{d`3BB@09KbPA0hq;!jeO zV6rU?f9jH~g@OB~^#U+_0T@iAb!=fAVDzR-S_d&Y1Tm3?u{c?~B+S;I0LIk?T+j(acTVglD{yQKyhDl->`Y8S#_k3c&v45hZ_ z3#^0C7@(msK;L3u9zKMjJ6srx7hCp4yd2ESAV3Pm7_D2$QGl08&~TS0AiVGct%2ks zWP|IG)P(Zil~SWTGA89DE2fY7$PTJDZ$n-U7Ud|6p&ZN$x^0_g7!-%ZsL_oHNLT~% zz6drd;r9?S?y@g1?**a%S8XX96-n!>WRgr=96pS2m9%@LoYl$?XuIoe?)*&gZKT$_#@51#;L zM`_h2-GObVtu=Y3v`Sq#?8ir_&5*W~R(Zs>qhDm(={&aGBVyZufnf@yn^5^F2s8M5 zW&Sf3K&H#`&`L=>%|QK9Af0X!$}StQ{9mamSnS7I^UK}ZbVtEbw~CL@%h6bdVZ!{4 z0CaFathu^#`4xOnHiyc^_%#QLhY4 z62WW~dhKNhyU&v9m4TrPa>Jr*XSWON@4MMqQ@xoaCHBPC_?8xEIhEv1h|+^_I(Z|k zjx^vZ0hOSu3UI3l4ixZ#mb^4LEkQa)ieiKj0AV~0DG?tPRh1$@(iFxE(W>j<5k=%s z(HC9O(JF6Eb<>z|*8t|SnIm}%l7uj^feC=Il2Z?=9i11cnzk|EZ zp}0R2Z0&Iv)-@X)?=oJ2OvBjjM$GIQMHzu}jz>D3j@OP{$$P}&f5He`q=d+Tu&q%K zG3KG{wW?!)cxgI+QG^3LEbUGOhQyS7pmJ0OdmnodWy$5nZU3-RiPTQIiDWT#a7CaS zR0`O)1X8P%3c*a=%L_C!fE}I%nD){hIzm<>Y(I|MkgYt0L)BmBHbf?3Wi4iVw9slRx9axyuhT%;T=&-E(iSgMhx1Q z%x?9kHU>|L$R|at&mDWjb=ag6nVDU?@ioz@_#oj?P84%JoeqGQ&C;FDXeMb}i3z;{ z?fFUCmtr(`rT(zE&(-d1ClXU@%%XjBAZ|Z?w>dDkA6NZqzD$+O(^#LEvZx`v%#*Cm zu>KagcN}@)YGQms2A!CLE@pCaQw~k@8JR1&dFxNd)}~tqU+NwtrM} zWzyksfHJKT-Wj z$lNi;jblUMEPNR51r0A}F{qj|#J$(I*ciLP(e|sO?!Xc;dY&_JE(z~&!iZT`d%tF5 zv~^|P0>(B*FJ~Y1Y-6Z(N^GB``K(`^aGu;nGP<76=hH`NaE#7Lk7Mc04Pykbc=qQ6t~1x6o>vMY_fsR%8pu`H@2umX^hiWWWN z3V6Y=IW&tAb2%84)a3Hxi!u4b)pVd4#h7ScNp5aEi^;?`W**PB9~iQu4O8>U@xf-s zWok>J!|`~b2@cJK_*B-q>T`2~hpT@w>TrZ9%J4U&&aVJuLYx!}Bp+`4J& zt^LFL@642}pwRdUP1CdJkQajwEQYMclvFYaV`h6Z`Jp5WrwC;6bc&QmRV~u=!gD zG-2gl@M#a8qiVo|3xJjA)f)n_$ru2Y3 zUHXnRj?6ks1gLc$455M3m@FjO42{qlAFh~mgD>EZsxq8_KVe9~z~(;Ams6X9XR#dG z13?>eU3Lh*k|{j`FXoYzDgEx%m?xRi&alZAcOk;j2BtQSP{~_g7=6^vB(^s<`(DT$ zeKeXR?Ux~MBA=Yi!&CbX8>4Nku&m!%AM*%7%*w`C61n$Ek6(e97lsF@`IQ5n=AKv5 z0WitV>e$B21-b=wCPmfGfkJ0incjbp^qLPaisBwATGmQCA0^g;h|dlaWN{Ra5kzqH zQFc8bDQ9A~ZR#T;=KP&Z>1RzqX0IDqWlEdIz&_hB73+{Udgt7cbXnC_7jtc4zIkLJ zhMZgTaJ|taUsxz0CL~I+PR#TAU9d5m^}9S9^TtHU*%;0}8DIx7e*At5F)ITSpIad- zhRmuaxOH+bB}d?F*6*2t{L}ToEn+Yd^c7##TK1io zf7Q*uwfTedaQobScFJ{9`Fkv9%uN%ssgsgo9uf$k5)khSOnp$^n2RZrjZyAD^J6m0 zc~}r;(nOoPeI`B~>hIB&A{xYZZyZJe(Q#I;L~1C;TvvMFuK>muV{FnBQYD_WA)l=Y zwFKXd!1tS}5)H}q3sO`A4iLar6X5v*GeK&4Vy&c72q?C2j$$%u;tgvWI>78wEe9M{ zI*#JN(9d*T5m>w*IeK4Mfa}nj6HSj6e7; z1)&UPji7~CF*c4M8*lDyMREJ5$%T4m{&nKs7@2hB1KQT?R`lawzC9^H1F<0RzOMfO zZN#zz1rSC+>Q40^Jy1AVUaHidn=_i2cAHGaKMiBNiFx4JxefX7PU-P*7Q~9Rgeq_{ zqeN%%GB;?VUGdiS|>up(hgM*v%63H(|j-#Ni1vIiOd_bCKm{Nug7h~Jobi^wk4jCst-`G zMnY4!1}8WrTd{Yu)p+Y{y;ldP1n;#NdxQd>jj1*xUktSoW-g3bAm(95FLAUk4P$nP zPP|}m4q{U2p(G8`l0|q~b(pmBH9lX6S@i{&<9nFuwIssvbT$HSe}hBXQA#}H@}P%{>L$VUvWwsOb4CB z8`>&-Z&{d-ibtQkX<~pR)E2nh#Yx$Ot(eXROG7~(a?3)LyHRzng z{5@;ekTlQ7SxVkoqA+IQpGQU}f1!ofk3Dmx=y{FSBlS=yGHojV-oaTDVDj`U{WOQXOSe@YB|ZLUP4J|s4x z7H-Mum_a{}jg;9gp@}^ueq&%y9-x@7q;&UPNz_rCVE;hX9FJ^N<8f5#$eLzQ>To=$ z!(mXjTU{xAI5J6j#R^!WW};OXbgn7}Tz*(3z^cCNyOIE@ACA7OOuULBV-;aQ^QR_! zEy=xv*FfD6hL6l!;W{X(*tgLV)(M!mkHt8c&+p-LamN9CSP!2<6f`3V!Tkk28+u%1 zo#mA-?X?BKtVrR(#OPpR=x1RJ+Ktam%({F>gYMlUGBJEMG5m>%(fdwG&nAXG8-`qr zo#27`Ji5gZ0lr9X7A7MyF+P-u2*I;iv4n=ma5y_9fZ)sgXUB`1Xr_JHF6s$IlN1}D z3_g7k<4AU=S1IdDCx+ef3uQK*9Rv8-X#S}@3&8LO3~VgS5|PKu6#+$s^} zXm5B8>meEbLkRmAjj8DcFdT-Ob_VQn02^h}ZiAX$j32!)|AV+b(wG(yV-=-AjCnE| zu-!5CWo4Hm-dkUx-g`d2VPnOrLx9!0RTGk~>bWUd__T!=jIh#lDGRZNnURh&i`IP!>JBv(pyk4&cG1sr>*0WQ> zp3>SW<@S_*aTjtgjPXus(NlU|%dC>TIj}(YH%@ArjtBFKs$3XU8$Sh3NhMtYsf*$! zYwocV2Ral-*fz28j-$F2Zr7~sMCKsoa;hQlC5Lw)hAt4pL*F_=jNRP!EBe-7$*ng6 z-;8REUV0_Sl&Z!gM$DdoA{%UTJ5;hBL_iFOwQgm3r@Ek#!78bAj>(*)>AIWjyI*dQ zyiFKmo`LMp_LY)B@Y{!iUY1vCPcF z=a1X)LtnEUPY>qt5fBuRmNj>BIRkC8Ua8;tYK->P7(4mu_nd@k43DcZ4@v{YS9>y* ztFibg0(&QMh$wG%9o~6E@omOe!7P<2UC{+{E=` zj(q27Nc%ynF@j{`I-=kC^4)jncXQ+$X7pc}xQ0z9#{(UN0Ch3QdO)Q^qOJ$dL#u|e z6KK;}D`d_JUc>%S&9KITl|CihSp9Gt`EVzX1=HiQwo6L3&6NA#&N>+448*&3%Ew0 zi&Y$8xbG04S)!?l$#P2C0fs(xN{Na2l;LYz1o*FYIg;0`!FmYX(pJ%mQc#PToU|@% zupf~0i%YFhJ8?z~Ogh2E`UaW}3Pc%{;J6XRM94tFwD`k6QFjxX4zj|{C()uVFjxG> zNAyoMEY^TmYTt^oQ5XC-lF_7C11Wlff{7qF@AMrMqilfHY*G~?V*{aG|8c+5u`syI zTBg*rr^DGG=6rbvVe_X@e81I(jEsr39?vPh=HPU0S^8_(FYQC7#~cFZRD3TNVgPeJ zekW7n~Y_6vW zz7+S;brgeRCvGm$zSW1faTEh!SjN3ys`$QDH|9yE6!s}0sH9tzl*A@e0+OZeiLTSD z>FY`0TG9#DDEYVfcY6mWmQ@?~KzcmTs5?1_$k>=<#_Ph!6P1EcN?6HlFq6e}Xkiz@ zvA(=vd&Ra>di-fl=}vK;8Xf=gcYQhu=rD`a#_+~*s%*HSCY>4$uzVi?Pni#>-yQX3*V2y)20SUDB zYCvEtJa1vmhJco}oK&r+Fb^AKeU;s!gqtQ!KwJ_*Lv}XfjRX+%WkCpT zbY!*pgTkg^M$4k7v@od@HHptyK>AagESsA5Lz`Q5YM=w`Krc(uk&%^zmOZBh5^C~N z>-Jc;ClG($93psZBIXfgO4Joo@mj3jJvQ2q)cFT$Zzx)(xDAOnakUgu$Hf@>*mg?V zErr~5O4gs*>4s!dX|sH1MSQW?3nH6)TSOhRJFs)@Tzu?`;;*y0F|e|MaN@SqqEzQG zsrd3`oi4(N3xvMSJ^#pg!Hp9$Q@Kv*(vS)@k!?s$b=u;Rc8GM z_@u4kodsuxPtyBg?~?{WFfqPN7oA;J`1gF0UfGkqhLX9{Js&I-n6mY_vWJNy6ONFw z=)mKc1D4R55XAVTuITv(D~dwE8qtv=hoPj#`QV#sMnG)FxT&)LF#d~F1fc^9u9=cK zQzu6hjX9oH5+Dy}KGu0s?2>H05lx+7%kg0uDi$#4PlARoIvH$2+JqX7nKMtm3F*K} z9b1M*duRP8cn@XynsM=t&Mt|fFLE6v{Yb-Jy6KXV@qD#0o^PEAH}}E!4((}f4TA7= zyeMsg(!YjC+5C&KuC~KUqt-X>F+z24Cla;Z#ll>8lVI{3Fc?)IpI@>yMM%UMPMJtC z(!kOj07Zg~MQ6+?>0qG&#nXArn3|kggP3!@R(*bKn3(5EX4JpoIhQI@WYw-16L*yS z@AZ581NsYGs@=?5t4Jn%VLpP|*|dsb;&!!X(8K|Z;&=^zw$-m6Wsp*RfiBb0w3;I= zc$jki0sNAquL%m_#31H!c*B02A5y(TB9++Q*)|vCK8<2Tbg@09BE-a3hfmngtY_=- z88q@yOm^> zFe=51u+-f~*$yGLbcAgQnWSlwQj5VB_-ws}1_JMd&Idc*!R}Dd;>8M%lt?6kW|DCw z+=ReWEjfjsVuml;7;Q3b?uv8?1Z?Ir2+Npaums&0CRJAcq^rgZC8=m>f?X395n;0J zyrpf2^BHO}7xIRMO4u3x63v0`m3yD{Ab*LC$sNxR<=%x;S_HCjD7%h(t<9N7r4ysF zG%n%zyjQ?h5B3gXO_J*nLgNrb|f5`|^O22+qH8Ru&*GruKSY|ZP%IHl9^_zmeh@05NH%|SS)w7+^* zZ6Ri(dWWZE;94fASY8`_;L@RmV>PYxomItRxs)l(DM8cvbR=)$#r&eUm;VprUTU@P zV10$M0*H7EfiI<@4ihIz6b4U?Te5aS&4qtN35L8X0n!9j83=k&69x-21IHyvE-IF~ zxZSu-tlT#i(DiXYhE!n@?Hr?UVnU;gBvwxL}WXG{_UT9ma9&6q@ z^cqG8PTyNy@tZUV-j`Z`he@?E8_ydA;e2>mTkyHBzw}tf#X0rf8uT1a=gpmMug{!3 ze!$F$i(oJx*#^uE08FX9w~z;QRP!Y;rKAqQMEN(~W}Rqv(0S^%9^J{fU7mmQ{Tw8X zUW4O9eWW9^aDbw`$-fIxhGoTB>Y7(j89JTcQjGbNdXNcXcrw?-xx zHurA6&tIN}+(^ae&jO!RR$T{uC6cK!bT}ErT+ZLwBl_aZ3H_xrCn?L}^N|yGESyqi zF(y5$(s*Usy5{MzC1JNX#OgtEl(g)~NJI$>tNZ&+L`oeaCVI%NSc4ZsQ|O{6 zw~8Qu$1R@4eQZ+6XBGysV_Z`$qL(}aFR?MkqKF{~$%wi)Z|RHP57TBbsm>6_oa^g% z5EI&k`mNG;-w$Ixm%iHwV}w2UR!eCw?nNLpY-ULLs}Fk6RTvOBr~_-OipA}QsU1Hm zZ=9O>`!p(ewgAv4#_4H}UF>wCDrkmND*=+*QaVDt*8pZIGPIbS4S=ys0v&h`bOsEc z6+rmRbMRg3)f$4C7~^cr+_lE<{V5-u?<;`l;FP=qh_*AoGta>}CA#K0&<2N;puRQ8 zp)!vkFs-Phm7ZGBQeD&PF3p{m{C=0!I7)dpVJ|lqGnk2NhOxTw3WUrCu5Y8(LoxEH zEIA*?&iuS`sw-o~D#9y5dQQf{I0cT?SK~cM|3R#IFs~C?5xlnyT0{4dVQgDfwD=gu z3b+`gsz_J#MZt<5M0x2r^x%=k@q;+SGTsVYDW-Zxhk;4SSk)JdvZ|=eTSjRvH$O?% zkkX-Pj|MT#W`{s;L@~=*>utn50%DdM+h)W(wh^4#}=_Q+{1Gb^R?|8DkfKlY)!Na`m)`#X|5YyK0P?dkPQ+j8n^nxC%;9Lk> z^L_4=^b#@4Xg8o@Nw?GKhfOic{3$i9oKpJT)cRQ_exfF2ZQ__RoXA1U_0)h@I<#*n z-~AA=MEbSzo$Vnd#r>VYw-QMhGbuqs(jFluvNyAdSj8i9^}s)i17Y?tpD(E-9d%d1 z6@6#Ry;%w%p#5<$h&dj&t1*u~zUB{B35T8%M!qj1MneIADJ$0TE$9GFM z;RcgW>35$U@dq%seAFLV3}57gpHI7j4$j8t7%?MK7g z3FDDe5DALVA23pjuba$^F&I6_+m6!`gP3#ujyB|vVLE_Y`ZSJ-OZv~54#_ZD^pQS} z$(Ws`Pb+wt$D(56Ti7oNI<(|+GJv_hBk=tb0W+sc!6{ia2ji6hFkWVrZ-+69d}WSK z0$>y{@>mjt@VGNMO;S)HU9A}S5!`wqr`@~oO2YaQR-tY1AnKPotiukx;O%7>PU7&A zn^A#J%$bBd!gT)my5ds(HM#Xh6a(hPWlF8LGM^WG_jBv?R1(Y>I2rW!SqrJ8ONC_Nlc^h;3?ZN94STapMBq2FaF` zjdsVI80;p|FXKgT;$^RGO&55H+SY8?8*IbDk<`Rs+fjQ(hlK_VJd@F55Y^*qc2iMK zt#D$_rr@f)^Ws*TNLZkzBBcDfD@yaIHi2u;0WScgy6VafHP>MGfC+YaAxbh9V@D~$ z>5{Um@L)qPVbKXKX<>rw0v4UXaP3G@4Wh~s06{L*nY4R7rB^J>W6Q!=qyK!^8h_Qd zhEBd_VIJ!aL6j$bW?@h+=J{&tDF7z+SSmmwLd(3O+Jj3WL?RUF^n|F189On=i2Hr(CUcFQUdf82kq>JadU%tYhh9++T0-mp43?I zQFo*HR-Fdkov{hAx)Fq>*0l?VdkXnr`ND!<;GR)Cp)GfgJ;1D2}}K5Z{;PQ_GyWw&rj;B4RGzS%&^chrOGLHPr6jv#5BoLPGJWWlJe3 z^v)Sp687qKOKIcSv?$)NY!A7yIQ-383V8w>bE!{nNR)o>!L}EfF|W0hR*=E_w|X3X znm}nYw=&fA4(1PpX}9FFoom}cLv+_=?4KMaqQL?|qiY3>6oCE*t4pbdF0ToBfW@ zqOWGwyZm09qMwvrn2fY`XCJ6p>1;L`-tb zzyMT0tG_pcWugp(!fQ1c_{gEIfr8hJM?bZ8blRNeZjG`UTJwpOImfHJ$&K#-k_Of) znNQ(GL8Xa+Ip$FMM$J=`KG)r7&KPgRdiYy|ESmC?N-M|!v8(MAp;(oPhnXNHs3sZ& zfZ`PSfK#xev@Zw_;8n2YKnxJ`nkx$e>q;$Ob2NZ4pO>{v>2SFi#E>RWTl1IcIq28* z99}}q3-Y}gBLsRw&mkqdEWA@UyNPB1$;I&Nz^2`qn0`vh^x~#EL**RPU zjK39%0G*@!paZLD7Gkaq?47-kRYz&jwB8jmqcp_5k2#9jYFb-uO>{g<8>43~RZi17 zMa*{7n(9T6e##h5B zN<~ofjt_iH)HyH~O}Q3U6ig!%++HC;UTyw);YW{zdZWUy@meN#XCLUOL}?IHx&>TW z#h5c_%3qszjL!bvyqIn7-BvM%a`s8stG>XCag9kE?L?wg5ie$PA7rh&a`KuuapEJE z&G2_K;9W@P&2Kb;i#Ty0+h%g#1~>7~5)xZ8DrmKFFHZ)GjqK zP_pG*3WG+Cf>{G;>wAL;vfj1(&ttyqvwm~{k7t}_>qn)4~p@;3MFk=tTJ$x9kS~x^f zrkLG>QlM5Rqhp@(-Ju1<99Ln?7i`QALCk^fYsN%|WePf(An33#F==YOYFleGHnYpI zwYjKZV?W$|2>vOPkZAwdk6xC?ury6ssdalfolbz5Z|PF~EJpY%#hCQ04!3>7E3NN% zK#I8W4Xh9AaJv6w2`YfMpQ7-m7_2Zq2*fU=~5j|tx1)xmwHQOXq&B> zrb^t#sy-#A-p07V7X{&Ygc$vBJO}tJpDLk7uQM`XU)uxaAbhIyB=DtMfv>fC@tl%3 zGTK5$r;xFMZ#-?SlD!oRoMT^6OBqn5Sn6B_idP<s^5K@T-zONOB(~f+mmlw zuY%y&4(W*5;J(hS?nU`d=|G85E^dk0IvXYi?mTt@c9UyMmNDVahsm0x+ak&bn3c>* zu3J<~ypq#*c0&F%F4gBuhixUZr$!@2Oxk$dHRxh)3t`5G`j zj(I$ZW4z7~8mUJnBOLjfJDSh)9bF+ts+D1hpF~IqZ%R(bla>3x`QAwF)(}5cbDc1! zN6DSHDNU;E`1(MHeD!H*PSd1kY^poftqx?z#=5M7i1g>F)4J_TlH4p;Sm_+ zXQW~#nWt`ADQne9dp(295NTI3Lw|DtZ=k;Fi}Q`r+XWE}i*#HnZAo%A$rgGJM#Nkb z_V1r!+X5T!BTBfl| z`1nlHVT0)~ZHF<7Pe?w-9el?N87LK=m02S^Mp2N?LIT&^STlpGt?g~Edqs+3WWpE@ zYQkCA=|`yD>job&l(Z}fBRyzQ(Xt|C$6?n}Q4m_?NjexSV-U06RQ!ch3G>|0hxDE6 z$421$NV`fLZH1##Ma$6RK4fO9wCY3ZkX-A`z}F46VI;GI7`=sElr-}KQcd2)E2 zYBf<=z?guEno77gCKi!W*nE4C@PCgc#?oR2#ubNpE-Zzl9xv8=Nh**;MU0i{s2~+B z$XyWF9hu3E0poHte_sDe)tJX-;0tyTQl&RheB%mRa@M~#q{bvV^hD;KC%hVJ5xeB9L6*cl@TXSj;|P9Tw2hZ|XxJRJd@pUKmJYwaJp1n(5&+PM`?c z)2M$0hlzA&u@D;mP*G|HzYxso%J)5ZBtk#3c3WSY0|I7q7Bau!tL=T!tGG85e4mKl z*;b-UuH&ZC`}lX*?OHN6cMHBR^Y83ETo{_Q2U?n<)aFo)xx5Buoqjqs;7r9AG#&;w zMiAbLmbA3<{aKmP0j3Dr`fz5i1e0BH)|A8yL zsjrlI*|2(Pg)skuH#|8`>3Tq?bR=)6-{Ji1A1N%q3VdIhJ5g)IM95et#C`xFy;g@y z;XvP`98_SM$hnr!7q(dlMBt$DAgx$n2P}?j5y(CZG3UmdZ|l>yL_mI)jrsbd(z|TT z`UtoG+e&81(Yp_I<0x5(Y03Eth&jDuY1<#DGqi1EaHG;^;&;yw6Wf?nm`VX@MTD!> zjF_qBcr`+}=d;FcEAvxz|z`7q|;?k%;y(g86Kdna+03=%5KzndwWevAb3CHlQo3EugzKP~_{ zwJ~~+e+OV0KtG!b&yA6maLYAl8-iu2m2E^u3Lm!{u=CxGw~>lXsGAuPx~}R*-RhgJ z6>o03X&MMs%(t0DQ`fh;{}%e2z4@0FCF!B?RufQVAT_Vi z6k(lTQ3*N@bQ}vtB`XRFdh`}2Fz{FrQk5v?P?kkOM)P4X4aq=}-h@e2R>k*9Y5>gn z9i;Fd;**{&5?%SENH!*+VxoEQK55_rU!?W}ea1fdVH8x_RL(txd|ufp75g;|o_MeF zEUy;OZ2V+Ih2kgCUMSv$W_Q-^it7P#>+6c*`}aDf&sK14MKS)Gmz|QeM_+bIuODK~ zj(Dj>K(6%>0CRprz4!M{l(MNY?@g4FQ!`THtHUS62uu~gUjxO~ri$LfVwMCMJapXk zGK+n8IYYgdY_5gWukjrCH|DMB|6<;nMuVDjx;_E(tnEyXi}UHT{q$mF1W6)3^15BK?zqT1R_wY*CjP?s_Gh@ z;yf&3Vi4mxoUf@UDGOp}(-<0wfjP0((nKd9q{w?wfXHf9bueCs=SWA1FhNXJ_Psd` zJKKOwRdhwiSQazpbhy-A+tjDM2Eo$87ymNrst_3k{xeNV-38|Y5|SNusBPSLT0u_ zv;3kksnMi9TrL&#T{Uuj(OH?4m~f_I;{^FN?iBEd_+Tsy_7j&l4P`kXJb=GkR@CVX z2?%NO5OXFbQECq7JsXpHCAty;3Aaw$P}EJf;RFh1*7I+CA5(hZ zP*DsR$aZ245E=kOodyVe&u4(qc5M#=hCX)U-Z-^}Sb^``7#%X6$mL8=jtCU~{mzQ) zy#c?W9pT@MrlUlDE*r-HjqOd&fo?6UZCy7PK+GkN=N zN%i;&lS+TTc=uS-d%cB8ML^8oa6@J7aveVS$d}_iXHfN}vZ56;R}IzO1iP(j#)dVG zc~T9q#^7#Jrp9DFO{40T>BdC0x2j^dYN)G0i{@sog~zIC?#;c_<~|lTS;39E-FQ)S zeNVd*ioCUAgE?6gJ&YMy)$iUUTJTaqNKc?_Tfm;`k(fV8Uzp3734YDL<_`ixNo;^? zs}9x82<#m#;60W3YF_GdG<@5ZkQNZLl63gOfIb%Xmtl+@d<;&>qF@omJR-iPQ?H_! z+kAgOUx!pFl<)MD20=7+BKFoPO?vh&k(Y=q(9*%AS8ZIj9rdI&n}rQp(`Pj5Dn@?B zmwXfX-jYVqu(2{!1;%)0;CnT=Y0mWbaHk^RrE1RZK=m3hb2^+q`Fp}4NmVRs6Ka8h!1Qj)v(muyke_zU9AWawCKvw zc%MPSX_HL%=KVOZEnK=yp+}&~sw>M6u1a3!=GLH;2=%-3WiM6I+o=*?AjX9;PQ+J7 zsS@hJcpKwm6_+Y;XJjnOXo{FDE|qqO*-VuJmnuQbXBA)60KsWwtm0wc$mGRTQYy7N zfe46szz40SLDL3amX=zHCI!clggSr{nwmi78_z+BUwH|6uwE8%LgUENh=Wh88TGtx z2gjNw;91$`Cj2`NV#IHwg+dajW&vs@6dB6Ch94NzMNEoKorES;vI4yiX3zvH2wpGF{Js$LMt&^FLiipl61!voE-ZYl#@Y z_)*r%kz+ud3RQduh;5MBJGXeEohXSCqKtY75lmDCV)9^5@Ew5%0e91G!caP-#$`GB zj^l>?5i5;ee@M4owEs|)^eD;-MdOZ^d2!77K(2t8Z>_ifO(yi;Bxd%Fa?G|}nO62< zdHGyQUgzbL4{BmlvRUu_erA=GE47y+TAIH+SJG)85^N0Hurc9|D-|Jz>!_8i-Fdtq zTB)>=EBPbd?-pZX7{o`P7!az;vGiud>Y8DDQk05TohbVb2vt4*Fi|`0St;HA9Fs0} zJyEb>*4UDcng6UbRS8>aSuqxcq**qG9Dw56uGd$1a?B5>IxM0Xzf07c{^%Yn_xo=B zM~ge}JqYC5iUiA{g7HYG4pD0=Tph8h=3G6XX}W^DT2lg@v~T2>;E%0E zky$3@a5Z5}+a9*m?;e{bCPoZ(^>@4$=ae=!8rPVWun4GvcjM#(j$G#{i>S~(r zJbnPt;a*@RkTK9j1bzbr8hJ$tk|&7y0u{il`W-f|+iii(zcaVyR>$nu-0{tU!`=<~>^fR*TwYr#Bo z*l%R_Qv}5LJ1nq%;>2kb zS-8wy$rt1Kc%E79GMIZ3oR?i?+>%oFoCP`LWX`qi`ShL9Ha|t6xt^QsNV4% zPB_%rq)q?>N6>zlQh%bOck1}b<8-II)i4na;1`IGpBS*@bm>uIWk4-;qOv0^tlENkVA9*j`rQR8wA>7;(tV@`gMHGEfZs(P06vkJe){QMJ8rB!}8S7yl?tuTJyKA_s)pXYo~-FLs-eP2E;P_@n%t1y~9=4_EpS_?O=gj!RPC} zHf4@yAs>l}p(_(Zw@nNU^x|rKK+=hWQs@BrR(7dW{6rx}? znU_IJ9@ZY2pSV_ct(%)_Za4LDGoObW5_oCMfo5ECUX(RA|I{#6Rhrvg5l&&;ek9|l z2YUY~sC7XFrNmWc?2rnDD26Zw#TbS|DU?zEqu+v+R@On4b_pO}UHL&_ zH+YhRnEG%)CEvqWb%x?p>Gv79)~i%0U(|uJFL@=mQ*>J8N(;p3O?MPu9@MT9U*&$7 z5CK!+u_aY3>HvFMZ2a=ktJ>1SCLzH7lSRVcEdMO+VtxgP zoYxo&B|#Cm=sUqkPsN>=qacK3?uBeFCxe*t;h#)+u2WJwPARb+!APu}lHKhV+YtN_ zeZH(En*yO59N+{wev zp2P)TJr{ffV(7~S-!)>mU1OSsnC(<4q}QK^F&n8;ROE}DQfyMfwJR@|AcasrznF7~ zA+Lqtf#xxlm0Vw&Ga_bP@qO7VJ!WmHrxoAFvf}GYG*K9%T}BnW(i$>4Kqm4^I`v9X z#rJcsG?!zTuVZZ4n90W~J0@X@!kD>*6tJPzVNAFH_las8C73s&d+gY+Vt(OiXDK~L zW%}Gzd=<=JLsSDo;dx3!ZGCCL#$1kXn1=iz+lr|ZeYVq(aHFJBF=8TM#J6+?l$J|; zrSA+>8rPiU)*cCIr%A#}QmuoH6Ve4t8-@awgj#L0FqR#ekl82MwcY-iptNMTH_G_5 zs_ad+onvE8HHeuVuIJq@q<*(kq8r1I8}+-lIVI|yl6JNw28>=ir91xEBMz?*u&r2D zlA;ivO`R2=B&`%I8m=Tvk5%T21SP!6c7X)``ttx1rK<|KG3GlAyt;F(YKsxn(NpEE z?0T-5xi@{MHB?I!@M@GC$6+jR!l>wEL6jPAJ?ME<4Dw_KnQ>LAVd#es)fc4r7)Mws zIu;-1r()1p?7NB(^A<9iIG5p)Vp0@T7N8Qsh92~u$D-)XlZqBu)tKwC1`~6=zQM%& zkT3?;p>w-U^u_Vn$;HK6p~-i`m?(DHXbkLi6E-3qDdN`!&PM_jLP?P=O?5nBHHPfl znEwUSVGA+uWje%&;m@5C%d5r&w##6rM5?-j& zCd!d?fIs0@vj)ecfe$P-V~q(jAnF(@^AX5!zS$VY{5R`JJ?j1V^AJ+FFO<2NruU3>s{A7o`DYi2hHzWUk(J~-ZRM? zY}*#t^r+PStsros60G8!q7O3u}$sIe3IH5Y{SUc z2duEt+M&XrOJNBH#@;6B1;eROIxHqID4`(u&Vh6%Fv4Y%aJI;NKWT;K0g?iz{*mZs z5Z0L}hfU`HSRhkRLg5#xiYEMlpHnDfiOi|W7?h9E^HwyIe$xIauW1vn zz`zmBtypJ{c&Kj>m~`ph!wMC`s3p#14$roLm=(qME9JW%G7I?yYRpdG+GWV;BGKqG z7dX(X@|``zuT#OJLPT}GyxbZT9gf!({chovp6ok$*fiEPgO*i zor!LM);sZvRnSL5Ph@_dhf!6$xY6Ry%bN*rMQt9JMaQ|xWcsee6`w>Zr0$lCu(Iw+ zUlyF>_6o*j;LKwTljVUsD6hoFMup}rqsrJAM!Q?DuR`nH{BPbBV7B|9%fsX8> z=z(}g%`XQvX06Js-jNGA0AkKReuyBu7{mNR^b8Z@uh@-y*Cs{_F@-G2E8>~h6PuRi zg9LSasrj2HQ*4_>US_%AE@Vls0_r#QHdEjbfwbay0{%}q%%!-?vvTXp=?sW@muk#U zn4V>>as7f*a$i-77@z*gDL$6dv!Mwdol9VsxFWshpC%PRk$u?4z)P9Z`Fw_<>-HV; znw;vnK|r&nXMOq3S549e!8@jBGaCevQ@Y;|z<3SC7~D{=eXUM=MF-81H6@T>0u%P2 zZ8jrDSpn$iZifbe=B_NmJxK6c46`jrr5ZrIh|LDnvpFRf z@}M~hLV_e$xv_}ZvN4ay#?T!b<9eoa8OA&UVrG5K zB(>f#F*;OCTq(}G4ef6K6g7(09%{BF-lr`u#V7cx^d`j`9)m~ezSDeRY|ij-TCQ4U z#KizFoRJiz-HeybH3)3nhR;c#$4Y~@ckiy@7=c{mxz(l$EtcK zj0txdT1v@%SJ5DCbakWrB-7-k;-uoSp8XOEBHp3pj?%RqmEAQrj46W?f?nf!3g@Ou zx3&#F379oKqeV&b5mVPEqZ3atL7PEP=#pF*}TjbSZ5H`+3DM!4SxqOMonoXC8EDsOZjy9~{_sPGJ zy}@b>&S0yoHYCI`SA&?tYI((r)EHZidAJqima2I?z1I3H9gv-f>+!OrTIt7&oAnoqbUf@cC4 zD)ihbSF5Z^yVre;;sciSh{l!%EXP=@l@-T81JWSI9RCQLaXH3$B|48wCnjbij*&Wv zO#Ic6#2LOS%-KQAufesxxnRtUKLz~$IDVDR5s?pfvp+E(K5=$%u~HB?k}v#P$WjR4 zR*gWeau;WY zWFO8B4ZztRI5t(HJVfiRF=|bh`+;0qX1|e( zxBU?T2D|Oh*=>ZVgCHH;Y!Cx)A%dx?1evPIykaMd8k7ldSi=x0P17>C#ZyyPRdt7x zHc+mr8?b3h*m-`d>yO$!FkgCDQd1N?w62T3C$u0%UyKEgK2{ZqO5qao9>)M+nz6vG z=oJh%_7yHSht1}&7MNDANHi zbKqj2CFh;4`;PaluX~1KWU8*qzUqlF8(=!BUX-u{g_S)Fte7i4K-m{G$_VQ9Rs^AC z{w#s|O;pzzXq7!MD+n3!Lyeiq+o(jYkSdvA=BJlq@^xbz|87~1nQ<^XnNRt5Av#V1 z<>VvgIvt-YHNo^byk;Ie`g7+Hf`VJZ#_+%x$dFO_?a> z#zkrV(>F16qltJJ4k@boQYR;8BOl;mXh3l2jNy_BC zny=hg7%A->B^{Wot*(chMSmId~H>Dt5peo!46R>_-=Fff;z9t zf2`Mpn7j~8v^dE;5d1b`zo7<2POTWX-*Dm1ID9AGyUHCgi0NnZ!YXt{qxDfl^qddK z;>;`HT`$*@LCk4eDx5Ckm?dID9AigkADF}b>1*43uB3J+PbOEcLz1s@+j3kHZVqLp z)x#+KnI;`N*TL}kwn0poi4UMwsX*p*FN!&Q#=L+2nH5a~pd}THR#3oWA7w#@>^O$B z1mc+M;XAuTe-)ML>2!`3q#N8# z-iT@!Kw#`CqYASsCuUZa1i~58Ga<7O(}KAC`D}dCHFiq>KLF;jSB}{hGqdv@HZ1YC zHmToP;UAkHJ?1dR=1OQ|9#~|tcneJeM^WEE3??}bn>-^cd=dD%-jxMYQoq~nD*eCv zow6;ig9w07+V3Qu&)#KoZ*T+_sBkXsO?(m@T=7vj7o%J*?p<2b*w`1%&+SfexU%57 z>&zt`IzjFMC~K28_f{$hk*%tV3mCK(#)0sL<6Vj{d^1f1A_b^gZ0uA)?egRPNn>VyH+dtW=;8Kn;~3Xa_+Hn<*B z*@4@2vaDr@nzI1Y8~`QXT9ETePf7m{J*7~+<4$>EnOSQd1u+e!qX}Y~rh$#;=2yC9 zQVtRpL;^cgS5mdm?5$OW1w05dp&2Zp;bVLPgI75dllhElsBSljGp>_W;NX?Pq`6dx z!B`dG(Tqwrhy?e(r@zR@~+(vEx1rHS@e1(fNL( zo-u)ygNnkcKd9o7z9LzGY1#vhTYqPf=nGRPkKcFd#Ji-V+8c^7NNQPCdwmw`7!R&h zbkuVUqrCc%cIG6qOezMQns>Hw8Caq4Aps-ykai3X^*>a~{f_dZGm*Io|q z*b4bG1bm;$vFe3|NzX0-Ges9gFbW^JckO)a7>r+V3Ly`J{?;AznNRF;Zb zo;O_zfGhyxVwhL}1X~Ecoy0#OIlL0X1d_vQ@vPo#4)`f0_SrJ@pzAE7_&3mdC$Yp% z0+@{eCb`@f_lzMcMii>jQw=>w$$43nAYex*Ev=fO!hm-`ms9pNV`b5g1;i%BNTuDU z#ETEXO04OXC8|b1Mm6@-#4TNKqFqv&PaX1tUZ}$y>Tj*K{#h1=e_3N_)57S~!aUO$ z+Ose(%))LR2_1Z(u2KhCJhNx1$hR=f>2hki!=ZV@aQ;sL%#%6DDa0Zxz|11hFS;bX zx^@Yehtfjk=lL+{Z`m{8@>UgU236L(z;H>WN}$qJ1ms#v zI?!GW=7D+E7hOTjhnTBDiM6B^DjAI>Fdm@pTM|&5&me;-3YB#z zvrHn_t}G1q%`sv$It9#Z2ihp?KzD^ZD(!#~wgv|G zD-Zh3iUGl2p*%PY64)_o8Jv!AbvtR9d6EMG*F-i#jPNZ7VMWk_ApsJv31VO;`el|@eW}( z#z2v(?C%-7+F{^c6OqG*tP$8@rpkpR2 z8JK1O=GvN=_t>_dh=YIDd$d)uF#l+S5ZkOy>vopxV0(|DZfEbU)p1BvHNQs#e_d76 zHmf-l!3Mguu)-!s2P*>NUX}CN#XXOli=}Z-bEkVoh_cNhPw_SO1^x;|vH?T;LD|q( z@(a2mI4qqMbHxVY;DB(`y7&|PX#v27G7MneYY5@(Ur z91%Ny-a!w1@kTwZ&6WKqC455@CQiI#n+F&a4D~V`Dfpl>K6Z}Ea$~5LjVgNxHb? zvo=1NW@q!D%saCzg64%Z-SK*PN3Z(7Vf4OxW&o-%jOWY@il{ z5Pjg1OWes9V6o+l<}BJqFvx?kJ7(!CRhVnroDE(!rOWh-5e$ySZg?d8przyo{aqs% z9K=`ZKwE(`sWyx;p=b+%u7(WBsk*??NU3}yNr$Y*=R7hXW`bE-5wA)2iIL5MluN!R zxQ*t*W_EBv+Ev=0#fH*!Krd4FTruJ>2s5xi3ig&Gu<&NIbP{@wa=?N^n%B3w>w5F^ zGs!#9ts9sc%pa^M`U(`FtF9s_5`&=%9f*>~u#5#0CG;+h)iLJ_LOzNNz+9T+r8^zI z7s33oMcyT|VcDskf5n%f$Q*D`>B&f&p?ngIqM*qq5* zdm&IXidm}>*%zzdND1!dlY%PSIw%-&TfBjRM5<MOY@5z{(V&a-ZYbnKA<}#NA0f8d}(07)nIr;wxe#w%y<6q_GnMG)2?Xupn5Zc;UR)qBPO(w-WXgW7`=`-lw^gu==$ zBQ-y*`6Lw^KAaqoZW>^`jrQo79L8aQCKFa*Wbbi9C6!e{dusd<*qWj*2ykVJ&OFWu zVA{jw)SWI(?&9;ji6ou^2LFbhs3Kxj!~;gVu0YD>%``9zzyvlg&j-vv?Fd^e4wF&j zov)mh*DkMF8x)l3F7JYpi8i_9L)>%d*QG zVKaCk=k7!oe)j?YHRPKZ82ui4B%FCB%EUcCkTPg(8uG*?OmY zXutiA{YienO*6A`u(VMCoBUu*HVZKA#hgE%n$1QvT>^&wxkZ?KO+;8`59JQ!Ne!O^ zX1~gtpkoGBe0BJQ6#~QY&WWTaoy#n)9jQ+SFvq1!dV8aKwP}m*1id~h-6$UH}0ETT*VVrAFvkcu@ zVKZ?w;r;##2Lq2PZ$2Qj2t;_6)u=v`+5l#iBz+~Tw&mGpWtcEy6lc|sN7z*3 zCWzjz^Pyj9JcbQ8b&p|H&#ZtG-^_L$|Tb1iLdYIwb1fklm=(F6QNY z)`js7>Af@6`1Vgd(ySw-)V_MAl+TJVJNb}KkQ4i?HTz{{L)^aM4FIg*0!ZoOvlLYv z%J4o>!a#Njdx(dshXpvi#O0w`@yqpU{LzJM?HSnaI;L>pHb(VEfo_c8JKZTaT=H-b ziAL~k;Nt^cZ(jH};ujsvi(Z+RPvaMX>Gbmve7UIFuaU3-%#>a~V`cMd_Z?$EExw7C zLaAv9IJcTsflAqy4`NiHHn<$9C=7NgS_)2aexPGrj(NEpnx;EnSDNwO9j)V+wRv%d zfO~+&cnR_DvdH_};>EZlD?9ZmlUHvfzzOd=$&iyGZ!}LJlN-KaT6WBei2co0$l~}= z3<`CF>l;B)k19z*ybG#Rsw_k{4~EnY>CUA6PLcNq2D}jRs-2o$G=ZN68Jk9Bwhv1q zG9xQ?V*Zp44`&>1!fJ)E{CQZGC{>9Zk90nsPTi%>Tzo^HR$ypUfzki#3XFcbTg%$I z{diX=ZAS=T#&Ow25^`Z+q5x(gI)yz&6$dSeF^Z8A+Zul!WmUBv*iiK0?pLf*qo5xp zlF|i*tu_kv-zz2xVr+&rA|)wU4?#)~Z2~YVlu{e#5>}RdNh($1&LvvH{J!~2SeZog zDllzx0=Lv|^BU;q5lmd;otc?)1;%oow4W8gnONU;3?2!G4tyJl+6J`$R0|mXq))9s z4R{fr3y-vDBW>^+)UHSJkJO|#&U(mf9D4hi-gwF7>wLB>Q*5{FDZMm0$(G0`M?&oN zHfG4G@t&`>0nBPq(%1Ubuhe+sJ~fX*nAyU-qh$A4jn^Fsv4ODzCH;|sm8fYr_Kbm)s>$>UY3xsF-VOWQ zDndwag545Mmqo8iyIRX!M#{dg@@l+?Y$80RCl0i!bkLmwM}p<5Pa(G>8^%$=ry4r{B!&>_yw&TVRt-35`DnVg$A zo%IB5WRfDsl^7(u6J5G+1|H1uQz+%ZtSc@dmL_PQcMZ#UBnhc8XjukEQ5B|yiRwDf zCDo_HxdQ_D_lXVQ9RfEK3A8@uodLtcG!l)6OXy5&KpyYg+K!lW*(tUs#D-LL%H-7I zMNJs@wcTzd#eO7qFw4_3BPL3f-Iu*oDtR$G*o_@hbYv+ukoI5zbKTa2`8f+IXqXmw zd~RXHGr;I=z_^``VKtVV5wNr{ky(jtN@Q=u{5_@Wtx0n21-dT&B2fV!<>yCOB`$G3 zHXK<84BW8931ubkMT~wfM zH;#56G0oUOj1$!-6T@6iZx9ip9t{tBGn*+;%U2I8Wjwy!oFCzuRY%A z0vA?SxU|~5LaV-jf2f2L1`>b?SZYp2++p2U{&HAijT^9gSX~kUh80y#-R^Wfc9-jF zZQobwyxhr}d5?s6u_9%$M|vzK@DT_hN1E%ruX?0)otK9?Z#=yg*LgR{yp%JmY`64C zeBUDl7qbdg691#^XbJk7bs1PC0qpBB$tQb%$mmU6vYAlv9e_8@3R2!Ys_3#`rL?V~s!yK7y)VZph^iptW$=w;07 zNWY0>1LjO=%=_+ff$fRvEenH^I@<$}&tu*t{blvo-}fYwp28<=q|MpJ?y7`EX*pYN|B?tG@@P&o3 zs=gL6u4!%WIDMg6y=7q@A%>xG)N8pipUqT1XHWS%Ev;2?hGd3`vmmjz!vEAfLhI0# z3LeHdoFWYw`vz1evnOe^lzIgy3}Z$+tL97!?1psKCR}5{LLla~gKt?ej0txSoyn25 zlO+DBf$`+bhqiKKp)ugenU9AX2C!t_Fr-9d3q$pcm(Z)$bIiMCVIGl%p*t4FDNkD$ z79`mok4s&YDb`Y zKF%Z@8#PC0)g(y=6Z0O=Z-{w+$^e@{zAm&zzX|NUjDjr_*1u@P>N8@B}dW!IQEb5RMv9|c z)h|JFliLO~quxxw^`rS|hhely+XNL`_~8cgly$r|KeL3>Ab{C}6wCmpKv=&xg1;#G zdtVU7s?mHYU=K-O^$hlx0A+!yaUAjP_wbb!#aJ+`@Zr?M0ICchBhEimASgsgkLBKR zEI>U7%65IxkJ7v)3owVnrMsRlZ&<1IQ!LDUko2VR=5WM?MFVYR;}?+z0?L*~z5=u$NVZIl>@*y|OQV zZ<#lL1DpQbA>pYZ^+!u9lb~e<0cUM;{7K%X3KkIL$aa0mt+ViAa%xhqM%cbGFu+5^ zJ9nxvq@-!Y%j~Sa#u;oIo*-xQpypjlc<>lMvsX%9_Rs=4m;0_m5NQOdv*_S3*65!| zX$y* zF;pKO?i_k`tRMk0R9Nk&g3CGx)LCC7X=gX76*exh0q@|~JA(8#m8H4Ml$NA3=c}#+ zwea$TmYM63+V)~jl0*HS9kM?zrOu3CzM4}1W>lCt@X4z3Hb+nZUk%BAG35OLQmbDC z3=gNZS^{Q@J=OdsfBA{JjPSb0i8(`p=6n zDmfdIR6I16%!Y-5NkSO+F;S$g7bd|x9CGmxYIEC7N&HY`+7Vlp8M1U?+UwzL0CQeV zuzkhA{0t!>#-CU1^hO91fns4`k{~8EBhmBaDLt=vbKpv>)!vb3SmtBVXviC?ZqKA6 zm(@+3vq%XX^i) zoeA_z{@Qwh-RYtvVA;l0@?ymu%_{a^N`?>63-2~tm)8u99W#JBj!-v;xE#P72syx5 z(hgu=a!J28cPe|wBQ_rCV1Bh>^_}#iVjQ-AGVRkEf2*A&3pb~m0XqR=QnAxf_7nVF z-eG5QY%iK?sjV&7zU>Kdw-C(M^C1%7*zw4mQYQPrtcQKzf@&A}7=t;C1G}-_MJ;97 zQ(5BXoT@Ts`vHSrlIN1z`f|AdVpbzb0Wtayj(WeAB~cf~ykKJX(`vP47dE2arNQ_$ z$0{p_a&~FpNwXK5pPK6#5OeysBq=V$OYVI{w~E;m=`ery*|Yj!(yH%(z8K{?{XHs zVaGCj`7nvOh_(;cxLWq5<^VZ1kS#SU3i$RE8})#(0W6})hB25RJLQe#Zgc87#~Alo z_QE6HZI0MkZ64NfyPEDWdbQg;yxXp}TT{@sMRBinRLk}w=Z+5pd;l!Q9239_KU@BKVJa4!;Q|K^j?MUMRb}31kQX#nqfmkB4(}h4gy08uSYm=FNn=IkWzRaL4lq zcQ|x8XV!eLUwv=DFb+)v8ekOGHy{Jb`w{pl&s_3fHcG3^)4zT?GV@i@te3H)V4#Xd~5d(W~@tR7^&H=W?WJPhYSSauE zDw#Y5lNxWE#+6!F8&lO{m>K$K0#ycg9xSv<= z@Vn8cfp1j3qikQ+*+&~Trd^kN_iHka(<^3WR;7}Qe0V&AOK@F`3UsQ3F1RpnHtvUZ zV6eC-``#Fxni&UKn3q{unb%0ql%yl-^9(2O4f>3EVkK3gf^D+A zBpv4RImMXc;Q&tQaQvp8kd1hJ*b@O}-o*BBCdPh_LyScaY65*i)7(MKT2Sb-dM`?m zWBtV;2EUngUUu-Yp$a?`)Kg_GWldGvP-(o+)CiRRmQ*n*nVhQ1Tn>*+SOZ#fYO3_z z3Q0lbrZIOD_ft!#+T0-KF>VHSRoikg1J8BGip%8mH^JrEzj?TH97d@b^g%0PoKR0k zGJq7CF$jbb<7{C3vBG&s!qhMt2g3;NtA>e28|82HTIT>7oxYoglS{sF?_f|dqYP#A=TvRR4sxv3$DIktOxLjOY^IQ>0%;N0m6 zZSQ(?!Hb-W_$tEFR!1qi_3$`Sd06uyrqFmy9Ni32O*~>u%)y;Fud54h(;%3iZ1SgN zbe&YU!ua0ub{jHXQgc2ZA&5Dxf|$RT2>=-7qcdVzsaj%Sq)(eZaG35 z;~R}D@JzNYGli4U6)FLh&}kO79CX~d`#UnE(Zpv4tADctn!o%kJmW4@N1#S}^KyTlTjkK}??`DPBF zqqjztKodIyn<83r9$TVZoH$hBb_%zmB5(iA;aG1X=I`Zdx|Vg;dW!gvt9u(`V1bGF z!%~dJDAA)A2+!hbzGPX%hUv<(F|g01Y1(f$t+CF(L8YSZHvc&@`IuC5W)buG_y8Va zzN_hdcfa_Xe7PllvvO+(rrS!CZw}TjWjanfl z4qfnxm|vSs%;znow;dZX@oO`XSCENmnnUv*A*$bA8*yo4><_LNx*yDq=7XRYY;1Kp z4#x2)FSLLxDaj#>fzGzlLsd1dRjm$##Gb*#K-&y(H+uM>Gw>jsfZNH(kpppJl6+E> zLPuHH8EF9<`B;?Yz;raRacb40q!5O=F;&X^C={Q&ZL&kbSTU-C0XefG1a65OO30CD zX+Ev!Kl*|o4-~{V=_T&n5-%_C9A0)x&w`Ug&Uf>|O~Kz60Mj-NFv6R|^*tu$k8?_owNrAE;QSRy z@I`90NQ+k{CI#5)+E*-0=8}r-k2(w7dt06g+JT%(vHj6`}Zx$#Z-R2o%@=Sab!au**8PfOl%4fOlO5zF+Dp zZBQL_!tk|EmR%)Fz4ayEheHg11u=9FF^=j$^Gm+YzC4p)oi=QY+@5kws&xrqc4!gN zg;6vVdQNkpcf%OHWQ)+Wh#nFv$yKh%b9+P+Lss%1dnV@ROWsl1-029sa3Qp!8cE^i z=G8SSBjTNw(ypwZEF;|A+8U{tw(@H^k4mDHDal=mqWUBs{E{MFoW5 zH3t&jII7X!Zq^rLQ8A!=D_00m1uDPDp)i-pVI1$!l8)%SAn$8ZcWar(f(@jD{RHDE zM=T^{QNmB47h@39ZaJlA)7H-r^GFfH6U1n5Ngk0?!gt%oblDhA$v&? z2=vmj!_7)elrU3UjwQ`?N{0q3zJ$DC6!P)dU~E#Gtpul}aX;`f@WrrIyYZxr*!z`O~x4{1E~2T-BYl^xFtr5p5!WnllF8#!2Fw2irSEKi(nft z_K|zdT_eY8z+r$X(g&KRN~!Reo1k^5Z@^$0&ChcaTGSs}a#`|Qis2wX--)}vB2AjgC@7^!f|U56&KGo`cgjY;#@eoEYtu9r?8Vor>c~6Xl%GxALCQ(8J zY4h0W%D`-|u_<`dK?B-&81f-x>8myaV?)nAh7Sh|s{m=fP`E-2xamYuS@i|W%9O5F zkhRX6Z2O5qW8B7)%wF@Dvto6~NULU1?G#lPQ(Gl6mF3wyAM<=sUug zAHu)0YeS!MV^nq<(mh+CV{!Q^P+rPC=kf2d0%bpLjbc>z@TC^PnbZK784RY z$xRnh>m8?*F22?(36ZU}_{?(bM!BQ>cHLQAel4{mpxNyBgi2PX23&n1tf8ktuDm7m_!+(r&3TAVxsWZd^vLi^+}=tZnnkOE@58CaSj51M=8J+ad}fGz%Kbi3LV{^b@X5P!+-&@47Tf{O(Y#B6F0JDh)HYW@(M`mO&m4&>|qsH+6W= zlrT99A9401I53kTK^)8`7c@JSX=5ySaEdHIcOME1Hh3{^g**dj^;Qh#9yme;SX8~q zpwJfOI3-fIh?vtd@O?*e{bvTgue1o#v)g^^`INo;ao`&~kZ%#}SdkFCyiJ=GKZq&R z6)&j!_$;NND+a#g`p+pPh+#AeeD@KPE|s1(5iDegK*?FgXXuo}e(4FH|*erwP zJW_abhDrV7R&wob-%~bK%#ERT8#%I=9(W%dVH3Vs84#--M(r3sB8J=R9GQyZ`xZop zN8*&|$|=!pr$jx`0WBYW$;O~uBHQc{ApMTV_8}?OtfJc3oVZ>>oQ3;={WMh8i3r-V zp{Z=Jo6PrH#B9K4WtIlH370&?9FDblbZ9PHq7K`c(!+X=hYbQEYK+xDw15ejk{$lg zUM>;skP~&_+94*~T^GhsL=0k2Yu~(#Y>ZAbrH8d{n_-NtB|b1U+or})Yh$#vG4qaM z{5RCmeAW_2zU~G(8L9hywEFPSdt`Q@^#tmQ6E>lc1Yy2b<_BQfL=TFJiu2Y4HJT3v z9Zi}A!<0hXoCye?WjRuT+#+aRf=)EmWe?3F_?Cl!!Z5Q_;XsIDLeh$Fkwe4&Ck|VD4Lknw?1ACVA|_CwSKs0@bkFWbE)VCmui<1 zo^q-5Cw$E>$GtOOlkae;w&Gsj1>UkS)=;DtMki-eyAYp?9re%(3v5B}DNPLx8kwLlUmC3CZb5I4rwb@MtQfvuh+&FkPxU!^x`~+Ryl0Qch=`wj+ALTe)Q;svFGQ}X6iH4EPY3i>pB%<1rkMCpG!3(0@gEF`sufHG(ka_gP2cRo7|va&C`l_gjZrhb2ke_zGbSXIbP4>Tmj?oXE$g{G1#SW(bH25!(Ssevh`)tU|phb1f6wPB$_ zRrw+O%36E{>I5~bOXKDYH0s`9MVfe5bV!UL@G8lgggl!I05RWMBl_$51CRfpKj3%c zyuPq4Cr?t&Q0YYwTPmV%7?At?xO z0x5Y7V;1Am6fYQVV>U-cD7hCVYtOPVhie0fX_hktU!zjdIga7Z8a(2#*9tnIVbM_> z^N6Tabf!{8af}Z4HH~BFQ!3RwbwblPMq8BHxY{1rbn=Lq8w4xF!~v^Dt;Ap_sI+`2 z8CpjcKkjnhUO+7fDtvDU>v!6{J~8d`DG9r9_^8dj)m7aUMF$ldTJ|N^bzjqx6eVu@ zr8M43%E=iV-7+~*r3#nWmUpb{l?@+nwzF=YhaM(la0U~A_gC=GQOWe=`$w}X| z$C(!&;g}!cj|H75*H2P@JxX+7{A+V(5>YuJ&K*D}MDcr=Kq<*cd6CyrYU|_mfQb3d zEaV=~fp5hz*61vG4*VDM9A4-UaJQ>CSNyKFMd6`p8g__ya}^Rf567Xp3wM1bst;8+ zgN{E;G^)9~<~5rsnra2e)C1*4FmP=1s*F2&y?l4RU~;|QBl-pM9se1t2w=n=R~6H3 znnM(|E&|`MvuzOpS5ha>sa9+*v9$h;^QYr~OsnX7TDt>tw-_KRT|#ob>_Q@vq98Za zV3=ZCyawz)86yOihviM3)TbIu%%OgRTH^1WDs6WN_EV){pWhqBD9PDRlIwY-Bu8J^5Ta$zGhRXxSeR6`p1Ske3((UFGu_CxX$@@iV z1)r;uft0w7WXj5)FUy`tg2*eYiuHE~psLFog*=|GS47Oa9WhVIOg=#9gBpc#SqMcN1!GGEQ)%%n?fMPwmnYqC|0+Yk$v@w+GtE|n%< z>KSaI%^W&uNrSod@$`=2*+-Zmn0o{kDj|%q75!z8K+kkdhZz0kjuQWF7y~UxYI{o2 z(l&ehczTOtqJs{h&Ez&K$tk(onwA!o(1WDG#?V&~vzJ>x z9v(4Eg6|43dPdA=N@m*I7^TIE1Z6LtZ}O($32#biymLGq0c0H=!M{>%rS~yzJ~?Qx zP98*A^}EBVX%R7R*xH8E0;@TtJ-+5|FX8(XF-k9Oj5K~JwjC)m#@%Kef6Bsk%(>E( z5E_!}U?$rhR3x;#Y^2zTNWvVHu`#ryu&p?M6!L1zcc=E>m{J`0j?d|uTFMs1$~!F}3@yr%7iEtkFL>B5J&-eLfhLj1uMFu zR~4uu_V$n7%EzE!B8bM|Ie);=0DP|zzxIk%IZo-=fJ*rBbox%;+TYrrBd9Z-%{ei5R+QLI%Xx38e@1#TXO}x8*xefO85Uj%-J%O{=UM zoAacHWmf8WN(t_D+=Fx6np*Hl=Ec?|=u>UO=du$3h( zq2|l9iE;>GAd#m_NlVs&1)){1N|-{S9s6LtC<%;4mKA|v$EvGTDL`LHm7se!in8j; z{zE9h5X~#b9M4B+DP2~hvtR2F{Gg$eH?i-w9F$HHCHIo@FY{!-`L2ISj;L?2dnI2x zCHullk}>CV4HkyHrKPkSghcD^?(*-pv5oHnhGN_LfUOu%nldL$Fe}7BCB^k4Aow$A z#eBEOe1}6Vm86`Zk%88--EaenF_0SFq1k6GM>X=QprYV|04Xyz$r5FyRo!RHu)eAv5xFDa?Ykr{te_T^wa5GmQvkF00X zn3;~59C{GtAfc}`7Ep;HY$%O)EK5eS=Gw0H1uP6%iORdg`aj65qhagFCw;M{6v%hl z=@>n4DaFJ2a6}fy)1WNfK>EcM0(i2h@;&R~*?NpuGz-=zX1XAFpv1zWjq5N9z+x*f zP;R`7A`FpX{;NjgaSW0>#tbAzT6IX1GRVL*%7EHrP~}I_5#wvdVuaPk1~uLNLta%RX<`x*;auf}Kj_J2Z!R)|&pAv({aD*jmRQ~O zNn9+8cXP>ac5*hfnZLXCU8Co&>6uH~KMLFAHu+NC;@P7COxwO8h`}QKlIE}_F|;Vf zP@IDL)jlbmQhE}^+>K8fU|$H*ds}LYZwiPD5*!P&?1jlE!hjbhISeo#*a(7T#{V?9 zmB2^2653824AjJ=rN`W)@-Z>)A^_TEWByQFBT){bW^0F-gw`jZQSnhgkqy3Q4V-oJ z2mFk~fUGMfj9-x;js)rqCGJK6Q&}(=rVi4pR*nivO|a>NjKo+RJXl_4eLkEGVh+a_ zi+2gVH%!jjMx$*g>A3}IP0YhZF}6;Nh_UzC8X$zsBcb4R|J3PxRmxa4% zl`chw@RvJ6489uTUSErO@X#R$aqlD6?|nR}#q2k&Ju8FS>Yas*-m1mK{9a~>F<6TU z@|Fqzag-g zix_imju(TN(+fIge_-{_jVaMZ^-j;3lD0ian<`<@3sR*yiqFAh3{cXG%l=_1X;HmX z$}xS#6vZ-_UEy_T+qf}2c+AlOqttB!PUT&Qcp!`F0Eik-%!>j zQ|d?2ixSFr#v|38$(!q%!JEyOl9<rj^%h$*p--)CMr=OT*DXuF&XTMyhBfK>*)iLbwTap2Ffq#yr`CgI@nQ}b#hl3| zV19K+iGHV}^avfL$0vAk9VPBNN~RSY4rY?9HDJNtc55AQpd^VjB}k@D@}Lhu9bnAX-(Y@PRqzWqO$QjvhUxrP*$*0& zoq2^Vkgx~R2R#*hlvJ{=J8q|Gq3#0n{am}^GN`r=*jPZtSm-e#x@^Zleb7OGXe;bU zg8z}q?~EK^(7J~;e|^ylYWXxK{U)Qy#Pb9|u28+Zp5HJ)fJOLK5aWLT)*~l#5c8mW zbC8d3_6KqX2;32gu|g0t;+GA;xL6`Mm*}icA`Ugp2~Bd%A_I?5$y`^)*frmXT*V!c zTS=o5sl+l&q1ECAKPhWMW)ti}&)USh0$2{CRGHhiPFKj2$Vvd@Yd+~W8*BTLwx(+j zyvFWb!X>KQq}rO%5yNKrf$j(>dA@|;>4?b|dPkx3j+^a)z8uUfvkff)pi zdDQuxBx3D@UOT8OH>sC0>tlT(u;s7bp*dtLYvS#}-}ZD2k}gUymi_%$j6u$Tan)Y^ z<|rm&Iq*EdjBE)QseEIZOR2(*m5Y)-oIq(Q8_UEmJLBT>@3mLLI^T#t5XJVDS|rK< z9k~(USYPFb)5f*zZ%D5e%ePG4#FgX&1Uy+$OiO^~a6UFm#Jr73^{{vOcVkk0O1*oV z-t$bES=@a0+T<$6$p>y9-!dr~1}ST?^}>d^#?vcYjZ!kQrPjPijAtq^Hll%;V>Hi4RE04wf*6_k-<~*-g$MWxm>B%V zm<-yyP(yrSWmUDp<)I}vxo>Y@s)3OUnK3cp2T|j|M$qD|DoQAks_J0BNo8$F!3gvY zB*?@oGU69ZoM}}sn{!uGB3+!iKP%Fk_k7T?D1bDcl_(KdQ+(cTDCHu?#T(e835dy{ z-UYRM@A6g-Cfs{ag06Ih2m6l*(t^g_@tg@yCG(dV7k^4}VGRwrA5HRN#gc2Y7 z9mrVBA!t%zac%AoJS%|9Wg(;gVWzQ~SJ+qwymS2l$4ks9# zJzr1T^xnU5YXvr~#cnai)1WMQVZK0y)Al$hXG;P~)U??7rc_|p@Hp~?SD?9MzJiiG zN4tXAyw-6IdVRbCsS=ept#QFhjm)w!$7>5-=}O)(Ir~S=oa|>xFcN^;nrfBUKF^e7 z#0dB45uT-DHZf}XVLEXzdzn(6`N%jn3dS!DDK(c1Y(l@(t6fT8NtNRA-Ak#G_GPv> zRSHChcP#jOL6j;g-|h93%831iowuImW?ri zZw-i9$(a4PIOYXyO)xQEura&UnDBW1l=$l86C&0`f}P!CdI2s>%!R%o-SmepD+@90 z(LB7gD^aSyw+l&aPbsT@w|RCZ;zqDaug60sh}rZ?xkh3(7T-E@yi#*ModGeIH%Nr9 z;+Q{Xux(ZG6`2y@i(}KqEI4!aiJm(J!a6gXgKt{~p*wz}YS%+ALHCg(-m&*wd zbA5y2^ABO)`4++dekWuE63^rM8Wi>pwz*{TrO5YKWBA*yQl-B)@Lf(@E1NgFYam1G*~dh#v+semr^QFm8!V~jb?;Z_;Zz` zlYXY6$44$>W{Q+`r2M#Lr(pM{aZ)_ zU$-G2KCpF7?A7or-eGHEEC$y)W_%48Z(`i(O%vmF%n)*qjEb)-;6>aRKHu-JnwTVv z;c9-9k9ZWgB`fso_J&S#U)&$>vN@hW<;@GShaIGHOibGx5HSDRkTG8qX9)Il>uCot z8?2Yj+*+?rt_~*=@dO~B%twmtD42%0`2#$Z9XcsF%KYW!)(tQnE{D}>ob9gCa_R(| zka&|J&UjEdPg_TANa}7L#qJQ=kw@Hy#GANcgzFeFbf>GN9V2`fF?`7gr=N9|s7aM5 z{&k10lGO3+EEgb4rKow0xp>rkbYzY;(+l)E z^WkJ32J9B#rY;`|GyBA3X+gj!j&NW_f$|UWRX2|9Yb++zk-Sf=&#dC>9{9Dzj2IJB z62fRn(DDpfQOprj>*KeW4t^5y@raz#D@+F(PeT6ZPRSNx_6xpx+MI;caoc+3m0};1 zo>$8^!8S;YomU^`zZsmPi$-PqCBh5yAW0!4eRy2jNk~R>P0Xdaf{i&J-!g2?*Hi}{ zc3boCWKDI@pAK8EbL$MM!;_tmvs2PDV&a{Ud}mI{`jOBgP!V-=njKPflhY>=$-zMd z9FC6hp?!X(MqUW6YdsZ3*%ftH-(-0+_qOgjdL!elHmOrhg~_Z2RVp?RyzhN4KA^+c z^HP?zNv>|UTi3Df*42#C8!bVtij2i=EGt&rKDa7I_|z&3F?B%&qmey%PUIZcRba0Rmt4Qs67o{Sphipp(FNvTn%+A#DEKMN7iCrojEL z>)z$opdQ3(N@z#w&MFB(!~$DH)zBf`#asePAiN})G7mGsS4as92h=bat1^@o#ek-~ z!;4viN@oJ&`IpOz>F}jY$-Q9#m}MA)5C?&mJot?{d@TEfJP*-8uL{YC$Fy6;9ej1XNPVi1t45lKD7tV=$P51(-^|k$1 zjEUmj&xj7W#TY$p`y`K)C0WKlmi8f6&xU8&7aK{@;a!yri)kKWVEfjE9N*!T=G>ak zh*?jzZFC4IT{tDZjAHiVULGUH0%eVu&nDZnMGV$^?bJFm;X+c^Rr`P92F>P}MS*mNZD)|1rL83 z{{8msc zl>i2I#7YXmuU4a^u9~m(p>dlnU;m4mqPC}vZvzl7tj36>rLZgcVb>C5F9inSp z>iX764s~IZ!HEI<4^&_pwZxfEdRK-}gzg$#^d@hDS|=rHy&ro@^Z=5or|_YHPkm z2|u1;wC#Gz>?!>Y)jJ*H-pA+FyO7!M#Jx5fbN{C_)}^8Y@HVwq2p3Wcj7~Ysiufuo z++sTzsT;+i!{bW((hsRkF12^e7!%ZTF-*#J7eV;Dhb9wlH?aC=u;mk%Y8PeCim|jx z--Or&eI`Ps6r`SE_7UbFLG_slly#;9ZW9fEaVVtWxW*m>9Z2j6vaDP3Ji!x^hbNe?E0WpP3kDmG3ZF!W>u4 zzw>)h44mg`UW?>73Wy5$6>evzGzDH=a8vN!$@a8;V|q47(=>2lmF;2kLdduu^#Pok7S zbkIFv##;?&Dp=PMSf)hF&NZ7#`Y}|5&G#RYALta&!sJGx>_3WbC~2i6NIDRPEd>TL z@@1t=&{K>GVxOYxKR^+r?DM+Tu(|RCh`H>JLh4uQy$)iDS3(Qm$afEG0)ki4z6lw; z68%*j0;*s0O7w|W(o3U}RR0%#gvUd*`;3VB3DT(Uo@VpRE6GKhdxIJC?11$-=$Qtq zik5|#V?z#zn7tW-rB~t!UDGB3VHDC!vS@5gKlmdw1rjcmCAtK2<8ChZ4T6W2fzeSI z!(*?c14|selD2gqKc61}gTeO0em?lZZDP#BnV2Z>1px2&l(fHJu?vYFEeTWc)7Aam zIQq?2;+Q+KCN|UWesA#!9_;SU<{;*JZI1>q&1z#?zE=t@X1LbHt5)sZ^_0*GID#*} z34;yXJ%Tm{*!>$ehQ1Q{(mfmV2sUQHrCQk-9oZQAJo241)gkQIz(vZSN{-kB?O2;B zyg&J#8#ZPGgzon|7hZl@4$cx}DO14)E`yl%9aASirGDpzPCn@=>1legemApWHtKiS zr(6gxdMjiM#7k0|y|*2^Fp399vmj)nk<{{>KY|j9g;}%`>@6;K>VQcqy z`NYHor<4L_HXI3d*6jkABT=gt!^d(Fc7BETPLmsavjMMi>%mk#|C|mO@fg9<&}?AO zjTPijV=k8(EzEhFefO6FCiO|uL1OsS!bqhO&_tNRhbqN}$AWti1r;q5Pb+FOcIYfH zIN76D;y5y$LKlI%1T3@$fB3XJWE)>+2PCC{1!K}uSJiN@m4Y%}%Yf*B z7mXFW@v6EheIGFT9;@nJ+{S{_@+O3+Z|3JKZyhupc-8Z9AaooVtVyp}F~SB}LYNwD zWq;`DNU0BPr~?B|V`878ff;%>N`9 zq!G5Ovw~J?%wal^>xlqhR#D8~oG2}v(%Qu6{U~M~t>hCOHWKCU7?ACwOr{fF){JRR z#vK7-HuugtX458O^hOv%Z9rjn{Na6Y1YbHAe7W;B-0C*q0Jhx;YiltNU%rbJKpu%t zqAQ<7BcDWPpG3osKMZ9MF^^5eIG^(NiF;cBZA=MLSlb|DxV<*^ZkY4Ro z7Prd55%&C;Zi>6IleQ~9e7Jhwc82958Q2La6t%Fo6+4MEr&|KP=o=Q&IGhTg(=2+uTV|g(i$Kru9zx>@o*Ke zWH=?TM%`SQSBp8FA+^L5IgL5Oi>}vwJ)K=3jM{I1#zr9UFfAXNQGT zv6D_(nUMjB-?_9gOLGOLsN=y4z^|)@}w9`@u7n!TY};hkmMX$QzvzOHHbOALy+}f;JXMa)`r1f9#9f& z!z%DiJJ8>(_dYu!o?hPKS!6n#u8W?Mt%vYXzr*63Mt&Vn`qlw4aLI#PegeGgy^`h* zO-{)So7H&BsIT8~OGB`cuX8dQ{fw{1JVN7|CSJ)lu4(W}_?>63q@7pVQgop1D&%Ll zF64|DCy}v(7=OdVtxZkD%7QEr<6nqad~4(Ki2>PZM868He)}81@ z{f_%oiMu;#Q2|&>H9+jDlE}tzSHDX&Ktdct!+xB7HkD3ur6+L=ccd+_Fdi{_CyvoH zX%Pqj%^EQ$xVNoD8LES9q|}YlC)j;d-y3#7vbPAVmZQ>5Ge}S294gqlRN0G!2iZZl z?PyEItZfGzh1Aupgn<&SO0Fl2aeG_wf)`ATB?u)H22koADlz6_R#`zKsd_<#q>A1P z)N81L|LvPf1rAdV1tA?5qADR6F3Pbi$VhVf)}+2b+xpsWx2^xR2sTZO-f>DQRBBiD zVt&G-rqU{94j(J*r$QR$DB@NNE2R4>{bSP%ZI}`1DZW_kWp)ll=98%o*Gmn6IsCGE z@8gwv?|&J^=y`}>&aJKDD!0NvLOSE&Z*Tw%NZ=_aR;8?NPyk855g1`haWIJyH2gYHf?Q>k-p%QbbNLh%C5P1Iz7GC zB#H!lok5d(z=@6w?4-w%0+B_&gJO*N?f*kSXaXwSLFgiT0`Wa*AM=8Y83#f2B)IER4pMjagNEw-DnS1{+o;{*J3p z+n~toEf0O|$PQz?iP@dcU&g(BRf~yIYiwN49ReTs(g-jQ8(siN=F}R=SiTnHngrT5 zBEJI|Tle#Q>qV-xs>RHyHMXtkzPe_Rf!TpmiUBo6Z=9ls^J^P@g*95{6_a>`xYz!h zfrrIm>%7Df#It4PkEiPy5VNyL^luDfOx6HLRv451#xLzka{FT>EiaGj#W6CSJ<>Tn zr76$GJk{UK3Vg}&h=_TEO85`qYhu`I`I`H4O55eT`I^|@tgcxZlmv#Z?-1i|{%{(x zb?lLWl4l3nn6d+dlec>I3W}@-G2hY|@;2sYD6)Q~;OmhPsetG-I1aDQuPw16qsw3C z?>=iL(akH|_KFuL^e=6FHHb0c%uk$C`cgSY_~Qk{to_XVbb*)|H1W0E^{xFSi}>~l z8eY_NVDTL7GNsC@-<=^-x~%qyx;D;hnG$ziY(b{7XhOi~I8)NrLHXp`Ix=4qj%>_# zBpsYn!nfsRN^Uz29(r-Oh(pT75J7AirbT>`LJqH*Qty_S9QUN+DnUw){P&X!JxbVY zPYRP~LnqUiQpvNo<6N&b#J$(+Us}G?YoA1y#hCR5^Vnj@<=J^iHlSvsd?(bR;Om@a6}cJl~hSP&~HV6GZh_-PihY5Jk^-TD!1M;F&OuT zp3;6XMyF0mN5vTZ=C~KopJ`%nxix54njAYqLy8zj+}_GvQk0~xrpQVh*uunEL94Yh znimm#Q8r@nGG`^=+7D+*ILpBg~S_4rkpp`U1D@(R9jwT(ITk7upu9UHSUFRsV1iPhA`?AVv= z09e5CEj7nIByY5hkgjd(o%J|>lULF)V%+cDGsH|eUMVsx@L+wNb4W-cPRA*IrN_NZ zTEIl>kY@w7M$PU4yW6UeB$1?IlP7+6J)Z$FBwt`NryRpWIc9eSm%Wwiatz-t$Kc7d z9K$_g*5w$zD#xr8HhEHx(bg+j5s(JgTaUwCP+C`%%BpHbRaGD~FPW^Vq}AR0y*KhM zbtNm@crw6B0`sAgsm6S%_@G;*d98Rw$J?#oWm(?J8}A15u;Q%lI8nMXw}cc0{8g4F zUoo!?_m8n3M*@1h<4_cYFlKIS(N*RVY$6~XQ|4zeA8KB+lhiO*Vy>aXP<6Pjz}Ly%)HxeD1c1`MODyZ9PJZIa}d+km&?&0ra3M7cP}=sZLUNgGnn{35!m|zyyI!wPjaQRpl*hI~^{y zLCoPD3v&MK?0VTGSVX@5zI&}hd-vxdejd3CWz!qDn|7a)jtj&bDZ7z>!k<7w0{{ez zxS9zT?$e|C3Z+jP)KqQuujyQ%DPi)Yq!FBoQNTJwdt}*|Q*$tgIiB~t(oU0LlN{r1 zj2lUP?v?B+i`d3EDe_IS>lf=Wv!=|hXYvpWoyW~f!;#cuLXzb_Nrv{q z_OWk=(qqvj$g1ES4;O=&%LXfa%f@UEw{d4<9@@t-GXYP#ibt?9{szYDEAW@y+n8TO zzzcn)PYHN#L|@=0tY&4F1y@%12S>zTCUj)cJVoNnA+N45N0I6hDuRFUC%_(B_mmpZ*#7XjX})y z`i3=J5n^&DPkbEnSdty+GLHEwpK6{w$z6EjJks-ANv>+XI+X9$To}EuNefFPT)1Ob zX4?b4^`Lb45tN%4C`Xj;RGQkM*Wg&FaCQDwqt>`A2Wo6m*vLq%+ zE@oJ5ZYrfyHF^||QVM0Ae zzTBZC86<#AIYhpX_uCkoe&>26%1{9Bsg2ndeqL$dp_`hCoZBX5<5@5)9hXzhb*(`@ zrt7%kV0jp;IVneij106n1Cpd9OgK{es`=kBK!S~bWm)wlNDeS`N@ZEA&RCkFH+ePb z2ov03sIOR_SE>orV-9uuhV>O_S2pS~2!YUvq`&<((TKDO7J_?m`D7;6fk7pRS=@4T zHbO7G)C{w5t^wQrI3)x+&m&*pZ$c4aj9$w?%&Bcn;CndK?*Pmn zOQ=cOF&}? z-r)At!Vg!1fmAbNxQ4YzY@l#Ejm;Y^48|eQEvw!MI9JrXjCY-xs|(54Fb=3=Qb3EK z{}|0NjQ8g2Zg`fGhC@V#Gh5%Xnq^y$f!o=*X zpEVu<@2~>|H)v?*Uwf%eOv#x2Jy+EC4#TxJj^JGcl#b2+H;hEJ#Ix)jO*q3M!}qF?_7%RVC>eGd_!n z%q3ixOG5xK=lAR@MwFz*n7x8;G7UNJ5_MbKbR5L+?-zU%_xC)AK?8$p6D?q5*m$Hp zCbU#K@h;%6>{9cMASQUA@F+r$)v`)nn0bBp!S*3>&jV9ET?fcY%XU$7Mh&6+fSHq; z7Z*^9OA_Ee6TJwNDD~vSL^LHR&JTHwYcw&Z;~%TZ>e`kbFlp3FeV+7K3T$pmQ~d?Y z2=I`WbXT*k6Yd2#vWhXFE#Dr_t2X2-Hs+;*FW)NoM#Y%_ynHv#6W33QF$n@fF$TNO zsmEDd;}H{Eqa{i#BPy7gUjrIj@UY7 zBMC+@Dk;X|!tX6+9xmp=FbwgFY=ET44ZoRKY$(is3@;Hv_)N>!e;MnTMGni;raHHc z`LkJON?$ZFkHo}KTZ~yE#+sQJF?vQ!FfsIH#H_h7UsFu9xwW9UBQMlzximqeolf#5 zh6~JSa3vqyYl#qn_!sWBU19=ZEug^eF@X-Zz#j+da9DKVllMF($2LH-Sx<#sfPssW zn4d=oBIosdi(~nc`N0d!ge)1n7|BTP4j&~o5w7|Fph^q^QPY2LRR~$)r{t|GhQRmO z>@Xc(Ld*|YU%|fRlr~5UbCjz?jZVjL4o_$`YD#*OdM*}DbEXtYn2DK}x8;SFok+3v zO0<-A=pd_aeQCkcw5t_4UqQ@oJV5)3Vq%P#P*9sIxci7%#j^9W1VHPcXZgxgo7@<>j~Km1%x|&j zFUqay#-_i$76Ingk03R~QK@~GO(nOxkQ)wZ>4S9CN{Y-+#IXu1x;8#;{wQcj$oWlM z_9q`oLFFM&)}ckje9J5SELjKmsT=s${+`c`9*&J^2v%Li7rKzpa*}pgfTrQjN_?XV zRwQCj;U8#ZOqL|PvzdIo82)Ao9@!-P!tDl01OnD;!anp+vVrFVI+?K_*{swHV=T~f z!ez7+vLH0CoB2UMV}f{z^GUFKtZveAT*+PYXSZ z4W;j#5tAnb@`Vw8Puc|RWUb+9+r!}uh&g>np6Z7&9n8;|YK2FLc^dg{7Vy;4z04o! z*H~yz7PlYBHxLtvU^$}_7@k@Fhi*z$&9RNh^-AOdV+0$3`BEQpk4(jNa0`e5oA7S- zA$1^AEyp1@dj$G85Igjhrv{$#HCNRKVBFe{e5YEQXQm&{EL8XC9DgK|Hw1 zG7SLUG&DO61%!*iT-K2O;dE%a_SCj-=qmk?0^aw!O5PL2un6xAzzHAa6Yv=;OZOGR zm{nt2tw?!r+u7`Ab~{dPAv3K6Hm-Y$L$?LXlBYT!Pk$t-+H&pn&&E$=7a&^yL_5WwyPN zQc0a;;gw!)M1~+Iac471bh7g#iF@%$JVdh63YXp`1+53sg=>I4!}AJwO>^nm^Y^4< zeimY!h1raJ=UbWF`xRi~*kwk{BEDTCCVl4yVzwrhfV@r<)52$%Fjvp@oue2rLUphu z$9N^A;E^mT-?qLUYlE0m{e}vj?K}Dkp4XRutR@MZSDGt$9Pj*kfHr3N?o{xWlLRZ4 z@9QJiclfE@=*etBQX2!(EB2O3d#9ucOpu`zjEd4R1ZK?XC<*t6R02!GYBpM21-GW_ zP&~$OmVBx=F)DtH7G%T1ERRUmA-f)Ps@o&16+Lg&V_XS08deI3nTM5VurV01_&6qG zScyk*4D|!Fa3|fclD1uJL=1gW^L<#v*i`xv5QDFA?h!+EWMr&ubW9A37;J7Qj?&@3lr2<6%0%FCWwVc{YP2!sCq|m% z6W>wBY2ViX(- zV(Op|hvP8#(OYxN?#T3vj0LZ5#f|I6{8x@b-7LI#?0^hUk*(Jh-v@Hg>UCd5n`y^M2xVc|Lt5U4NbI6 z+>uHzO@bW{Mvl3X6x@ZriwSNWrT2-G`urf_k%wW{eB1GaHm2EF#fA4QOPN2z)FI zhuGv+{Vf9&5Qu>R2H^X5tm+lz!sfzEkbcCW5%XU~1Ysp(AotO$awLL7Xkxrt57V(_ z2@c7;pY%|9K~79pjs+veMHOUZ0yJ7Xnknnhz}DhRdw9pR(z{5T`d3u&vSzLwp~|=< zDpIfHM5-`A<*fn{cy%x)17=Rp%xJ9N0gFlx(yDz+9Mu+@3#~+}IN-}X=JtOw$g`@F zmw+N}X;C1x%kjqg-~F;=630xq$%tJDYx5iBuEOky;dW3aK0sXt+>NSKPd@H0=Z0T{ z5}XjKYzmVk2?o2{C|n2sw^{U<=1gE*>2f*c%+NmmitIWT0)g9Z3on4d-*1M-GUoX7F#j{Fm^kFvG8UNO!i3z}qKOCdl0nR=`QfC^uiF^wUfk_; z_u2i@PYX572rcdfd@P@mHgzcQfna8AokFqMIK58ls9q>-gjLtPwl0o(QViQ^OS%n8 zXk$VJZ?KaDpowQ3Uzxj%>v}ebxgLM~GOq056RhCjBHAO|he4t&5Ym?7qFk^sYPKKn z8r2I-fZ`NpI^aII6aCc?$2eeEN*9ol5lb*Uuzc}Dt?x-xHfhjI zOoScgROzcF-`9m5Tw+bxhOmPdZ3w~+j<6ZGDd)E0HWh}$Qlv0vGQS~iLb><&FZeQ# zafpKoIypRnRDmdkNKDNvEABS}N*64{x+TiNDX|=hc;PDTc(jPH|8L|q#(%<{}r35f~O1)D!xJa9lKBAeg z4*m{~(xq#g2N!(%M8FE$j+UW<*Tb99gJq=F1~6A#bA8@aOpK~`de0?s>qWTh9a3*$ zqMlOPA$WwIQa}tU+~2E^OBU$t&N4w5@A+CCI>Kw7W>xizceXW7z}xLCY55q z=&0Q5?aYSNn5M+#ObR0&Drc)Q^R9GiZ#0HSwh)+FsD(G8cFE3`mi)4RjZZ6_lrTYW z?_VyND)L;?k(|3jef*Aa%+CT$(ot#`e9eW0$*A4MfSHnag{OY1ZLtx1i%Vr~#~yGt z0rTV_7|6V^6&+j>X`AkF_!eM%e;M0~8Guk5&Dv9n%fZZhnS<26VQUG^obk%Y_)G?`ev zyNqGU2X%;EGs;c^??%0|#;YXvI0XQ)OY>8~{jd@x|5dh;(#LPFW4FACTs3A-UjSPfb9<8TKV zX!GoIUd+KH)b;sj0Fya9`(L)KwF8V!0P}dUW&L>_hPvGq!yp(>0K?}Bi~AzyI?P=+ zr;|R})LI>^6H%(c0Rs3JgAvZnjk%$>19F~HjvLdNQ@x3^Yo-QLaw!CrV@c6UFl9oG zs;tbvurw4F^I3XTg!(2?3bhw;0nRj&QlBjn{T&FGx#9pi&uz*2Ml})RiZi}-pmLP?< z2MvVftAx58QYzDC5+uYlN|Ie4CTG`3H449FVMeG>6hh^YGA&qpi-_I!XoV-3N#^>y}jqle`}etvV>y9@>Gc zwq?2v6C#VLxmi6#HgZ~)b}}2c5Hcvo)b$PPH%ikIZW0*;6%xi%da9uF zU9mf}JJ6XjP*WA`MW@Kq6g`w>Xio=&m$J_Q478C9URHCE3y~qcXJFFKfHyGIrrWmA zOJlrfPioRpR&9UdUC~%2tyomMd5P z_o8!t$HY45^=%^eqHxTHIfe;)+rrybc)=Mp^N>@4^j})qv1$czQOVeP`K&AHLO4gu%iJUzK6>OuPG4`?o^S z`&k)=+JUxB5|zKHNTTxLPn_E%iP8c)%f~S>Z+;h4cX^0p6$-g3CTumB^RD0^_)wFw z9HE*I*|Z6HYW=eMpZCQjUVer275%Ica{a1 zYg?NT=6d-Khw7JP)q0U6Y5nCSN$+RX?%1A!JX$KhUyY*|Nzw$XiX}2;unapi)*%7Q zs*$o%Fal|uRHeM^8Mv&*c%zDO6CziGR!uY9nxQiPHMhoi9usI6x3Sdqt-0}$7W7uv z##K;YYZDW(nqcF@{CFnPVFY$g)SQ`HJ`@ZXcqSsIl9>w%8~Z?+r5rK7HJ~s741+yf z1Q(!`rHSav5`*2*&E{u?FfD{I$L5f?4gGhZY3i?7m?tr>URW6Kl13`_ToUXWnH`H= zb<^Ow954C7g5_AKb9&$9q8fb6Q1d}6*JBFr5QF*v31M)KCc1?$su+9dE5ZiH&fI+D zCsZ9J=w#6MP^LI=^JvdXVv;12Jyqrlj2xhR8HB?oIKCIa-|UJW7hIc9Q$t?w5md~M zhi}aeL>6X?-H=+CEtckgS+`rYlEAp2yr#;V9q`j8RWnTQ#W``leQ(Do6|{SAw5DZuU_*V!cov!uc4&1x!6Ce`;iwtE|JWv+qIY4SeWDS zJyH%oqW5SIvC|pQOTaAPCLwBkXAP5p17w;NQ`5(4`)q^P32R;2AY1Q znHs37!=@xPS(V7rlt;jvj@|LperM0gpUu(4V@-$QDMxb=uF5UIY<3lw zwjin(&mk|^k!}uj{e!58yRsi&&_skI!JPobd7(H%eoS7LWX>B4L%tRC{uB#C=Q0e1 ze=!H4o++t>C&9p|z+`#a35gR4EG8F+jT3I!pNUI2CO*_zCLuH*xq|H#(wUdh{7@!@ z;INJWQHA_CN3@f;cG$6SsSd0FH=7Oj=jv=UB(QB^ zVnq;dTG7S`g`R%2>jdVB$KQ{M9}OnYh11~99=IRANC{d+X4w*jM{ z#UydOn&2w{B_+?qHM{hk?xBO3UVOg|Nrx=dh}pa0Tw73z$qIVgram>@>2P_^TAZJt zU-k-MHe#4H^J#uOBP*VJb)VqX6lshGdyAvHodJmK69Y^t(N`4j4jpN(M}wE^ngWjZ zC@bc52FB|1E(*Q7z3QmY>#D?bS?GPiBki=3&~oW?e<=UVBN=CZaDRb2c(F>ElXoQP z;VvmFn?7<2F)X@>N^ztK^!{KZU%#pld-Rz#^M=TP9 ze(FDw{Ghp#AD++qDL?aBQk^_a49pr`_HlGVPreAeIySt`r-AS81-3a!%ib6yL{&}% zbGpDn$jd)M@3L&SIWlli+kjbFjkMUZG$_f{alJpkc`64us)n9Z zjrb{JlVfZQ<50nYT?ILSIX36EJJ*M;m0N%Oj0v~eQ}_qpiFpGX#sdbaQ)_Yqwxa`G zJ8?0SE#(ceQHe;5(t3H9rBt&;Q7T23v=)XVi`Y>nWGuI8KdSx^7Qcxm5$YPb*+=eC z1JIED0y}>6F|{$2X933i*B${d-x+)SBL=JU_dt5Sox;7rGXp&Nl~xS*ohSB3W6CqR zTP?H&L}X9k!6<}%#AVYUq=f$j>*L@DTQ126_ZAh%R9GFbfI@*H3ss*g)^Q;B)?DzhK;+h(^yW_>Trgc@vaN@^TmnW~gfPI(zy`+LVsj;{ zaNdB00qJ1?)$q!AC&mh3RGELXJkkLK9L~+*4ePd0a%f}3WYtJoaT{oeU>=cy;Zc(G z#K1gBlJsm~=(DPw-mTg#7tLDxFXs?uVPF;&+G%ZI<|FbmQS=f{CH5t!&$!pj=4zgD zw4WngKR`N8ga7m#k90hot_CpY-+yA@JtB8I7Dn5j#j>Y-t2*L;FUl}mt?DU_M|^Ct z_yqSbz9dA%e!|F$GQfm^b1X16fZJNRg}UKNC^w(V7YlO z%Fc$jRoPk5O>=et_S8^zUD1iM>dLYc#*=}_T~%c@l8lKlfVphWB)vE>K!3N1fyY-S z1_HMsnO9E8FjIIOj71a)d#I>2Bxb|pKPg-jD#PycC+q)`b)y;rBm0D6I6O&lOPW;n z$0<5vJG^mvYKVY0MZ@NkESGd_i2=-N%j~yf82abMFrhI(nUe$Gu`pb1yQJ8Btn%&X zkvCM!mSccbU{N?=&c_@K3;jgQA@iuLODf0gLHxFbwI=7oMj7U1z&vcQumKpv!z1bp zq=0diUaEc4g5Uc~fY~d<@O;3)u#Z5OBtGFbq+nQ8S7O9$K8asUG(t9jvN~iYSf=Nv zXZ?7kx*V58>hq~PADcIH27X9Z?Q}l&_?fXW*e&$NPou{v6jBgrpH*)i>HYo2!9?j~ zSECn0K_;tiM-J!ieEOF9)K8(?EwiLOW#_$C(xzF_PYAv1nk_yVgrjC=AmEe5Y+IP| z4}>7hDg3%u@`7H{bmz7uZvJ1(?W# zO))AW3zNy7f`3QoaIx-I-^_nCtLqytZ-vQms=8LRFzHWSlJP@OWm2RX=n#EzBgL(} z!L_U`i(65aHu+np5qnS#9o;(-hmX zFupmh=JZz1yCS!yidHs7O%iH?gFq!jZSZRKTZm8+2k8|p#i|`pQfLEtbzb*nC~0X& z@8|28eKb(J^W0d`%5E^`LK?)WJO=pja%{RwohkM46K71;X*Es@puk-M7#T}wI+jyv zJ=PXRP4OozprUR4#g9k z-#bgn7In5<7$n^-Vi@l4v=diV31G|z+S$eFi1!{#Yz7QAIrsW>5F43CCax8wZ9AE} zYBG3$pRq6!b9tp}?jvrdN9 z-LNp!-Q-u+-~5u1ffv@_F!V}Wbr?tM27MNh1icFX0@7Gl;)H%cnp&d58mq|*%c!Id z!xp<^!=2k1Fy=-mT9Mq9=QxSqw%z4;`B%7$ToT1~ z#2bK_?STtX60*#a=oHId26Ln4!d);%10P_PAR?8sG+_$D@1oy;LjO$D*&cpvU;LVn zpE)01a7l;bwdtqf#e$Jo4$-?1>0mO8O8!N9B$5Sx?Kv~1V!qVM?R-%7!#mWcev>4L zeltn>KlZ*Xv2k2k_W$jLlV*svNX%M*_P>^gObi4-P=;*v=_>bSciFN?$*y8J$H++Y zU~dv@>jQAH*L;t6g717%g3p?o8yU3|nOyaktGl0cgj7+={s zLLL6~@4OLMKc1MLtuPE6jf4bSj4gq%luhGo!_#0HkoN6r{_MkK$9Dg0@Icc%jZld6 z7-^yD7>y;31Q! z8*FDl;Bqtfvz7p(?^>n5@fo`z>JU8ZJ+7=uh8Fm_{Jp(0P`#u}+ecY<)L$b@(fVgC z43xdrxFdRXU28T;C<04D=Z3*hK?{A>c5YkPDq8`V!|`x2csc%r0?uiwR`xPp@W$nM zc{2r^SL4OAVK~g7siY6&cG>Txya$;v?#}X#sXPg+j0O zE{Fggrc8J;WfI(!8?1h8mF>PaSj}xx4Vc-k!st-mPh3n=vW*Q+9J{EU(mtw1%9Jwk zE@DktGL5UpIaFEAAy2~*9O^F$#5qJFwJBr&3cQ>yr;EYMYOmIR880>dXt}+h9b=yvRF0QqJFLaaEGE6+ zMctYDn5S7u&2?h1`h^T5g&G}oYFo)Jif~s4_sD=E@7Q`ajp)aIj&ku@a^~?m^!+Yh zto0o#%rBBdtf0d1)Fa8r!i1|A*ApYnzh#6j%#FB6&qPMXpLt?G)Iy|7kGO18Kv`6| zscKY%e&)~*yZ`e^fo;c8Z1pWZDT-%Q@JTwdF=%Hrw`tXV(W3F+Th#z1gNq_j0Y(>NUhTUK6#%jlF2kK?NYBIB?(BlIrzYRJS^db4IVrK`OZ$QO3}%zvzMSEf%DhfH8nb`l^GwzCqN20C}^C7>9q zxm0qmn1ubH$~rjs0P=`hbru@NzW@Wqu7Wh*9|qc#?qJ9tyV8?rKhyv|?Bhcrai-To z$hK{+I6VNk5`1Rn+ly8*0^*B(Gs0p=pAXy5QcTmt+}Jc$aK{^KrJDShK*K;H(LP=* zl(Y|pz@|JC=5^^#@v`oq(NHR(x#XPxcjK} znxP3@2RBjR-xeQ0gWUcc&cxWL#Fy?kf4JpLJeU~U@6xuhW$mc{n?6DqgPR6+v)I9J ztlub$zuma?-Y@xd`(*XNZgP?xn?WgD>BG=Ti&((CrB~C!mE)q?by)9z@YC(thcq@*fb_&={f?N?_2X-#tx2= z&=!+sa~ZrGHqXL?7B;Zako1h_h4f#m+gM_ zqXsd)UrkrD)!g1jXJae|uwNa`R!9A6$8)IRH|Pp}19$ue>N^p1)vwkIWKg}~b~B&V zM18ea`j$J?`#Az;Z+`VnZp{%dz^Q}s?>wtO(p2|71GIqBJZvD=wlUd#0KUZCk}?&x z8ezmAm)e_~^D#{dXPpET{TdUz&O#%FuQ#`bn~&T;foa(@bH{1Zf%EKH-+@2_BLcuQ zUO!>i{UpqV26|kXzfEV50gOEZ7GgeJp|U#~?6so-*UUx`MI?k&ReMR_17JuUA?8cK z=r^Pon8x=gy0Zj1{ue(@mVeXR{OhaCFxFbIYnKA z37``8-s8YNdN1lHAkBkGC=KZo6os_8Mg7P}u6oYf3B4pUCTE)ZafGd~{Z+P%Fh<(3 z8(Dj8KIsG*7d01ku$T`io7TdBExN+;>Z`Ftu!DFuY1Gx=hFq)K1H1E{K-^1)ynOfH zm2b!pv%-XhN{CI>X?pZh7C*Q5Q1#23+9h$})bBUkE?0vsd)+Y#AGl*ud4m z#>TLAJyKNv#bAX2qh|7BlvtR#0)d9Gsdin`uaw zt<)>Oo-|q>ZX7zG1P{SP!zM+RD3>V{kRXrB*pC}7IqyLu+?Dt-t(Mp*pv^pEoXgC! zOnR!gps?p07gRLnRaX&OkS2Dk)6qO1FvA|oO{K?HZe4B>a2vxu9+o$Uw>AX6j^v5v=z{G(~+1f!E0>=C+ZPk{#?wJu#WPuUIzo;G*<$E+X+h=$}vy1dOc!Zti~)z zSl9|OUP|Jbozc7a4uZ|4vSizTPxJRxSKFSlW#U-ebD_lmh0|`KEgg0++gl|mL#{B3<_abLy-?j;e{xE@#+#vG3)K+HR;F%+w1c{PTw3cioL3#sWc zI>47|Ow@(^W;KR~YK*S0#(byXtK+lx_=YCuycxtCV~!x_Wzq*wM3<0G-|5ZF_Gg)HjVEkTO6Gy21;i*IW62920Phu~FB2k=OXa+&!b zc8Jz*@jDEJ0Afx*LCo)(mlgJSlZo+eCi5-n<*}cN!4zQko^iGWU=Y6aEPkev=y2IV z7<1g0<6izHVtg3$z_Sq<-1n3oHj4Gtn9!P}sZ7ixRAZ?2Ve^tHl_2w2HLll?aoo~) z+{9LxLm0C_#)mNis<;wGkP1imy~UZ>q^}9z@zsu)M z^ftYhvONb4TpD%t{B7qLzIwz+*-JM)EpJlMJ-8S1K^<)E*SMjsgW?Ld8P&euv0Vq{ z7Y38l=*P6VXt#5cn!_M~kOyjpN_;p~@LqiMNb;0w0SrFE$Q*{Qtz4^?t=sg3P-sjZ zZs)H_D{b-(bY62v)qFb3GE$SPPtdbsb%F2cK;Yr&9h>}qNgQK0`GMA*xgm=3xHY{< zBHBmU#7F$QFDJ!tVI%;92dmNX&a>>$lnB*+d{Dm+(lS5+3+^3pLpjq6HUvcijVNwAhq|7G1t)bAO3))1PRD#i=^GVa3~BQE9rmbQ zVFYbZp+}%CW0R(B$bW0Ci4%%pFV^p9;xZCWgig7e#hP&R1p>;hRDXUf$XRglwYb@o zyo?xYMq4hb7oH!E7s!>)eO-^>PpJ5QtE;rG`1;1RJqqQRZNw~IDLO0eq2`+uT(WoY z_*qIZY8ke*(~)Sngw&HsV!ZP(#ssNz9A!H?ghq1(t{wOqCO{RF2?=RWOUUh0w71WO zi`G1=TFTlhyc$~8W7}Q{qix#|KDXx6I2->p!os$yuF`NmA0hHRuhu{Qce_fQ(XV<0 zy1Z+P>-##2EhA=q6l_S$7=7nq<6T@}jIz~^XB^$kJLFI3yWin=v+otLR z;$IWSSW8nS!8+{EfSCR75ir|Gm8wbMUbFXqN(#?sA-@lSSB~C(nqOKxaM`edm`{SnDD4I%0M`IiihO z?N9nv{VuA;ta9teliZp<^tu}JcuQ^_x=ODGzMv1j&aK^DE#%fREy85zh|r({Akqjp z#R)4$w27DEm4iqd$7uW3mQ5w9`FEG0zu@Z7H*kE>niN^#ItgP|h~Yd&%#vHbkScAd zm@QqUpKyHnVq>uTqhbVjJl)eU20!_2H_`RxU%+A|7%ce1+|D;w&%syNp%!)TLkzT_ z20l&g$ox9m2v9hL1ZCY%HAb8Mn4iCucB2jkFpRndN)xIjR@s=Lhrsu+`-zRA7BTq6 zj}e1UFW-7W4C2Cmn}8VaBqPM=4a878i@8ROM+FUY1l+IOD%9_6F9<65M}4OQjFVD2 z?AfjFx^5WQZeaTZQ-}Rw&-7tZ_YU?tcHN|Q<~=&D&1HSpPkPY3(nCKCxBkPt$#84U z2Xliy+P;I;u_Fv0!PHUD&9kI~@uKz<29ZRIwObex=tk19R(3Ez0A&~R5;9&z!{N0s zPSSRxd0mH}%xfEB&bO??5hineM_@JupOZx~vLs*d5tat${af*P88rNtP?V2AE2LT6%; z^nIO-#}A`#oa3p8uVte^hM?u}QH_Ne49`JMMXR)_dYp;8T(_h-t-opc^XfyulgVnbocY71ZT!(#6N)y-r%{C;?oj5!G95K2S_{P}C z-OPfO=*_&>5pm->c6RWC_$}5g&*d`j+2@TT^;5E-9xbalzPs~fIKh(QmBi+MH&vo` z`?>ibtZ)oQx+;`g@P+ zQ`$(FpPQHv*y`BCq{nmXksthp0ZA~X3%u&aAuniRj_*dy-|LhtUXZl;G*j9}O#Wqt zm`Saj5(dAe0$pqT{L(I1CV&bGlDE$eaCq;Na@#@BJL>svqmr*SXa z=EejNWL?PQC`B>LJERx5*fxgY-3gC}bjwzj*IXHXEw>zN4$pwFO7ql`e5WcCbATx3 zvf7XHN^1Q-srTx)6=NQePtxmR4E)psS9I8OHAw%v?SJ3xz9i}Z(<|&!!b~;iT|YBd zQB&LnEBBikxy^`%5R;G*0^A4lXEdL9OX;LKlgmwWLu5yqxuJxz20;OAhq?LuM8QO2 z-ll0r))7h?^BVzTtn)kpoq`(F)JeRbO6Qv`3`kn^QeT3B|=sl~=DSJz}p zZ!F`%++>?}2$uM!#>AYUrL-qM8Y1{zEv2{#2{+4H86`oQG69UxS$Q^`z1uR)0v`-Y zcCLjW!R4S}I6k3NDEpcBhsr*5148NM-b*QTJ8~;lP9@Y^?48!wrfvx=Xu%~H;AMap zJ{nUL2Cvg_*XxH%Sa)HJ4LrqE2Q33$1(*SrR~-6PgWx-(ke+1uvIt0wn3%5l;J$dL zzcmU;OLR?VV)Q1J%32rkbQDsl)Of&Wl-D(P9G_4!+8}B#g5X4bLW$3wVk&w+yC=nW zmJFaI0!!PF*tjEC{ku-iOWO7)ej%I%5)UZ7xpIe)9{BUpi0Bu;up)4JWa#(6G;bqU-XMP zcD~OqJ)GI8Ld}c;*hO(+m+VF|OGO0}t;wih zF~(SFVov+x0RXcS*!)f@2JH=EgvNl`Y(Y9W=!bPu79kk9SRq2QmHKxpzh0HbMc6G-lW)IA$l^lE!U(sPxVZK@D#J6K|N!T zn>UatqxoPA*ABpP9QfZj+7`CskmlOVurlR}Gbh*9yldUI6S9X%$iwm9Dc@GOkM)W|_5kr~o1eN4lcHDq*vOp;pjqK!e5 zQVwInVcx;{B;47yJ#!L9XN%Qd@2xU_QEVtBo+!E+J4NkWSS;-EQ3eE96+t=k>ncFk znP8oOh5WfCN|F{PDrt9Zi1{XVXhe zf>X?|d&D4DjvMQ%-LUlpLys+CChDC!=xu)ooIjl(J%^?LCF3lOuQ$Y8QyoC4^;F^%+1`+8_|i{w)JFh-I+W3vXBB zl1}{|5Ob*VI6_)lCh#_q>M~=5nZWN_49fqvIIn z5#>oVE5+EX#^*_a6{E9KjD9m4JSxR}<&=;Pv#I9b0b>c@*}X2tCebMu-h@V`jfe^{ zGUQG%{c8CZLZ^p%v7Ma*&O|+2P2z=y(f~c{!-bSg%(~i3vuZDOzrA=ZIHXdwm(!@) z`>=R;B-P%>qfqVDF=Do=z1mfKHxa{qwbz>%`c-C)!}&=m2FCNVyUMK`tG|^jl4A8o zsIUac*mUUbMrspSEf8{IGPWL5#9#KvdAOC4dJRdc5;42sF!VUv_8!IOe~5Yazt#V6 ziZP&SxMgExF;TLac(4pQ5FVmyHcSx~V%*D+dG`@sdiUXy)X^plEyOl3HU_h!F(&Jl zd{8vedK+^*LksfQpUZQQkFAbUDe(OzTOsRPkUB=pEU}VjPcXgM7 z66Y3w;Wh1I*Ff)z^&{`$x9%F(pf$ttUC&`9wpX2E=xd~nP~2hi>z0C6nOFd0mqc1s zDx^GARqrk*P>eYaLtcvc3p1s+&JT!})+r^mn7dy4jJ%_ipB69%S zgaF-1iCgRM1bsZ^Bp& zj#Xl85KKOE5_D?YZfsl9G~*Q>jkDq{ff@a-$|;>rC)9l&{+{L5??#L}UeFlQy0qMS zvySV&M~sKK?WmG#v!j_?zt`GUU#Qpfrh*gihadx@<~lhMa5hw735L7ALJOX1WCY#= z*AR>fb@4{hc0dmoKpOv)bxz3yF%b72ENJCAgJK2pUU!OfW_bWfjLIlyL zL0TfNaQQa4EXAoKR$;ls_Ons*C9+FPi~0_G*=!)K;JZOk(?jLOi_%k>$2bws3$=LR z@Xf?eA!&$sM(!Un{|O!#90?N+3ksIX#ltlw=6pICz+8@RXj&udIv0#74G{2D#SB69 zj{}Cj0+>?MI;1!YZvlOg3@MGrOWYw)c3_|s*hEh#AI7B3OlOXy(sDV8KUTD6X?c_$ zQHWg73^yPGlpq`27_Vx;45uBY)?}sTkP%~YCRK=HQkuiM;A=~7p?t^j*+7hOPKnwJ zoFlV!xr`XIKt^ZfJJ;oY=9KWeJ%>yZ#aQ`|En>_)LCpH>lS3zEzVy)=Rag#^HXxp7 zfs-c2e@F?rw6Q;O9J#TAsu2Mm=#)nG39-%y5pOn*1kP*yWZ=z+d1mIfwTKDA?j1UAUUBlzWR@O*AY%?4_}VDP`872WbH_KcR!)&GmSDr4y1` z=q&f8w!i#I*qA!OTOR%fj71M3OSE851S=O(dh|=n6o2G_O4=cG`8c5j7p>_*?$0L+ zm^`umWyQN!rXVSXFiu!A3|cF_+#%)FyD{SkaLR71_EtkFKR`XqJiaLZqZu1 z*8)Y#jOc>;2W`BUqZr>T7l;hMBD= zt6Xz(=t1-D)W2gdPN2oCc2+!sh0(YoXRVBvutPO0^pm!pGUC&d;f31)Q4;?+@)GC!gi364oUFTZ*sawY$Qv0fB)NgAry+ceDlSow9llBdsg;9*)BW5JP^V7URj9_DV~n z(lJ?+2NPo}Hi4|Ey;I^PvgSJw6Uds?yqHbIERgVzO62W;DINF!d#VUWF464Fj-9?82&p3IwfCcN1R zHFS*1&YYox>|laqo(+)ZVaJBsO-|j7?LKb1T{m?72XY)fJ_dH1`VY=KKGE)D_)yTT z)^Y-7`0)Ah(YDRUNAuBre17!(N5iiz*X9o6ZhOKS6Sw(ke~u$jNP~7YrSf@+%^%C8jVedO|&BPJqm+EZAM4t%6u^k8JWF>{)@KA znIjvQylT+;bUIvzbAS2*Fi&fDc18_7NvU=h%f&3k-c&Az z2Q7zCyL-fPF^?yc`P-!!8hjEGyj<<>3$CV5lyKg|-#pYS;R{FT-b`#enrIj&@;eG6l9E8z+dzmw*V@<Ivw)>N;5v{rF5b~4ib03;t>Rs2MhR`7WraU!CnbP^e@+t&K=IB5 zD-xQ;ko8a}VX5zXDYKQc(%gLb&)k2ikbCLNbpbLQMz0;@RA^>`VL|)suSEGW@N5v) zAeisyZa4x?Ses^zURWfmCqEg(kbcP~mVjwm@FI|MrI@eY2qD-sN$7AZT2;K;oyhrQ z5VPO?pVo8Mvlg~z?fDI_a5YmC^OUO@VxfC@IyqM6NUhKN^Bd~D5B+@& zGwdwsWYGGb(lC1p8N1}f4-iBXL0@t;Ww8wz#kWgk#yy{0kzB1&oUs6)YCMTjA;+FE z?#M)3b~&4zAdC^%#x$u?RYwWdL7oq%_jHtgLlncXZpT@1ejqdklv?O75uB7#k=a;K z`DZarPXUcB;k$x$T^HrnYTTv(73tY|ffxK9Z z;W1)-E&q3{gZz5pWN|0ye*3ju$!fgmli|(u+QNKoL~4&{(_cD;5{_keWL3R)Z_W?r z<6BnFT8UL^6^3-wAxIG8>b*J^F~jPJbw-vL<$6lmeot33rVEh@Acy()eHhR87_bAaolAT=+u-TUHJ$8+VPN_W z>WF>E6nq}~{wBN7RO)zVj$7ZUZj$EFWM1*12H2*4>)XC>KR-S=>_F$O?fNuh3yOD* zKSub&nx?Z~!n<{2LuM%5VG(asoqd3p;yZwtkh_?41h$1T^Uz$y@Lk5)zL)u`tCMrU z-1_*V4Fpk7Y3UGD$liqxf$b^rRW;^)ZR?0h^;J*F4-oh%@wJT!+EH0gX<-2SDCeG9 zl!7s`Z|C1&X_JHX`lz=gFSM;AJ3~7`1?6hu)tk3=`{A+&#E`n4(w{JLqQBF(eqn+D z0$$fs+6a7gP%%pb-&M?;_?eXG5Fi!7@Sf5fl;G@!B2#^TwK~o>Py@1hWu&zw8p%M& zxxKbmkjd;v)s8olKu?@EH}fn!aqZ3jtRJseHGyE+RkgHjfS@HMd47Rz_!fnBf;va~YNIINoGOc>qxdgDoMg<~-M^3&J$s zcq&~ayg<#uLBzQ=ybIS8Z%rI5u9$Ak%&QCqyh=u`H0A_Us&0&}1}NG25;bE&nvb_h zNL!zSXXQ2QF8@45d;!8hDm{Tp{yz`6=BF0ev$$*qG= z!fVtD2qq3zk%ka_lg(FQJ^r zG0W)J=PFdO2HbSW1D^!8oD6Ij?5t>;v4*X=-yvX%)jL`UVzjj|JfAqR_uVxiAGQx{ zf2(au5I2%WZ3( zg}tsA^Jl3&>u;$&gOI~>o+a?VkidfRif||S(y9jCAwR|hqnECyt=!%(WnQgk zjw{m*Kxb80AUaPN+wd8!#XQd}gt}^)=V^*?;w4#Ohtf4X0e`C6*2C#^xImwvFQ?XZ zi18){Q|2{d@(#ftzT6L+kX-QZOALkI`|?ANDCsOmc!yOvaIqXAe+6)lsHY#M3>bQ~4{GhPT$S7t4<5$x7f9 z=C5Bs%>Hs1uow9TGS#1LW6I^bf{jVKkTSh7U)q>Lup7Zc`=n~kZK$*{KR+fL>4v^_ zJZDhlmCl2SW5|9rJNxH$2)cw+@FoiN71K(eIA^nIK|{$u-RpL$k4y zfEb@IYp3`Or8CE-@}eykF*X&qJwI)mJ{)hI%dN4-=B=BfLeOODga#8s-PFlrJGZvq z+{(n-`rT78R=pJP=&UF#k+1{!z6s!lNFYNJ(G8Y%z(gKZ8j@rW9HwB?4b)ti=fx!L z0eEi0gb7wGxl@icu9a{oQ$B^);=`AQ@Qk!YB-&8t%%&n{{6V=ILpWWx8D{`a)p7Te zT{^wzxf68WwnqXT4~v8i64w+^cE?8&@!F^gyE%m z3Z@r1n;)WPp=(w)2`)V>;3AhFtyB7CHimIV*>`MAp7ku!R2h<1SAS_^o~7R3MvFUM zihNH;Acd35ihlPEHq~FfhG3aHq0Hq4a*sRgLq2>`M^ScSVN^GFAr|^eSU?FwP4v@h81;?U$+R?6~xZDrMt+DB(fFXgi z9pE^i+oM0a3GP4qh*M|2{P<`o{}8vE&;!S=5uAjDwe8#xDhZ_ajL-8Zkx4aA0HALE zH2E6?QuoM!;Eju9fek*wMn}s9?+9s#`EjolL>#)2;J!x?ZlK5?Z$>5YOOSSe&-2_+ zLJ3n%-Sq77bTZNJZnw4AnwF^!uT1D$pXGd#w#{n2HKD(#Yd*Z{!}lB4p39SP!nK8r zUQEnm6~(aUk`7A6Qf8%KCG#<jH&M- z7{g5`Wd0Pe#i9iRX2PGja?{Cfzv~A;%;ncNm8u)pSnk~#xVDFMH@!;JAvzTR^RG0n zMapehzI=rob2#rV1~8}7Pr{hr<&?Y#2(>8>geUP9QGE}CN+!F3H76m%b!GNu14lTs3meSU=fZRAGra0h*u@jSQGe7lIMUB}X&jvA<)%vYpAH_Tyx5g;~ za81j-%c2;UTE{-=Wopb)lv`bE4?)GJbr^lP%gbb&+s3Ano#~)>2P?OPTwF~{$CkNw z5LXHM@)S0EF_RIB7PRYOqSicFGj{^Ynox`(D zi>n}4kZ6OC!lDZrCiABO>Lv`&Qpdil>p||%M`953GiJ8$NR+ao z@G4Q#OV8GQH|I`#l9T(O_;m$zInNc! zG%84+*^e?;sSiI_{Ny*FBTUTJ(ibwN%kct$ITfpSG;b+|xHqS3T3^6z#~NEo&MDa* zk@oY}zJ-l9(RZ!UFQxO=ju`Xh^z8F_>mo7cX^SA8m*cOm6U=|#~ zlv>)gc2F@G6!33gt91?>3WkV0nlLvLS=^&~IBgIt9EgP!^XT^CdVz4`ajr{#G}1z@ zBurSmEX5F*BB9tBV5Vjkav+y;kBBMBgzFbGB^`w^FJwx#U%94I>6l9Oy-exBU#C*} zdN1Q;Dis2uC^J;7`DgDCn-`{i45J;)mP?{lPG!qMMQJfbjt@98;P}nhOo@@vEAA0( zMi7gxqf%kpdN`in5%}sTx28E_9ywy7z?T*g^J5$HJ@Or-*3h=*Pjc(mg{czsxEF`G zw{U&Fit8zQ)^G}s@&U-@uz+h{ozOq-dkACByElX}za;SWOa5NzLi#dZ06=v08Zk@g z5?!m4z*jjvaQ+HTN{2xBhp&9+I!9QG7u798C*YMXt7=R}joH|C@{*0we}|3nV<%|N zJz)59uR-`-Qz_%T#@ZMeMXrIk>+p0B9&Y9^lZMJd$jpL&5{PTHTa~WpM5Xo%tfAM? zGpkAke}XVPs_`bb-O?4LWVGb365rIee%nb)#pq@u>$d#u_TZGr;XN&aUqOxGjIGLd zUS2FT0M^tPorE!|L0UPb_OC*pUZe)ot>%qxnlf0T*&D)6 z;|ui2=n*!VT+Ode{BFM|fSBVu)}7eg`s)@!HrYlochB39@3t{+LT=EQB8{=wn4R%aP-yb*~106RJ+}IOUtZ=;QOFHc6}5H zzFH@l(wg84-)lwi{b?6c$1Q@V1Yey;F;JhFK>!us8P+rc>?Dr7nn*)vwNuVdfexPM zI>0@JaD<6F+xy>hlOuPXw9AuB5IS*Km1FCBLriuskQkKy)Ix(|4HwXZoJBLYVDK0l zVZ$u+ESQ~+6>Upeb9Z^qx+`hh(L}DIB_r$PG*69rVw|A(4{K@ZXah6C2%SKHoXQ!v z;H15__By9MPi@LX zqn;AJ&H^U*q(=f6x&jQ{F5kIXZ?DX_5ym*cJQfoJxwU#S*j8L5y5(bZ?9H%NYx|r6Kz0_&&3IOrI7(=G*42oeyj>(&Pb-(9m@myFNQJHm55huNUnohD77JN zIBX%~Mj~Ap^W*xR4`b--VGQ36W3*Ghi_Z?OLbGw7eW+YS*6LZk*-I!N_2`KF(elvs znCjMB_rRvw9nW>#nEyikj(v@o5co!CHyFOf`rXseH(6Y~Izrx@Bu%eKuv&2V_pp;~ z0zp$3`1TiwW6t}MS9;T+66FzM@Yy_$`J+~edK>dZlU2!xW#_Gd(i=#~5%UZ#1%%m0 zch{j(t5ewv7rRqVx!37A_nCXmEacE1_7*Wek=XoQjLq+`sa7{I##hp~j*qQtqr2{p z_vs9=2r}Q1zdI6gfNg)P0^WATx0ov}@kwT5Xli4S@5a};QXt^b5XW%q>we;u9wApk z8d2IBEF%|=>R0-=fdgYXUoLomvX%jL24&-gfB4X@~RkFAs zN2MksH53sPQlPK`f*d9=+(UnK?-8zC16K~dl_peZtVv7Vb6C2ipj6j&Y!ciuf*S$D zkTB%PCby6S*FQb&N$Qk;O`{P%Q>%Xw%n;<4G`RYO#qhCpt zvgA60Md{|h&{az8h_<8~rFzyIbo5NOGa!Z`oqJM@yQz~26Qh=1nQCWXhZ{^#E~HDB znwi4|(4cO{+ox!6&D_k4b`*BJspi^{%JJC~L8o-?OL6auQ4C(cl1p4=N_5q@MmW0i z-T$De^yTtd!LRXGA4Or@-E|@k3MfnhsFhK3o8P6CCgbLI+IQfvT~YzH z^CQH;p{XRT0i*s0m|P!%vMgs}L>Ox^CgBT>)P@AKjY6D~P3SXxVsTtmPU!&ryZw1H zQ_^J_e> z$3Q|Y=#sd^TR`V7~aTR$~ekU-ojv zSI1{_PZU*Sbn>bgH@fgj_t1WOJG-WF>LcaFMU1n89xKVpS@#QctZCk`)VRfk|KzCaAORGEAhTRX#( z11fJQ6);eZg503c4U$agV3SwckeA(Q0N9-C22cKIZ7N!=O=W)|?IG{~rshCpd0aM+ z!AuSqcVihgm0W>V8moTCAHfGPcPGkV4ob6CfhLg2lW&()V=DloBKr~{=p=}k6oe}% z3J$sft&p}UAOveWf^`|jM9aFCST0@LkXs|#k$su~hnUkXHRnu3Gn%iigtjdThc{%L zP0(b*rWpz}vs_?{N?(c4w;f~Ds9*AO4S&*=r8mr zS4|}zWyu~Bd=cKv+)sE1;ol|F3E6fxMa!^yus+ORLru_#Rhr6RSSs#K&Op#iIS?b8v+?KRfIQv>}hi(o{8-{Deq3@J-lM5n?RW zin}1!$G0(?N=MUe{}5uT`FEkI#8x&&1hKM=e`k+uR>=jyu|*N+%4F)JQnwU+$Mhom zUP3H0PjY#=i-p`9fZ5mrQ7||SYWKGIfrB1IUEoXhyB-j;s`!3~e@FF_D$y@frI@kF zqrjK`6Nq_zR0*8ZN@^&$rLA#Dw(t{D&fUV;RNTKyL}jpLKD9J6w+u*4;Jb|y|Iu#U zf`KIy=5XL$)!LZzaNYr6NF8AQ7rC|mBDdBr0LB4PZv%$$gNq(t+~vc5=*w< zC_GEGiU1%zVpfo|VtR0vjcNB0dsXJAo*ZT-GPCB^j)W;-wh&^h9&KimLYrLMq{U-< zB}ioGt}sOncf2*Jc6&3)vPqyvDE*r!l*zEENwvFC5j9y+R&zaM-n1LdBP2gGkaSm- z?+(XZd$62F9c-R@=-=%@HJde zlPT?CME|(26iob^cF&?6u`kVi(nIG9a-2`nk%^fmA%X!4fDw*yU~lgrfn*KC*1EsR zJ=z$EVu!umTp;f^spZXEBve(Lx^7Z84Rjbi43k#FGz=zW=DjgK!!-5nhq;l%@M&V; z5BkCR2ec|XKAM|1^0wi+HxG#wb%5u3E`?UDF&8G&SFCO38=KnEM7Ui$c2L*r%*iYC zEzJ|=Wk}X)2y5Uqv&cy5pbO?@zUEPTOA6@W$X&HgujAEb|b3+*jA#~Db z$jX^4hWHYTS@< zyYQ;U^Y5B#l?=%Kf{59dS_F@xjmhNi=t93+m<~y%gj@$F}XFzGwx|WQ90J+tN^i7%Lfrl{fq;sojj6#xYmbEc< zGI#Mhi&wH$N7z2g)IMNlJB;aUSbPI2ZwLGPb{c&bCR4`Wu+EzwHn5M5P2FzC1_ft! zyWwU$5xnqb?oDU@2e;I>Hxs$u+Ww>YXj*uByaK7kIFZJ9CDpHw5iaVH>CzdqF=G9N z5rVsE=gu8ybsF<7@Ms(@&3FDpqXewqI^pLFGzyM~(xg)4mHyyKrDqi1f>)x&`dySN zLC`C{k^w1*I^=zT*es+a-@-+!>phOOw0hu5dwCFOLz;m2N@-NLwK^VDU1$7~t0?)V zdCMDuAUYD13CHHO0otdmv@yHO4pOC4aW&-QuRzSpvuzeJ`UzqpuawR^`kR_a4aJCA zqv1=$*q5p7dPE2Okm&8;KEpUQ)52+p!`?Mq5^1Z`K1qUIN&;uw;Ky$%weqr~7Y(NZ zVqeulJT!~8qbQaLmdB*ob&YG2Si=s81O`D)IZt^SE3#Z0=rx+qxl;EqyT}VfqV3TFD zpspT>_6n@)Q`bzCSex-jr#h-W*?1*P5rYdcF)El463e+=H^R2`W_BW)%s>bL1bTT_ z(h7pk7YaUvRwmcO?gXUpBl%G|W{cxXZRa{K$NbwI-*O)^AjVG-yh%7l2jQ5Uj+tF+ zAsEdGOl+yfpiBoUeQ$@e!4ZAEryxCG8C=GU8u{QE` z0(c0SLgdSJ=v+rSAgKUeBu?cc--kCtKM}ywVSx5|H&ooi%nDi09& z)~tgt1d(e_8GAvkuL)SONRlQ1q78`mQm^)4XZR^d`>I-Ts-9d7V#x0QY)l;af}Kfi z3~Z86zD9=tK<|?h@D`h~vM~xPx~YxPfu7)^7;$%tE*$aKPUC8!_{g!wtg#HO+!(Eu zje&bjEhjGuO>jXiCPyaf9^C@;ZoG!1%9@&v0XqgM%_bVJs>hs;r|a%KoZcgu_^;+l zZ;E`GKa?A94jU$t$QO|?sAP59{q7TG|7GYqsthR$e6r%OdZ*Q;|-6z(w$0agNwm+m2%Fe z4M~6YBzCU=Ge=tECfcd~@t@W>O`7 zYsSW8OSLxUbO2uX<-FpB|98C-rB2wvB1U@SvX~f)7`>|E=6)yfN~^QcbL&^SJUcQv z_EbD^C*w@bHY0HEi1TroniU2+q6r95VMQP;Z?i=_cddclmxwuBhu-B%Z&}?&t%Ld7 zg?te$Gz7jeV7xcU!WgX3$n4g81gQ8OVndZ(m*L48E>#u;w;)*`+~0gqOH@nQrEoN$ zcp_S10$(8IDORx^(g0?^gJGr1XY!tXlO47G}RYU-yUo z?^p4y%98*vvz3+?mI&(*vlML+H54T_bE1Qz=zn!~`iE zjiMZOH?3URB#@5|5%I)raqq4m!8Ujk_`^zr4PnVU;g}ZWu}Mr4Pz61N4wCMh`Kyb% zmeS#RIi8JyIg(n{82)7JZby)vxmFn%a_IZ({!)|)Ujyd(oRW^FPU-*?x&!o&0u0C9 z6|}_fMa!_4)Ioy-R5(hd&Yc{>%)Jbs<_W~1dS*ztR)=&xKuuV61S?FRAPMK;Y=F&n6Vp6H*I#0C*W;mTs zM+2DEvXk#MAb%6}?v;H?Ny9OVc?0z>4|(F#4u3g2Z;g}h9vN15=9^y@b3g$rvp3{tcTeRt@XfhP^(3S>XkuRtRtVMIuCM_*d6=xr1#M z0w(*xzS#bt2^CNr1pWf>tF0U3-;Bq&_geKMT3v{H5TH{-h3>gyxxmtD6HK)PX}$_J zow%a|2h-U#KC>-o| zVUW7J2^O>i=2xE!4IaY${@*@P=Y@2HT@5xyMQ$wtpiJ%c29vN?(7q$IWrSI2Ue=nd zrx7jrY-U)E1G;x(+l|;Bg9x{s7)akia@2MPDzG!1P+BGbVeAWZs}Cko`fo~7Z43{X zBQDwj7ACcL9PL7xq9m0|wk_dGyGdBT7zclKu%V{w6D$cjiqY(BYkvshrjhc^ zIM$&g-6b)8LO??yE9w|+Qcr~2*~XR%G)2g{YneY)mr|dI0n8;gFs}oK{+0m7#V@Y~ zFlssbS^yJYyDlcXL&-1nf>aEg=ZX%tkrazK1iV!MW(O*)CBQtqyV(GYUR#(4R$QnZ zPJ7_*t?MIE>P80;JrB=+K)`T&UC^V_wzd-2>>i0jqAQ0)%MJ-kK6WyJQ%W~1j0Frj zq=#=jLiDn~aXO{lpV?7M%{!;eJmSgZf8m}|SUaG?k{C_ordr^bX^Wk8w?Peeb z-*Hk2DseF7WB^8OGh6GqE-_a;R; z9L}|jyJtlh9%+AQ{YB#uZHg_bq+Y|T##km%K7R`A>>KB7?OpC}M!s6xSpvAOmpRFP8)`o5Tl zPl{gQHSO4%yqbSKYhnO%JRJ>Sj=M6ofq&_ee3kcsXBSfM!_%i84~GZt%!zF-*-Ly7 zLzj_RPPL!vUjS3`Nsq`U(TyO+;X7u-P|F|nfYBj{2{CU(>RqZZ2_!^(cj+Yckx3mY zyB;$xqK$lxmSGVSTOwYAiivA)vhKT`Vp@HK*(eNzMEmi994-nKA7d_hedj=}>PI3a z`WFV1OS0t#6i~oj_`6}nER0KzxaI<4G;m80C7Z+xH@T^b1)@l87zP8FQ)%PuV`X7z zHqB;jI=9yXSW3ztP9xDyvpuZ+cqGank4IseO~+NcE#(jGrr9>Dc08YE^HceoNs_lP z8_FLVvuW#OLf)eyGES~d>_R);kH>m4X~Bz|exwf?g5FYVyC^!)J|q6CP?31 zFeAguA+X^P^T_4qyS!=Hs>If_12i2}i+Bv(s)H#QVg>(9f*K<;)=^=Om;LqF5APuK zK5Yv;p0)+vGBlvy)YVHeu@`# zX^-tXFHi77S9M;FcnPzL4nHpXSmKPOPPgDCTyG!1V#j-SHU9J1aTTS*#y06HJmJm3 zm=%CwRfY0cHi~XIUmy~J#rPsnlbI;CplB^#hRXmU%n*_roRzCRms) zRq!MjiETz~KJL$s)WXc#c|eS4aOb4NB__J%Ok$%em>4Fc1SRNCXHlq{95}|$03d06 zRj;lsoktZ?Z;*kmY$WXt2VwxT?|-ssmIvg$XW+V#Nk-s|5_YC+ceWkI2GUUxT*?@3*~`R4%lN5TCxtW2?i``DwNup2j5-Z(tRpL&03*+M77y>Gi1xgLgl0L@| z(h^HnTWUC7n7u744IWFUMF{1Hxy{fEwcNk2PbXZxKZ^ST` z0JSHjK1a+-BTONc*NYG1bYI9j@o&Zg)K%k3W=duq9z%7q+F^ zbX>re?Q9K?T79!6hQV+paY?$OlVt7EW+#a*F^rDNFuu-PLJZ{869^N|=EEd_&mem$ zk^DW$N#Ci)s;X4G9aYe9G)Jdq26g%nt!V5^qGVauLE8~TiH48CeAAipj&&T$fL7Uo zvyV>*Px{_`vjI61EpMCVdZivLgkh16^nhP0xhffL#<@Wa-f`^4YlBoT6!#|P|4*yL z3!?>$!MewWFlLi!#t5jb(wU6J_+HZ7%&SOaQtc{c!|q^G>eKlh5%2F9c%%dLVblN$ z@-kSMq-aM0142P!V~zeoXIc9hm*sTRLZS*T#0D^_(1%A|Jf`4(idnOHiFbmz@Wx}F zW;h6#nP&c+Su=2ID`(^x(D}F z7<=Gl@Ta8Oum+5O1X@DokJkH})ZMUS2UoA%AD9LShdnzS*gnV*b=^nT?Ye%)b~|DI zSJSSWth769HW~Vx$#?rV+7j1haDv5fL0H!7O)Lz@}O5ID%1!LQ^=%)zmTI$*nRuZ?@5vJF0&r*ec{NYNDCg42{m;WlUVVRI! z+DP>QZ>ZuI%to}MZ80$3l%!V|^Q7>d?=rj)zIHNucOEquvLmkT-3unhMpsKb+}9fz zVoZy{3;9V2CPXk%35LcIOu&oFd2I5sLH>ReOE4P|jD8-$gbax;8Iu08PYpY8nE7=# zPnbLu!xcbA526evvf;dE-2ik+V;js_+R;H83nArgFrmZXb|eXNQ)Zy?!kkhkx#Twm zevo%yy|`9<7^Z$OVXhhm9B;l2A0I;TzUM9f;QdE$LeVR~E6)1PyzmM{IzCfnBVZHG zzz`FGO+1X7h(u8n@8Sk?Scfw=pEFD&0dc$=S?6}(G{BRg62fJ>s)97IEe96u&?@E> zIUI)TX@7hNV0?xY6JeGN>4~_Vt>dkVyzdxad%eg@-3pvik$3s!w}Gu(&Wfm1QZy5J zbdp_@zd)DFPLyx26K@^ICtHGv(r#o!S!W>_y;8q1RMlyGn3(r;Jsp0eRQ9`yyg>b>N+N<*>m*7p;GWjw&ULr4S04ABJwPYE|s|TuK%Ybo7=f0F9 z{r!N6lIj=aGwpyZ>^)xrCgd*nsJ~d~3PjAP!ePlTdhG0FiPdmlzy$RgFwp@mqBV2Q z9>zkV1g~!+DY+iKrj}ZTB4=#_TSw~v(;rWl>v`x)+{{|Qgd#6;3|vyZD%x4VSRR9~ z+4&fT=Y2>09qQB*3lq7dXI#5v8!_XnZNv@@%8_#OfuwMa%81>O;n5)7yr3@eN!l+U za9?K)rbspoE5B>=;Xc!qC6d$fIGa#1fm(Y*UFW*K<8fyIV+_szO;o45Wxq<2bOA7{ zutbGklYiErp);W{d|P*Eyrdw!VXx8&q;G0ChhruAfcAyhQVsQQUjdi{XndS^t0jI# z^0&@cnb?fS6=Z+O+D8!hu&YdH8dq`|XmrZWGvXJ)Z^g08s@Uf3g>n|??z-J~7^W07 zWX=Ay=7|XtXm!sjfbNmeYCBr~pz%D~lxkGaISBkY_yw;H=kX8wb``0NR=)B$%;g8_j)Y3)03&{^| z+}2ni=M*;}F@d>JpS7(}DVV?J6$szTcg$=hOKo zT;3&u`B#@xJfFeWMZgrv->;W>w|Nbj8-YXU2rwFUZ2>@pe;N<@uDY&mp|d1e+l@fdgvQhS*^F1(jiid2IULEw z7?{i9=b8<_B}KBtDe<0tkHKg}1}41a+8Aw$PW?1RijSpmXXhX3C|>YL{pDyv-pl1D zOgkPz7+x2`q|*XizsDmz4`H^ZOI9I_bw~Ap>8$`qCjgy5V!!n;Owkn`Sed=b0n>?2 zL{HuOw)tp2c*~UlMt3N(^jv)@>JcAp(6*!D8Go*sm$NPHKm7BZXiTvC8Lq8jHeB|` zFPY2!=Q@Z{nK$V?`ZDi7EIajN&$-~ZAaA=buwh}Ar=ueiI!jHFOM{ca?8v+Ti`&Mk z{gsg`VU1{S1im6bS!jEk2lX8+sz!RZIoV#BXn<>JF1}uSkfS2)*fVqQSwAA1ff<(t zJxJq{S{Qt6KAVPCR_#vx>2$peyE>h+=YXMl={!b#0h$2jQ2>k`8Y=V!-d5(V>ggCq4m$6l^Uy%+4B$j1x<&|WwM2(q?9w6#*sm(%5P zz3jg&gc!LbYMBOq`@kdjN2NzV!%X? z@$y-!{|aYtL9l&P3w}Ru#N&`geIuFdJ%4S@6=r} zsJTk34R%+qYT$*+0B1Pkw#e3L><2jz;6H|=$}G^qEAWxf3kuDp6BCXC;}mMNh-Mli zehcmlaOF2joAJ{mljijmBeR3=Hg{&4KCm-FU2irgbP=}evh z25*n+2-_?<7j0NANLm2ndXFxv=6WGdEr2NyzhQQ$Wz+>{Y{Eca3L8l9XtLa0g&xdWrbeR@h)jS0<txjr>InSI$nT9#8UKaz@0ugy=zA=EWn)K%q> zE<60~4OIx*D~G%k~gY$)tqgEyt1{>|H~`J|@04QwX> z%yFNWdCP#I&u1Q^R&~^Qv~TnWwyJgBs;0E6^SA&eKAXvV==}cY5sdaM@G@9T3@j$m zSZP%N%M~eIxcv%N+_od5PB04cVMcJd5-em_&F3uI*?1y4;khf5tha9??9k&C7AO%( z+orwFY;0z7GpTnKotNzQCX89FHLo==Ix50MGD1;O?OWCKk+!N`&(XH3qmZ{gseTla zYKxOX$h)*29bkB&YUeW~^BY@)i@moh5&6i#r0!WfgZ5=XPi{<>(Ix*JGK9-*=)|9V zd5jCkn=2cm2uqwYm>F-z$S`4$BNUz6nN^Kc_x%|M@oVAoYl$xdg3+-*ip&eP zBHe4K&7gE{UcQHxU|s+eYk#DR(*hK0c6(3}TIiX0YlYjEhSOs(AG`P?XmoV5&x`#` zi18qI&xXRn9q(Y9guJ`GYeF+%NpO==bAG>%9oUq%F?OM0v}wJHF0w_zbTCRF`(+H# z(z4F(?UU}hhRh(oYsO1-RI}S)^Je}?Cj5o% zkigLqH^_EaF!EyY(2CpN1dC*v0UhQh9`QC&7s5In=fQj) zK-F3Kwdc$>2KK{n^IIPAmQG@Bnva(29vcBYU%ZjxoS3kb;|?0pjin!@t{aX=sM3}L)Zk>exP(WgaDgv1cf2<$`Pb@+tyx9t{C06O+#8F+s#%3 zkIJ^IBHNvfi5dF+{te2Rzob*`lrvvk+UHN_79{By=~bsXZ7AjXLiUXP1_dj!I=&8T zk|5|i)P`ZV8z7e17qv70qjft`Q@2aFD=*aTJg{`^;@mr|3OnBe10BUWB#!HLQc^pb z$h-j|#?S4hK`ddy34Wm`* zRnmmC)Fc-$@d5J90`D8Q+VCbJg*P%+NKJ3^e3KN}Gj}Gmn)JkPH?TU2OWtuz#2pG5 z{$rU(0_G-V^8s^79Hy_$drj25RFNLlUK=WVMXwS_Fvf#SvzjR?)xpjHn3hD<-T4D2 zOxV=SR5Ciwpyce_5FMwcnP)Hc!C*nC#1&kufj62)5PznMk;+xF`wJrGQrZ|E$lqs@WiStOSK~P1~=wb zM~FFM5s#eeCfVrI`rX6+!=tgGUnbX&rP@osgP4C}Fv<0k*cXzBy0E%WubfBmWuR*^FWlQq3MN8G z2xEmBVY6^g82DGO=z%RL2Ad&oO?RbCkkMR8=9YGq72nfoN6f=v_l9L`=!pt(3_>Az zCA!L$=>I%d%F8jS;0LADc_uLXartd?Z~ze=c@pDiI=S?bfB?+hmoOeHxXXxuqX zyxg5jsU=b$^#s-68B(L8Lajqkpu)hyoWB79VlSEAOUb1N+kJ9(-x8UnNqBssgO}A+ix~4=+e)$H%S82>FqDM9CB0Uj>}%gt*mVOLmnxpg zjI4^4XKhzT0{}DS&pKzNV0&ac&Y!FmxV*XsF((I@x8&9DP)4Lah@k;5?}&O|7&qZ| z+~ga?m>3OilptK>U+3K%5AA>yin+7#{>0A=tUXJcFR7-{7Rc2xi7>1rP+l|cYu9P@ zo=u*#>rd~PH~FE7(cd#MKNVlKkcpxgh6_y?FD*3j7})3!L`;X&GWfHhT)aA+(v#2aZ!wZ9D2J8BoRn3Ka z>qQ)FqU0KkYO=rj+LFPAdlRlFv^WZ1tvv3Q=p}r%-$rj7Xk3(4RE%s4Sp&vdEW4{3 zhMpK7Rn^kR-JbM&Sa!bJne^YytbOh79mU?I!;28ovFMKq5r^m90x>O1 z&%N>SZmDX?*08?-Vuq3;I4v}-bvBjn5rc%NhjU5~_gnk$!9}n!Xm0F;JXi#mTHDZ+ z=T3>za%!zB<6dM^Y|$L(R;J`zkkEzPv@!0!-{aHSKwAusQY#>FOZO-%5ARvHE(Kq~ zg{>e~eaJd0)o?mO7<1VDWb@-6C@qv{O7xYcwat_$W=bA3`He5xm}qCYUfhW{8Bwio z6A11DhAK(FIy@FpS9&wLwF&fMj+Qt$a!_MXkYN+6UgS3nmowU!!%qml{@!AonKuaj zIssPp4X?C9!pa0BrYXBzXEx|(wjw(+A5{o~Urd!QM~HlnyCV6Hem&V1$1zL6mxffy zMyQ_PbR zW0J}?lrBgvUT~)}54VD3WQ$$}m2fbss>YmqFfqr|J9sg_V^0OM$7~f??P-LB@=DzS z9AVEVJ|&~O9ovulaGZaUbrlKZc)mh2grAa1u&c-t}+=5Mfcs3+rRy7c;v2w(BF zhNUA=AXcNR23PTW2W&#_&d2X6BCkDoQq-54w2U-mMr^I3cT6D$83CG{a&00(<^6ef`=~L+iHH*gx z*E_$HdLz#+Pdtne82{aoimv*r`LSm67my0&HI>AgzQGaeiA3F9Fez6y-5{K8XT^k< zu`_TnmL)7^o6Ony_6e@Ya$;Is&Q?X$JnS&&CA%N4uhuTM{9ezR+Y-Lchx5PFv!+G! z@&X$ihD~meYEaNvkV?v)!hPYr?4)@i?9bBFd|<5Oos^t*=A-$;yIUu_4>o}H5$uK? zBFg8Fj?Wz1Mw~ad3FilJv*W$zS70`nf0ZlO^F*%HWHfWrP)WP7C;jNXNoP|xMQ$iH zrZ9_}8+g!}bcdMNHRNjkG-Pa?rUML`C~S;PuypIn>|0F=z9e*RV3Qxb1YSTWVO7Pu zeZSuUVD|sSnu;ynT~5^eL-fnI4PkDU+iQP+LqdB80T&t;e1eUD-+Z;O-;`RGz@o(g zYj4ZHQPP}PCd-5IXM#a$xYl|JdEvde1SXE2+D5u><`))g)=rZPjEujb2z4>Ff`&*( zM0%{uyRE6E0;W#{RD;b0%_!<7t&hY0I_&$?clCu5S`2;YFb26@iHQbId?n@)SPpcf zq4d27ChA$!7Y0tiC(UeIz)UCctnFFLG&;%S-?$Q!75rFUYBW=&u4n)Mes*$!(>$R4EyY2w@ zneUH?cU9cpw4sDSFNOG!&U7c_F1RGM+wG{wiR$;l_$T9bq*Q}(ReF$}8uQ2p zvi8=u)(x=_X4hxIg~Eg^&5cbn0~w*oWYvP=*lbIt)_9|7nivi&byan{^Qk9*mrG7@ z_)Zz7IBfE;_xSG_Ho+Q9D8qbhV4}Kz_KSbV?e(bnjrPI15N_}T1Uhax!cd9@u3w(vvBWG|AmQ9E1eiw5*z`xz@p}@ zq=YrILeQ?gk{Mpw(RwO4IRRo0yEh=_H%*&t(5Ree2zm-CJzMHC>cu@)UKPa`Qhyq=%>)q2#Kx$Fi^OCIUcW>!fMg)dO*xOY|Jki zP{KtO0!Ik`*8@uHC^nB(|B2(J{^M}!j{_iPBT;%ej`;;+)^8e`T{7*g<(L>FTWL|b zMT#D8CPQ_ws-S7UhL}qabnyLgrL6fMtlovlmorwSlNeEJGr(jyaGrRt@D)C%Bx3u} z)$;C7vGcrDy`f~*Wq7Jm)*Q}fL`>d*{03rHxsp}Z%)Js_c_oZw+`!5IAoBf!6$4&L zR4Qv`UBdfP@%2~h%quybZv+dWKcXO5Cs&6<25YiaTj`(Mk9wc4dl4IW4TTXx2gM38HyPOZc8Tmr5npv)9 zjo!1&>uO0i0776Hy;4PI7oMK~GCZ^To?5u)1z+H#R#q-p95~|TNC=pfhZ3&y{er;I z?s7hWS0d-1*qGlH`Mw&*a8w+_zr=kn<690eU@EmSCO_`dN$a-x5I*v?n&g;V_TQpJ z#mBOw?Nco15u&BB!J&5WskR<791p+;KlJ-j+uHuR>JrZ(@WJVV7#fIFI^sDX(c2@& z6EdBR;SMr3Bai1G?S~CMc*?v(8&jS+dAK2^q;1XXdP-a=RzM7z0dFmZ9->bw+Mpk7(h(P^#DqH;f0k2A^kr7cZ zdv`bw=HX#jt*!Wmjrk=rCk&b3^jRF^)TZ=St`wy?Ntmm{E733q`G`ymUjW0O07h>EMlU9YJ_}=3 z?B3a~u2=(RDfLFC1kd)}?KbVDF9vgrh!1!R>4>OLb`#) zH(Rr6NeVPZ)#G>VBTZ_=a~*@ZfFrQXw=ZnIciMCba2CS;F7XkN}FSC)Dgm8 zoX(f-^WB9yDlru4_0q@04qX^DFBT|DmAFmD?JFEdD;N~q_s8O3%iE3ZaY>qG*|DJ^ zH4;OY-hkw=e}^X3KawfwKRQ!N`jC@PlVZ=3S7W@r32)}#IXoR7&Af4+Gvo60JT1_9 z?ZGCTv}Q1BZNuG>sfm#G%BsYB571RQ99IJ2-$TqlIC7%@mm?<=turx$y@G- zNUL0CJCa%4OF{B#cCW&9Z69KuXAZhDqe;`Hpyrlp!p+YxiBC+B2^B!emFh;xr@PyB zhg;`#1Uzs?bCDqI2IV^Hu=Hs%Yh3Qv5!O=J(u51RT3GJKI^+OTDq2Q)Rc_s%&wI$1 zF7K(v{DvVVlrb~^7ZIal#N?UMVqy}z5~W4SkzJc4>+V?ll=_v{6-Uqd*#tB*(rVOL zmnofxvq8+|_#UnUUvV8=>Rgz$PWX2;B-aVo;fEF9?@G#}v?k-*3}0CmzKd{@fhW1< zUOd7HBv-BJF&oDk2NjYJ_A-D8L9(HZn5(KWmmaGzWOx$;?-z8g8GB3TI=Xs=jPz=~ zRO`|=+5n8F!-Od6IWjn`UxBEnXO2YWVj$yssc}8=6ED@w zDIG57({(rwZ?Q1Hqo+jeVHuA02D{s>Y@lKF!J#k{{f@iL8g9I883}0vUv~`;k0-C* zi@xV>-ws}&dQ8w8)<}~!C&nWrN`7y#M?=?jz_&!~bmpE5$!z7OC6-1`Nkira){4${ zH5-WcQi=L^_&*0B5Nu(Kx=hJ@GKg90nEi!_dEOzY86t>sYgUeXU-n7G+&T({!?oJE zA0^acXMVDdLc|a=z@Ui?G#=#2q78p_TW9y@)5Rd>xG(8{{Ow&xnsgyE$UKRA1xh7u zGaZ1vpw?Z;Lf>!|(WraUt1Q06Rsqx4$7fPr2gDkw^jjo}>e|8h{MH6G=fDIF)iY7)pX$);PAu|*6jwWRBSY=Ok>1@YhD^?>I zbLmZHeLU~qAjXP@Wi{EBGziMGvncWq!rp~T^+IM{<~i`7dq?XtYwRb%KjwC3T@+(2 z4>WE7=8MC=XNa6qQs1-VWgrO*c`RybEX?`50}FFLzCmH=msl7;3T4j{>jeO%ASN!N zgonwPO7s>!a^ZJ`hEDp@{*g4fjNGDWX4XS`C7C!{`4oyeiDw%yx9v!cH*(UNVG^z; zPcnxA%w?sl`Ny>&%ZT|0T96ADS2pGE6{~!pufuuHhQP$04+#Ra?y8+ z>`Vz#Q;;bmCMF3(XaFTC&0}trg$DB-w7Bme$KrfeYQP~#$eSeR?af3M_`9TVIq1D_ zubem65o3_vj!TYq$PKEt7^6?06Ke>Y=GlV>Sl=)U3>#%x12d0uGO5!@y?zXHX%v92 zPQ7=3+yP?t@99DQVf}%hsP}#&PfDDM@Fct1{y_4E-pIs9#cmulHm*LmHE$A0QW>0$ zSCes1?)s|!!0vPe#Qccg`B+{T9C$xpzO#tg|3?$*hZ4WY@1U>x3|~dee1!<(JY{ z9_MqDxqL}zq7DYJ>3wg17?y5ro8JN)}DY;RaFo2+@D~EcAsk{euu94DD)tyr5XgU zgse`U*Lx8b*0kV9^h!}2!`CLpdnI?e=#{+QGYVTh-ovFr-drJ$nP#PTdUb2lIEI6S zUXWW|jD@~4bGtt{0zoGh1&%5FAay>8CMvFtFrXP(ju)|dP^al|gnezNo_tqR{;!E+ zmgKrZA>3-p*H>fwMUgScZhUyO2sbaLC^jtayJcbWG7I_MLkQ|LjH(d>TdhyKx0smU z!3d{x4?W18ulK?RKdq1Wi68Oy3O@q70*rPY0ENl6ui$cqJ;*)~!W(Kg_(7rmNzH_Ye^%m9MO_D3AbX_wUVfs7$ZWWL{XhI z71@EX!|p8#;kk*i{a<^V|0rpyr>WBSlk0+s!Be7`Td$JqxI?hs(^PLYxP!(5lG>w6 zA^ZeQ$kTce`orZxG)a{to{=>Ggv36zyU+z2*bx1^?3|os&HTaTT^RGj zdSU+4v%JE0X8bDKc9hJ7CkEF&H-3dOCME5G;cye9kW@6E#9Ty}Xs=TB#N7KUYpHpT z(6()x=1SZC+PAZ~Hci`s9NM*NhTse)=6L+a)bIX86Z16gwE`flyyqm8`U@tbXkwf+ z{FaH4>S_JXTAL_Qek;9K<-+t;+qaIOes{W@-_eD9(0@b0_aEZ-I-+3Fh?EvvE3Ew& z3`Y76DX>b$rBoQ+9D&3e9fdZ_Ln`RHX2qbDc+X9dD6mRjFlmxj=4xx%?#X z{ardk$fV+P)zr!Q-X?Cy`nM(+Qimrxy*iWz$vceO7Qdi07g~Ve_{hSVVcF2KR-$xpn}UyxhBiWv5E`}Nme>TxmOBtKNQ!>d?Tcs<1I)Id!0n8si*!EA3DV3T^{A)pnjTV6`wv;zp z>m`@o;QVD;gQOqAd zLco(K#;cE8wnQqEGWG3NhI~WY;7*T?%)?`5~8QZxb43M}jd)?#!RE)wJiNA~X-q)VnY^hXpUvD}J4UObn8OMB*5_fRulaEH9by}U(NYrkVtdTb&O(T&;>q_swYFVIxy(w-8P-Wice`DkAArG-;|&JK z+_MSOMUxTF=I>-OXZV+xxPhmDf*J&*dZa{}U}KC1teN5lIuLyD_DVrzz8i@x%yl+q zMQh<6P{BZ0oDwI8$fRn_MG|mll4=w7g7CP>z)eOyw&M)z+K7P>t>MO;cEbe#bACg; z_mA~SkJpB+sg)Vp^hr9c-Z^1;sSClYvg>bRpQLpsVxM%sW68%kJMWV^Rn{>p>}XpZ zlqVNmXVI}mB%DHR#$j>93c^ccZNvF^GAKE(I|Sbuoz>1MVLg6nTW5&TVgHtEN<$Dq zJ^gOkzxCF+Hu!lqQfqf4Lhcb(9q?S7!qTN$`^%laLlT zZs}`CU^=f2N6fZ93mXF~)N+H$ottpScTIY;r#`gz z!Z}_dTlUGQSTZUxfmfmw++aJ7z~eM{G4Nw`CT4fI><2*1`6qj=pVoUHe_Orx3!cM0 zYB|SF2{4juY*V%;X&#zz6QAzz*Q)DyR0D>d!AHXrZl?vz%G`XvX&n!i0RKDIPK=Kl zr?fvKV)m;sr3bY|*Bmjtz`k?tgIiGtn#R3b^6G-QQ_80Zw%Krxe$Fqq<$8zWW>*FSQ6n#E_w2o5Hd`sm2j^>ewx!3`J(D zrcTHS0dpw!AYY7PcwtNlUWlNwiPFQ7GqW+J2rwHBf*()k^HOT9OAP|Ib~YMQnsUID zmz``i6!WwBRXl;wb`cp)UH#8Q1++6aF($XS`$>-=V{4eWDL1Ah>*$*M!)_1J@8K;= zA@x#>Nm@#L=~^QOY`15l7~==qQpuQ&mJ+rk5);FVEu|L*+i=r~G34dJHtpim)Q8y5 z>xNPk%jRe(VrM}{gtS%T0S}QczP9E^d})bf_<$Ee;M3L%_+ALGHP6S(md4PxeUg6e zlW5r|>205+9bojAK1mmR(rU7e7XahpUKG~E^g0L3-r5#ev!?w01C4M9kh!bF?DUS$ z-Hk&SJaAl-g=^dF!JCHK{aC$;IT_rS?s}a~<|10!Ufar5HE6;q^UI{$b)gHJmDh)MMqv>>-8Aq!zI{M0k^i4q4O$doG;9_f^WZPRTP zHIW#Ou%#Rro!T64wA}fXIdW09=BsEYm>84V*{XAaKP5hKIF{K0jR|6?n0#4~b}b?E zCrBmQt80Ro({6`=`F)wS{;pDD74z&M(=Fc?FYI&)&dqs_HF|KQ#jlZg$gdBD~2tJ=LySHk}|9MX7 zMV14y?>20Vj(9OE8`$H7CG z>t|;}6B5CYGh?cXF^{Ym^B#V$a9Ua$xh&I)Rq^J_!(WU~Pcx zi=ljR?L-Sr$VnjIrfzn256SgqSZRcRr{GKNKYY1-$3yw9pi6~!mp&`seXrpAz4Dzc z_?D_M{+>#j7?5~?(OqHhe5@O^)birECTb`IFLt->O3&ho@u4;*s`G<=0T-z#HL11~ zS1>v*b*d})4g_$s8{VJ+@=GFL_ExXttwv@=(#vC@MW-Ub+*5>Z9M4Tk8CX-qwyE(- zq=&7Lm)%OrEK_vIH9&%m;ntDZN^WQ>MH(R1#ylKiEC>QM#V8&e@nJ ziVoC%=U`((yU6olyw9|d(bmRjXJZ}@#|T`ykQC-M*!GS;NQLeFg9`+X%3`qQz0k1id&C&BDLbU!uGp*=dW2#$0*Gp56$=#U;I8>3Pi6L-4{HYR?h zj%*B%_7c3)#;A}f)l$NDmpyF9xg7UzsNnsU?3%KCh(HBqxV>*OL?vqB#w<2QXI?2_ zAO>HWL))dsPJ+r3#sQbmi?Rd!?i)QPYz7WYvdp?EEMvm)lU2*IDpxulA@V(6-cj@Y z9n-TFHYQ*v1Hsx-;3?{AFGAaYrwB24jmQ*czH2?cF)lw!Oi~( z4K11utkx@KXjo$0XG^ zWca^T1aygGggq({^JN?p?tF%x?A^2*f6CTnKQ}QyKR&XF?Ha_K_RxtWySMCx{2g-y z=#^lZ=)pqT^{*!h}_&45Fgb9W1tz;&62Fh^k0V+rlYj?SKJX{}5DA%a9A zyb*8UpK^pr%qUee=?=5Nk={I5JHd=^5sF4n2`pH*qJ@1+*OtsmjppszYuB*}F80?6 z!RRPP$KE-HxsG1nWCCF z=CnT$K+Gn~H;sJ%_!<4}L4r4K64h2eH;rRNM81SYfI6wxg?9)jSOS+EN!F|=%!@vm zGbONafTtFcbR z9L5H}LkVacL4?WAn}B`#s&OJyiS@%o?JOJ3^?z5xT_OP$@VRXI`o7nX4@I8?M6adc+iZD(-xLe5OO7; zhi6biK-0k-S73ru)N+_-0sL|*$FA+j*o-wJ`orN!%tLY}@0dKXIZ2@$gVmNtGvL*Sf|x)!!_~=#6p=f7!(FjdF~&1+GmL;YlX0rW^z3&MG{)Wah!fNdB}bkb09~ zY{aUJyLr&SrW&0vL|r-Nv^yHa{EW2u%Syfs2iwY9+cMhjDpm6T`)k%OjNZ~igbMf4 z471!XmwbD2fWY^1e360|hLwJoa18x|f(Li@sI-qoeORjP zpgsPv#W$E&QPJF_Ye8{|!`GZo__zOmL4XLaGBClaLG$A zw7vTLiRS865T@Oj`Buo@1QRuWvMS-2{T@xs=?z__Kf4-Jy;BM8BIA}99w39j@T+X3 ze=avMTE)I4b~CZZf!B$}efgIn0P(F^oK{0X6biM^oy{%mRH6$vlNz{i52wjQFqBG4 zsH-=@pN6(pN~A7RxZrv{IJY#Ag%4Blm`t=};?qr`ly zbP``#YWxgyH6~^_p0B%p=-)7UvWj97fz722>F(pUz$W?_otv1)24Hv%Eu0sMF`7CW zSO5(71GLjjr=9s1+)zwSOEo&Z?MpJ1!X;@;x_8UCG08)%^=iG= zGoWN1if(!#YWd*){8)#WJSjm9o&`1Iy_Yg|RcSz#i=<}YE*QWZad}pcLA+>fL0xZMYdyBH* z3u0`cjpH4szC&PxX2%LIvZ)G`%}~D6QEI(azKiS`LgumFSL{=3-?LWR>rSE_T!~Rh zu1U;_zbc+IDLJAAH>DFR0>ui2q7GaP{r+MAb3FcEYQ0cn>_K4-mYgKDio6as zxS&F5vG@PV6L9#I?xh~g8L6ngI0>ESThy3HiG4Oet$GO$kQJAkrPA8NSi`(&pp zX30TvgwIn$6_)rD|{RMN8-e=jqID zCXRQytL8UB;MN?-y@iQ@$9ZNG3P6uaRgDpx_U9fDb6%Sm|BF{fj9w$=LG5yKyHgZ{ z1I$J64!RD(gFD;7E%)2j?xSrd__~aJ_edH9G_x?)e-&Cv)U)pp$Mi|P*EI+p3u4p^ zd&Fr@R8c@T&?}WYYcDgs0 zFoP%B^&hvL+L;&Z=3(Z88nj}Q`9X)<$A{2B0tB0)nC)Ikc+?AH{v_kD7rh5O-CIQr z+WL`E;88G&%h;3z{ckCSWD?p<9i0qN0aa>(#Y!f#0$POpY+J=f!m6k-7ql^#U7hap zf1z#dZ43x!eC?E$vq>K*btBO`#BbvUX``*PDLw^?9KxRt%e&YI;uv^z@YAB;3+rGX z0ds4t8H%!+XDwewQpT3KeFiN_IT6@1OV!AXkuk}(xw@TU0d|#O;?Q6Au;!YqwpV0k^VF)co%;PN--`7duuGl+28K6ycpPkQ=_fh~8QSS5th!wO}*|jEd zwOKElakgUH7mkz^4j)=hYbn0F6Ra!V8Q|1vY}N_jJwi{3y6$gDDO{(4j73ZkzX9Ttz~@Vj_Zy&sj}&qjwErNWU{-%rG4PZ{j-RYvK&8G}qf-?&PlG=TPd8`%W>U zLWAkjBUV=!=qLAc3W1ss8Z2>Qze$aqzUP@|#gsL~^<-`VWiwIprLxB!_)1R1&k85ecYh_Fd5o79Ifi;?kRjQGaf z$!crg{hp^w_&}ATp9^-3NIigzH{;23fHDENS<9^;fuC-o9|Wv7^}8mA?p0h_Yc?-Y5Xr0uJKfrT%U#m zAm;odZp~ogL7+uKO^5=!f~Xt-kl4J#YC()3hAu4x?;a*l#2iLHYr0=B6WL1o?2% z0pq_Rc4eRW)OA;Qt|QGneX52wa5a^{lpw4bSp&kY45$t>X-3rmKcX=g;h!sV9LyUv z@TTw@)igC);lu7o&ez>;cX&hR`j<2b&?9lXnW$i6ieBkiqo`w;fVcm!H$3JRn zmSm}_4|(Y?FyXeLXZBx=WBmHrnq0|ot@P1cD`UXiU8&^&O(?gBOW#e5tLQU|MOz#)OhDZY)MZr+|K=yGGjn_T7$Vr|%HG3nVfIK@b@Ud<&p| zqsg%&*kKU{bi{dY#Y*Yq4IgZxwFp!=N$2q-aiOTBF|A7WZmQBhL2)tH)DajppzSzG zRk^Xa?=LWSeR_59)Q%97A)mo~XNCnfY6f6Lbz_6=E*C6PTFG zN@UYdTsy$zvaCFFO3uW%CG<`D5$0#TQ=$l#FPxHgPN_I? z4X=ExoYL2Q$o=ic`Cf?KVUjxub80FAjAlFdhElS<)Pw9L?QhENUQ#x2IVg27Mjo|4 zU;vxrz_o(J9G`%NCd$&mnKq7MFy3OP?>P20C?HY@RMi|l1{5tr@22`s_#r+%xIiIT zJ?W7fmj(&bhFn}SuH|Y4{{VVAB(giSJSX>lC6CGb3oKjeE{h;5}@m-=A8rSbUV6+8{{-S;t z>G=8-CfEt)sSZ{i#&v$wdevWIZEPBpUzEQlc5s>5S}`{7HrR;;CcY3r6Dk(mY{$ey z)z$An&CI0M@1Sh{7FAXpJF#7*wNLV)iA6w7H70UE8a3^*C3|UH3SFhA<602Hx@%f2@rEGM=_5r8IUq!?60zZ_;Lz$TH%XR;?<|R30L73DY9CJwTA6K26=MlDD2jf-qBbbcP~)Ls?qqD5 ziQ&anHs*M^5QCWg`9He5Z8KA1FJ(%bHYW8g>ATr0-Hvn@_xfW|i+}qeqL10bs){cm zM}wH-@RJ_FuVcqlPh1z+F$J%5&->ade5r-%8=Z0w?YB=5w{cXXMkcFAgY&A|M{qdy z1~I4pC%N@sjF`7pV~U8;`U}MP;3#=z(jQ+TW@|dK+QKAXzGh>{0Xo=VNVFym`h<3*W(q$*!s19fo=6Ih|yc@7?)`ys(g=tJ9f-l zM^Efe{AE1`7g(TFxKam76TrCS8ueK{0nfazG_jIvgp(KOI40Hml!SfNX zn>~;wkg*AYfH!deUbE2C#AfYa!MYKq+eSK>A5N;SrSy~9KvxkZCfIAP2n?G@W+GoY zv8J&C;P7JBj_pV~#$cVgc}OJ)E0b$3_`$CSq*=}OEj z+&XOo3!5pSmkGMhkI2HBe; zza=_6Nv)$O1}U*6pJZ>4y_5M|Npz^>;o{5OI10B&SzWCxF#NW&38 zQ~dPn72orjTnu84$A4VT?4M^-)gq=Yx31fGk|s2ZtI4ij5aZcYQxeXkNL#MvIz38b zvm#p9CL(Z@hm!EwP=3MwHDw-DHEqb#&IG=P)9{9cCo8~U7~}5pzpLU)Us_**^t+vH zD%m}u-0fV$F^)UV7GWGJVOY{F=+ns-3N0s$kvZsXm>*)NIKn}gnARjgoXP+#^ZHiW z2`71%uwa|vz%?SopIJehnkcO@ygx313uxW-|VxPdrO-s`EMr^YbCjR$5&VC$mo;#n-Xt! z0D?86O2g$0Iv@vfd;^&Z6Y9T90Pk5C^Gb^#31dPya=*tg2?t2b9mJ5FdPj z-h!CBzU$TZWBnFOFNO{ke8eIBZUq&zFgQmmXh&<4>)l`wb9e_i=KHA<{gYB9uQlXs z3=fD=l{Th?m?x={?mnb{PmN*C7oKKy=R@QRAYV=4!;}1T<|^$e2<$&XFO0^pc($JE)yCKZG3zMyVfe$@^S_Un8$eRP4c!Zp0&Qo$? zo?E_>7EE>Im@`N}AJ6?eq+@=^7y-aMvN0KAY#+$qb(Xk=l2=Mm;@(0fmjkWN(=I(A zoHmvF<^Q%4m9-uw&!Tqu5*4%^V_F_47QNKCCVR3ofFYFvcz<;m^Qa7CCPa)&l|ma) z>YbDT73BbbPI+WfjR^itf8Z z)5;Wuj?nU1yP-Cv?)lX^Ik-cw}@!y{6_Glw*_EDY2#Yrb#*lR#|0cR}S= z^a1wI8pLeu$oY?PeBB%YOz78De51fO{LZIv35bb8Uwdk2ZBX1qi2PJ*Z5vgg(_o)3 z5UD(>QQa&;MtCICFO$QPVwk86V=nv4>AE}h8zcJvgK7+4S7Z41oYM2rw(1qvH(RJi z*>%mO^-3_0D(wm_uoN5xefJWW3+DF~h)uR_ER`LBwa@|LgZV8sap|4kcjh%fj1W6D z%?wE221TgAj5qp@rDF(tKbnzV$C38}YKID9T^Nkg@ZL`AFhn9FI3EmQrwP_Qwi0$j z6473%r}CP=N8_$%@Xdtzfe}3|8A%5_{wS;x1+!Y!hCJmC993Ng42B2;`s zQ^`@IyiIxf`S6Y~<$7Q)c{+W-|f; z_?ou4pEs^4Ukxdh9g01OU+DK*J_Qs0h%A#vdFo7ESLuLizU9DoMW_0&sLSL2*=!AMyu0!gT^cP$QJj5_C<&4ba=)B&tm^cI;n$Uz#5D~*2?7Ha&-wdaYq@0oK zz+}g}4?1&a7`XZBqiH^<>q(oCNVUy%O+5bPoPVDmgv3`?3TZHQ8zGo7c7 zSV>m%U#B`k%o*~D!7ZH|I&~wNXC~dQbY)JONVp*k1n}@Pi0Q$TSMwB5!e{ddKqcoY z0X#xL`uTh&r6GN5taU^Sr(5M1hZslleFWYEx|>JfIyifR$a;8Jfwi?zzzbA|Z%q-< z(iDL;HXCuwisXxUF{XML?B7sZUJL+7q3VJ;=rW`)r7OyGvL7svhTU#vdS|{E1~u$% z((z_k|Izgj^=gAga|-G$Y8ZyyFzek705l9A(}#IZ57S^Vs(NAm`_xu1T(?(9B^7HW z$2^;aKGC5AOR-VOUajec1$uz@L1R}O92ndf_1VE<*w2r44%-X zNuW0e3Ao>R0(H_GtRzr4BalZb=8GNr6%n5F@yz`miOR=ONgk&ZzKmX|Q7=CFp3&k0 z%<)5)K|m7a5V?fcun?+&@^B&IwcIPn!sz5y8$%$&+NuKHa5^4L9CKL@>HCsz?v?6< zeDS`dCErEP%=IC8RPseL;7Y#MBf0=NAZG2B5K=zv&k&&&g!}Lr8kDt_p{BmN&7jsBPAVN z%qGz)X%{^SfyyxgO1>kjlS~{gI1YI^9N*9+!1nN}N#JaDKn!0u32e*TBgO^2`rq1; z10o>Mi(zHyXS4>AsyIJ!<@_1;NaS`TU97I7w?WWAxzOq)RSx~_A594-bi>!Z@)tfa%{Z0i8fA3=db*06;EJ=5W*wO%5o)EqXivl0~|(7c|HS+0hf2#c85Da zq0X6_E8C%khRVKy&rayxY$Z7X4`y~430LBfSxSur# zjKhRf30*hB%;In|mgUMP`EyHQ**}nlO>JxJEKB@eK8)L=DB0Gwi zD>5B!K#8bGl)%EAVAbF0w3R2l0vPX-9=2v%qQ>ZuCq1IfdJ`}&TNwRwz&L75Ax~P0 zxhmNv>UO(vj?+eY(ciy5LQfR-`nT~{E7!Q}dKVJilR$boOgJOGIkIM|;KD3i%1>kU z#ZqJJTvJ7!WG&48x+DKpouTb{>+kMu^MR~N%t~xUVyoldCp(!u#E&X^;pXwWZ$iw3 zM>T-CoDcnVKfGf>&Tr{j2f#434tmRg$-PIhYwdEUrE9(LAVSxA0~3`u-t~$~3Pfxy zz{C?bt9y_~SVC~PT=&OQeG9T!>~;U{d~r!j3`6+>82pXqJS+C0at| zASI4v;y%+VM2Yl=mh<>Jk<(!AQPhl|#V1MwSYs4bdOWjEQg2Pzvt9!pw4E!5lu#v+ znDhO)REK%hB~jPnepqi}?<{m2BMaj}!gbt#ui>UN7%OpFB1?Wm2*kn*NKTY zSIT?MucRfU@5h;4rH6OZ)x-oS~M6pNQBZFV1%WlHvDYNzI4 zyN@CDdd0w4n5^;Qp}A#XwCz`inAeu=p49D9k97C3Y-%m;ap5lQKCVp*yj#3bpnQnv zL1ZIRBf>wkXJu|=T1p8Mw7Vi}+~DOZ4bEyj5(xJiz?79WA9aB7w7aEUozU*G5Jo@p zu~}dI)yvqd{-#ITD#Ju$vtPBVeMSw(7GQ!}N8y&AQzv zDK}jro;DwobDU)()^M6<7qiMj@A*W``D4zZ`gK`#^}GqU<4NKzlJaSQ>uke4M87d! zlB_x-gFit=a%Nq>LFOe6TXd?ZQ z62bEuTr@CxF&b&r-(n`K^J??UzjUY)DA|W=d!v7tzYR?FTrUH z7~d5tMlk8I04us~bt#0mloN>5}v6eMCOYpy1dYTN7lX)>|qY7(ca5N05!0{|wk^M1o4tp*1yfA_CY)r@H1 znYfU)f48!Qp|MMn+RA~)W78_~PAJJRiNz_|rL%YSf#fO;R4f^3g2eWu#o-nm)2AF; zMtZc^=akn`6p4kQpWEsouXU16mz@b=&d0Cnyu6BU_Zx(J*MQMT`qqmT(a&AdJ@Zg2 z2qXnL+f=wm%d{4pY~ADd2)<^eLxl$)Z^=~xk}C;BEJ&&ekZM2@L{g63moVYCH|n`Fe(9FuGt0K?PXt7tbS?d-q;7&J+!5<%T`BjcXKcD_Xb`DZ;|$@4_jrJ0D!^WVwyR`qtdZVYq1+3Ta2YIc?jGZQGB!ZMkZj&t_~%*Eh`!m$ZFjnQ=Yo$=G-P zj-qNuYkU;G4QZPZU)GJR??-qRUTv;v+6H!1n5!Bn-d)X4W4-{H*0$^6;wqQ4KlbMQ zjel`zcvUl(n#-3BhV`}GyDbdmP+Q6tB=$lpSn>}6$ zIUKJBGw;xO{v8Z(l!?J9(pRf6`K`Z47Zm_e>MdoL)OaMNnvZXan=$9A{2W46+3wi) z*W<9F+x?f>Q^3>a7$bfKm>dBmz}$T!$$xRr1D+c8R8rE}I$noMdQ<@BVm@AvyC3a= z{D;cCKQ}PnB2Sgrc0A(q=2aMFhpQ(TfQ_c{H3x6QW!@L)L|Wz@%nfNiW{#wq5K?K~ zb34A`@1?Z?+m?i`PerA>&*E!()YAgQ(CiNvZ`pq#D5JD!^rm-WHD@`bn+U_@v*`-{aY z;%O*67f+$EJ1%~E8l}`BLDa7cg@ZCmn2UOALYUKVx}N$Kowvroa9#39{15d=ywW4x z1{Asz(bfUG*+j(w%CH+~*U3(T=$A6L8fETGMA9jkKbb%Sgdu0nj(`MYaEyoStW9EI z8jO>qo^iA&#fY1Goj%Bn$}dj%n>Z~y*!E~{`22y_eKa={m-v-P(}epT8{%nrY`Ve^ z-LMHZk-`p0CnqE8s&=H`T~E6r+0Na)S}qLt?7WQsZ2_~AZg@n9p^r?ufx374W)VgQ zE{rR}=%ol#$%P3sY_SPhoJbnNESE*2NSRTI$}#KX2L@%s$k@X-B=fayREbZ#3dA+l zb#7r}Mdb_|IbTnQx6pb2?kQRB%DmL-u2MP)=-;_}2P;Hj*)aGKg%i-`E{i*bL9%I> zrJQ7UGk0YVo=CA-Xa>{=s-&JeZUF--tGsD=!)4F!y?MI7;;@{#Euf!32<`h(wYVz_ zp61H2ajl#bVWJ_%bbw+MX~&TeMqt@6QB1&(+@iL(r1 z3C^PjmEEvu&4jErLZQ>umTpHu1*f0wnnOh;^aWr=`psa4q5c$Y=Sn$n;C~;7>*