feat(query): filter by sequencing run accession; fix NA reaching download --from

CHANGELOG.md

@@ -2,6 +2,28 @@
 
 All notable changes to `atb-cli` are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/).
 
+## [Unreleased]
+
+### Added
+
+- `atb query --runs` and `--run-file` filter by sequencing run accession
+  (`ERR`/`SRR`/`DRR`). ATB is keyed by sample accession, so the runs are
+  resolved through the `run.parquet` mapping table before the query runs, and
+  the resolution is reported: `Resolved 6000 run accession(s) to 5981 sample
+  accession(s)`. A run that carries several samples contributes all of them.
+  Run accessions the table does not list are named in a warning; if none of
+  them resolve the query fails rather than returning every row. Runs and
+  `--samples` combine as a union. `run.parquet` is already a core table, so no
+  extra download is needed.
+
+### Fixed
+
+- `atb download --from` no longer tries to download samples that have no
+  assembly. Their `aws_url` is the literal string `NA` on 464,368 rows, which
+  was turned into a request for `.../NA.fa.gz`. Those rows are skipped and the
+  count is reported: `Skipping 1065 row(s) with no assembly (aws_url is NA)`.
+  Use `atb query --has-assembly` to leave them out of the results instead.
+
 ## [v0.20.0](https://github.com/allthebacteria/atb-cli/releases/tag/v0.20.0) - 2026-08-05
 
 ### Added

docs/guides/download.md

@@ -28,6 +28,22 @@ atb download --url https://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/
 
 Genomes are saved to `--output-dir` when given, otherwise to `download.output_dir` (default `~/atb/genomes`). That is a separate location from the metadata directory (`general.data_dir` / `--data-dir` / `$ATB_DATA_DIR`), so the data-dir setting does not change where downloads land.
 
+## Samples with no assembly
+
+Around 464,000 samples in ATB have no assembly. Their `aws_url` is the literal string `NA`. `atb download --from` skips those rows and reports the count:
+
+```
+Skipping 1065 row(s) with no assembly (aws_url is NA)
+Downloading 4775 file(s) to /home/ubuntu/atb/genomes
+```
+
+To leave them out of the query results in the first place, add `--has-assembly`:
+
+```bash
+atb query --species "Escherichia coli" --has-assembly \
+  --columns sample_accession,aws_url -o results.tsv
+```
+
 ## Related pages
 
 - [Querying genomes](query.md) — filter and select genomes to feed into `atb download`

docs/guides/query.md

@@ -32,6 +32,43 @@ In `--species-like` patterns `%` matches any sequence of characters; every other
 
 Run `atb columns` to list every name `--columns` accepts, with the table each one comes from; the same list is in the [column reference](../reference/columns.md). Names are case-sensitive and an unrecognised name is an error, so a typo stops the query instead of producing a blank column.
 
+## Query by sequencing run accession
+
+ATB is keyed by sample accession (`SAMEA…`, `SAMN…`, `SAMD…`). If you have run accessions instead (`ERR…`, `SRR…`, `DRR…`), `--runs` and `--run-file` translate them for you using the `run.parquet` mapping table:
+
+```bash
+# A few runs given on the command line
+atb query --runs ERR1234567,SRR7654321 --columns sample_accession,sylph_species,aws_url
+
+# A list of runs from a file, one per line
+atb query --run-file runs.txt --columns sample_accession,aws_url
+```
+
+The run accessions are resolved to sample accessions before the query runs, and the resolution is reported:
+
+```
+Resolved 6000 run accession(s) to 5981 sample accession(s)
+5,840 result(s)
+```
+
+Two things to know about the counts:
+
+- **A run may carry more than one sample.** Multiplexed runs map to several sample accessions, so the sample count can exceed the run count.
+- **Not every sample is in ATB.** A run can resolve to a sample the database does not hold, so the result count can be lower than the sample count. Runs that are not in the mapping table at all are listed as a warning. If none of them resolve, the query fails rather than returning the whole database.
+
+To hand a collaborator a download list, add `--has-assembly` so samples with no assembly are dropped:
+
+```bash
+atb query --run-file runs.txt --has-assembly \
+  --columns sample_accession,aws_url -o urls.tsv
+
+atb download --from urls.tsv --workers 8
+```
+
+Without `--has-assembly` the `aws_url` column contains `NA` for samples that were never assembled. `atb download --from` skips those rows and reports how many it skipped.
+
+`--runs` combines with `--samples` as a union, and with every other filter as an AND. `run.parquet` is one of the core tables, so `atb fetch` already downloads it.
+
 ## Filter by geography and platform (requires ENA tables)
 
 ```bash

docs/reference/cli/atb_query.md

@@ -28,6 +28,12 @@ atb query [flags]
   # Filter by country (requires ENA data)
   atb query --species "Salmonella enterica" --country "United Kingdom" --limit 20
 
+  # Look up samples by sequencing run accession
+  atb query --runs ERR1234567,SRR7654321 --columns sample_accession,aws_url
+
+  # Resolve a list of run accessions and keep only those with an assembly
+  atb query --run-file runs.txt --has-assembly --columns sample_accession,aws_url -o urls.tsv
+
   # Use a TOML filter file for reproducible queries
   atb query --filter my_query.toml
 
@@ -56,6 +62,8 @@ atb query [flags]
       --offset int                    number of rows to skip
   -o, --output string                 write output to file instead of stdout
       --platform string               filter by sequencing platform (ENA metadata)
+      --run-file string               file with one run accession per line
+      --runs strings                  comma-separated run accessions (ERR/SRR/DRR), resolved to samples
       --sample-file string            file with one sample accession per line
       --samples strings               comma-separated sample accessions
       --sort-by string                column to sort by

internal/cli/download_cmd.go

@@ -68,10 +68,13 @@ func newDownloadCmd() *cobra.Command {
 			}
 
 			if fromFile != "" {
-				extracted, err := parseCSVURLs(fromFile)
+				extracted, skipped, err := parseCSVURLs(fromFile)
 				if err != nil {
 					return fmt.Errorf("parsing --from file: %w", err)
 				}
+				if skipped > 0 {
+					fmt.Fprintf(os.Stderr, "Skipping %d row(s) with no assembly (aws_url is NA)\n", skipped)
+				}
 				urls = append(urls, extracted...)
 			}
 

internal/cli/parse_csv_urls_test.go

@@ -13,10 +13,13 @@ func TestParseCSVURLsBuildsURLFromAccession(t *testing.T) {
 		"SAMD00000355\tSalmonella enterica\t11\n"
 	path := writeTempFile(t, "mlst-*.tsv", content)
 
-	urls, err := parseCSVURLs(path)
+	urls, skipped, err := parseCSVURLs(path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
+	if skipped != 0 {
+		t.Errorf("skipped: got %d, want 0", skipped)
+	}
 
 	want := []string{
 		"https://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMEA104027617.fa.gz",
@@ -39,13 +42,79 @@ func TestParseCSVURLsPassesThroughAWSURL(t *testing.T) {
 		"SAMEA104027617\thttps://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMEA104027617.fa.gz\n"
 	path := writeTempFile(t, "results-*.tsv", content)
 
-	urls, err := parseCSVURLs(path)
+	urls, skipped, err := parseCSVURLs(path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
+	if skipped != 0 {
+		t.Errorf("skipped: got %d, want 0", skipped)
+	}
 
 	want := "https://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMEA104027617.fa.gz"
 	if len(urls) != 1 || urls[0] != want {
 		t.Fatalf("expected [%q], got %v", want, urls)
 	}
 }
+
+// The index records a sample with no assembly as the literal string NA in
+// aws_url, on 464,368 of its rows. Treating that as a value produced
+// ".../NA.fa.gz" and a download attempt per row. Those samples have no
+// assembly on S3 at all, so the row is skipped and counted.
+func TestParseCSVURLsSkipsRowsWithNoAssembly(t *testing.T) {
+	content := "sample_accession\tdataset\taws_url\n" +
+		"SAMEA103931393\tnot_processed\tNA\n" +
+		"SAMEA10029803\tr0.2\thttps://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMEA10029803.fa.gz\n" +
+		"SAMEA104156871\treject\tNA\n"
+	path := writeTempFile(t, "results-*.tsv", content)
+
+	urls, skipped, err := parseCSVURLs(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	want := []string{"https://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMEA10029803.fa.gz"}
+	if len(urls) != len(want) || urls[0] != want[0] {
+		t.Fatalf("urls: got %v, want %v", urls, want)
+	}
+	if skipped != 2 {
+		t.Errorf("skipped: got %d, want 2", skipped)
+	}
+}
+
+// A file whose aws_url column is NA throughout yields nothing to download.
+// The caller must be able to tell that apart from an empty file.
+func TestParseCSVURLsCountsAnAllNAFile(t *testing.T) {
+	content := "sample_accession\taws_url\nSAMEA103931393\tNA\nSAMEA104156871\tna\n"
+	path := writeTempFile(t, "results-*.tsv", content)
+
+	urls, skipped, err := parseCSVURLs(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(urls) != 0 {
+		t.Errorf("urls: got %v, want none", urls)
+	}
+	if skipped != 2 {
+		t.Errorf("skipped: got %d, want 2", skipped)
+	}
+}
+
+// The NA sentinel also appears in a sample_accession column when a result file
+// carries no aws_url column at all.
+func TestParseCSVURLsSkipsNAInTheAccessionFallback(t *testing.T) {
+	content := "sample_accession\tmlst_st\nNA\t131\nSAMD00000355\t11\n"
+	path := writeTempFile(t, "mlst-*.tsv", content)
+
+	urls, skipped, err := parseCSVURLs(path)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	want := "https://allthebacteria-assemblies.s3.eu-west-2.amazonaws.com/SAMD00000355.fa.gz"
+	if len(urls) != 1 || urls[0] != want {
+		t.Fatalf("urls: got %v, want [%q]", urls, want)
+	}
+	if skipped != 1 {
+		t.Errorf("skipped: got %d, want 1", skipped)
+	}
+}

internal/cli/query_cmd.go

@@ -37,6 +37,8 @@ func newQueryCmd() *cobra.Command {
 		genus              string
 		samples            []string
 		sampleFile         string
+		runAccessions      []string
+		runFile            string
 		hqOnly             bool
 		minCompleteness    float64
 		maxContamination   float64
@@ -68,6 +70,12 @@ func newQueryCmd() *cobra.Command {
   # Filter by country (requires ENA data)
   atb query --species "Salmonella enterica" --country "United Kingdom" --limit 20
 
+  # Look up samples by sequencing run accession
+  atb query --runs ERR1234567,SRR7654321 --columns sample_accession,aws_url
+
+  # Resolve a list of run accessions and keep only those with an assembly
+  atb query --run-file runs.txt --has-assembly --columns sample_accession,aws_url -o urls.tsv
+
   # Use a TOML filter file for reproducible queries
   atb query --filter my_query.toml
 
@@ -126,6 +134,12 @@ func newQueryCmd() *cobra.Command {
 			if cmd.Flags().Changed("sample-file") {
 				filters.SampleFile = sampleFile
 			}
+			if cmd.Flags().Changed("runs") {
+				filters.Runs = runAccessions
+			}
+			if cmd.Flags().Changed("run-file") {
+				filters.RunFile = runFile
+			}
 			if cmd.Flags().Changed("hq-only") {
 				filters.HQOnly = hqOnly
 			}
@@ -164,6 +178,18 @@ func newQueryCmd() *cobra.Command {
 				}
 			}
 
+			if filters.RunFile != "" {
+				if err := filters.LoadRunFile(); err != nil {
+					return fmt.Errorf("loading run file: %w", err)
+				}
+			}
+
+			// Run accessions become sample accessions before an engine is
+			// chosen, so the index and the parquet scan filter the same way.
+			if err := resolveRunFilter(&filters, dir, cmd.ErrOrStderr()); err != nil {
+				return err
+			}
+
 			// Output config overrides
 			if cmd.Flags().Changed("columns") {
 				outCfg.Columns = columns
@@ -336,6 +362,8 @@ func newQueryCmd() *cobra.Command {
 	cmd.Flags().StringVar(&genus, "genus", "", "filter by genus")
 	cmd.Flags().StringSliceVar(&samples, "samples", nil, "comma-separated sample accessions")
 	cmd.Flags().StringVar(&sampleFile, "sample-file", "", "file with one sample accession per line")
+	cmd.Flags().StringSliceVar(&runAccessions, "runs", nil, "comma-separated run accessions (ERR/SRR/DRR), resolved to samples")
+	cmd.Flags().StringVar(&runFile, "run-file", "", "file with one run accession per line")
 	cmd.Flags().BoolVar(&hqOnly, "hq-only", false, "only return high-quality genomes (HQFilter=PASS)")
 	cmd.Flags().Float64Var(&minCompleteness, "min-completeness", 0, "minimum CheckM2 completeness")
 	cmd.Flags().Float64Var(&maxContamination, "max-contamination", 0, "maximum CheckM2 contamination")
@@ -393,22 +421,30 @@ func printSpeciesSuggestions(w io.Writer, species string, candidates []string) {
 	}
 }
 
+// noAssembly is the value the ATB tables use where a sample has no assembly.
+// It appears in aws_url and osf_tarball_url on 464,368 rows of the published
+// index, and those samples have no object on S3 at all.
+const noAssembly = "NA"
+
 // parseCSVURLs reads a CSV or TSV file and extracts download URLs. It prefers
 // the aws_url column; when that is absent (e.g. an mlst result file) it falls
 // back to sample_accession and expands each bare accession into a full S3
 // assembly URL, so downstream download works regardless of which columns the
-// source file carries.
-func parseCSVURLs(path string) ([]string, error) {
+// source file carries. Rows marked NA are skipped and counted, since building
+// a URL from them yields a guaranteed 404.
+func parseCSVURLs(path string) ([]string, int, error) {
+	var skipped int
+
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return nil, skipped, err
 	}
 	defer f.Close()
 
 	// Read first line to detect separator
 	scanner := bufio.NewScanner(f)
 	if !scanner.Scan() {
-		return nil, fmt.Errorf("empty file")
+		return nil, skipped, fmt.Errorf("empty file")
 	}
 	firstLine := scanner.Text()
 
@@ -420,7 +456,7 @@ func parseCSVURLs(path string) ([]string, error) {
 
 	// Re-read from start
 	if _, err := f.Seek(0, 0); err != nil {
-		return nil, fmt.Errorf("seeking file: %w", err)
+		return nil, skipped, fmt.Errorf("seeking file: %w", err)
 	}
 
 	r := csv.NewReader(f)
@@ -429,7 +465,7 @@ func parseCSVURLs(path string) ([]string, error) {
 
 	header, err := r.Read()
 	if err != nil {
-		return nil, fmt.Errorf("reading CSV header: %w", err)
+		return nil, skipped, fmt.Errorf("reading CSV header: %w", err)
 	}
 
 	awsIdx := -1
@@ -444,7 +480,7 @@ func parseCSVURLs(path string) ([]string, error) {
 	}
 
 	if awsIdx == -1 && sampleIdx == -1 {
-		return nil, fmt.Errorf("CSV/TSV file must contain an aws_url or sample_accession column")
+		return nil, skipped, fmt.Errorf("CSV/TSV file must contain an aws_url or sample_accession column")
 	}
 
 	var urls []string
@@ -454,7 +490,7 @@ func parseCSVURLs(path string) ([]string, error) {
 			break
 		}
 		if err != nil {
-			return nil, err
+			return nil, skipped, err
 		}
 
 		col := awsIdx
@@ -463,17 +499,22 @@ func parseCSVURLs(path string) ([]string, error) {
 		}
 		if col < len(record) {
 			v := strings.TrimSpace(record[col])
-			if v != "" {
-				// A sample_accession fallback yields a bare ID; turn it into a
-				// full S3 assembly URL. Values that already carry a scheme
-				// (an aws_url column) pass through untouched.
-				if !strings.Contains(v, "://") {
-					v = buildAssemblyURL(v)
-				}
-				urls = append(urls, v)
+			if v == "" {
+				continue
+			}
+			if strings.EqualFold(v, noAssembly) {
+				skipped++
+				continue
+			}
+			// A sample_accession fallback yields a bare ID; turn it into a
+			// full S3 assembly URL. Values that already carry a scheme
+			// (an aws_url column) pass through untouched.
+			if !strings.Contains(v, "://") {
+				v = buildAssemblyURL(v)
 			}
+			urls = append(urls, v)
 		}
 	}
 
-	return urls, nil
+	return urls, skipped, nil
 }